diff --git a/.env.example b/.env.example index b70d1117..41a484a6 100644 --- a/.env.example +++ b/.env.example @@ -213,10 +213,26 @@ GITLAWB_MAX_CONCURRENT_IPFS_WALKS=32 # per-caller caps via GITLAWB_TRUSTED_PROXY; reject-before-insert bounded map). # Default 4. GITLAWB_IPFS_WALK_PER_SOURCE=4 +# Max legacy (NULL-provenance) repos probed per single /ipfs request, bounding the +# scan-fallback fan-out (git cat-file per candidate repo) for an anonymous caller. A +# truncated scan sheds a retryable 503, never a false 404. Default 256. +GITLAWB_IPFS_MAX_LEGACY_PROBES=256 +# Max repo ROWS one /ipfs request's legacy scan may read from the database. The probe +# ceiling above only counts once a probe runs, and quarantined or private repos are +# denied before that, so without this an all-denying inventory paged the whole repo +# table for one anonymous request. A truncated scan sheds a retryable 503 carrying a +# sealed `continuation` token; echo it as ?scan= to resume, so a holder buried past the +# ceiling is reachable in ceil(repos / ceiling) + 1 requests. Raising this also raises +# the per-caller /ipfs work allowance, since each page is charged to it. Lowering it +# sharpens a coarse oracle: laddering to the end reveals the node's total repo count to +# within one ceiling. Default 2048. +GITLAWB_IPFS_MAX_LEGACY_SCAN_ROWS=2048 # Max EXPENSIVE path-scope visibility walks per single /ipfs request (only a # blob in a path-scoped repo costs a full-history walk). Over-cap repos are # skipped without a verdict and the scan continues; if the object is then found -# nowhere the request sheds a retryable 503 instead of a false 404. Default 64. +# nowhere the request sheds a retryable 503 instead of a false 404. The effective +# cap is the tighter of this value and the node's internal per-request ceiling of +# 17, so values above 17 have no effect; lower values do tighten it. Default 64. GITLAWB_IPFS_MAX_REPOS_WALKED=64 # Ceiling on repos one /ipfs request may VISIT past the visibility gate. Each # visit costs a repo acquire — on a Tigris cache miss a full archive download, @@ -237,6 +253,15 @@ GITLAWB_IPFS_MAX_REPO_VISITS=1024 # Must be 1..=3153600000 (100 years): the node derives an Instant deadline from # this value, and a larger one cannot be represented. Default 600. GITLAWB_IPFS_REQUEST_BUDGET_SECS=600 +# Shorter budget (seconds) for the pre-walk CID resolve: the lookup that maps a +# requested CID to its git oid(s), which runs while the scarce walk admission is +# already held. A well-formed CID with no pin row does no probe and no walk work, +# so without this a stalled lookup could hold a walk slot for the whole request +# budget while nothing walked. The effective deadline is the lesser of this and +# the remaining request budget; walk and probe work stay on the request budget, +# so a slow but progressing scan is never shed by it. +# Must be 1..=3153600000 (100 years). Default 10. +GITLAWB_IPFS_RESOLVE_BUDGET_SECS=10 # Max /ipfs/{cid} requests per client IP per hour (route flood brake, distinct # from the concurrency caps above). 0 disables. Default 600. GITLAWB_IPFS_RATE_LIMIT=600 diff --git a/Cargo.lock b/Cargo.lock index b7050bc6..556d857d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3427,6 +3427,7 @@ dependencies = [ "sha2", "thiserror 2.0.18", "tokio", + "url", "uuid", "zeroize", ] diff --git a/Cargo.toml b/Cargo.toml index b2fd6c07..9b8b4684 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,6 +53,9 @@ chrono = { version = "0.4", features = ["serde"] } uuid = { version = "1", features = ["v4"] } # http client reqwest = { version = "0.12", features = ["blocking", "json", "multipart", "rustls-tls"], default-features = false } +# URL parsing (what reqwest::Url re-exports, so the shared redirect predicate can +# take a parsed URL without pulling reqwest into gitlawb-core) +url = "2" # HMAC hmac = "0.12" diff --git a/README.md b/README.md index 1588161f..a3145340 100644 --- a/README.md +++ b/README.md @@ -224,6 +224,53 @@ Notes: sends the bearer token **only** to your configured origin, never to a URL a node advertises, so a hostile node can't capture the key or redirect the solve. +### Fetching an object by CID + +```bash +gl ipfs list # CIDs this node has pinned +gl ipfs get bafkrei... > object.bin # object bytes on stdout +``` + +Objects that were pinned before the node started recording which repo they came +from are found by scanning its repo inventory, and that scan stops at the +per-request ceilings in the [Configuration](#configuration) table. A stopped scan +answers 503 with a resume token instead of a false "not found", and `gl ipfs get` +follows the token automatically: up to 8 resumes after the first request, so at +most 9 calls to the node, waiting between attempts for as long as the node's +`Retry-After` asks and never longer than 5 seconds. + +The whole ladder runs under a 60 second wall-clock deadline. The deadline bounds +the search, not the download: each attempt gets the time left on it to produce +response headers, and once an object is found its bytes stream outside that +deadline. They are not unbounded, though. The client's own 30 second HTTP timeout +is a total request timeout, running from the start of a request until its body +has finished, so a transfer still going 30 seconds after its request began is cut +off. Waits between attempts never run past the deadline either, so a single run +spends at most around 90 seconds on the network: the deadline plus the 30 second +timeout covering the last attempt. Writing the object out sits outside both +bounds, so piping into a reader that stops reading can hold the command open +longer than that. + +Two node-side brakes end a ladder early and are reported rather than retried +around. A 429 is terminal, because the node's rate-limit window is an hour and +that wait cannot be honored inside one invocation; a transient overload (a 503 +carrying no incomplete-scan code) is retried on the token already held, under the +same cap, clamp and deadline. The per-IP fanout brake can also stop a ladder well +short of the 9 calls, so automatic resumption is not a guarantee of reaching the +object. + +When a bound stops the ladder with a usable token in hand, the command prints the +token and the invocation that continues from it before exiting nonzero: + +```txt +resume from where this stopped: gl ipfs get bafkrei... --scan +``` + +Run that to carry on from where the scan stopped. Re-running without `--scan` +restarts at the first row, reproduces the same truncation and spends the node's +per-IP budget again, so the token is the only thing that makes progress. Tokens +are valid for an hour. + --- ## Architecture @@ -352,9 +399,12 @@ Important node settings: | `GITLAWB_REPO_LEASE_MAX_WAITERS` | Max pushes parked at once waiting for the same repo's write lease. Each waiter pins its buffered pack body, so this bounds that memory for a hot repo; past the cap the newest push sheds a 503 + Retry-After instead of queueing. Pushes to other repos are unaffected, and the lease holder is not counted. Default 8. | | `GITLAWB_MAX_CONCURRENT_IPFS_WALKS` | Max concurrent `GET /ipfs/{cid}` visibility walks across all callers (own pool, disjoint from the served-git pools); over-cap sheds 503. Default 32. | | `GITLAWB_IPFS_WALK_PER_SOURCE` | Max concurrent `/ipfs` walks a single source IP may hold. Default 4. | -| `GITLAWB_IPFS_MAX_REPOS_WALKED` | Max expensive path-scope visibility walks per `/ipfs/{cid}` request; over-cap repos are skipped and the scan continues, shedding a retryable 503 (not a false 404) if the object is then found nowhere. Default 64. | +| `GITLAWB_IPFS_MAX_LEGACY_PROBES` | Max legacy (NULL-provenance) repos probed per `/ipfs/{cid}` request, bounding the scan-fallback fan-out. A truncated scan returns a retryable 503, not a false 404. Default 256. | +| `GITLAWB_IPFS_MAX_LEGACY_SCAN_ROWS` | Max repo rows one `/ipfs/{cid}` request's legacy scan may read from the database. The probe ceiling above only starts counting once a probe runs, and quarantined or private repos are denied before that, so this is what bounds a scan over an inventory that denies the caller everywhere. A truncated scan sheds a retryable 503 carrying an opaque `continuation` token; echoing it as `?scan=` resumes the scan where it stopped. Every per-request ceiling on this path (rows, probes, visits, retained rule bytes) mints one, so each request advances the ladder by at least the rows it read and a holder buried past a ceiling is reached in a bounded number of requests, `ceil(repos / ceiling) + 1` when this row ceiling is the one that binds. No ceiling ever produces a 404. Every page is charged to the caller's `/ipfs` work allowance, so raising this raises that allowance too. Lowering it sharpens an oracle: because a truncation emits a token and a completed wrap does not, laddering to the end reveals the node's total repo count (private and quarantined included) to within one ceiling. Default 2048. | +| `GITLAWB_IPFS_MAX_REPOS_WALKED` | Max expensive path-scope visibility walks a `/ipfs/{cid}` request may run per phase; over-cap repos are skipped and the scan continues, shedding a retryable 503 (not a false 404) if the object is then found nowhere. The effective cap is the tighter of this knob and the node's internal history-walk ceiling of 17 (`MAX_PIN_SOURCES + 1`), so a value above 17 has no effect while a value below it does tighten the cap. It is charged per phase: the provenance lookup and the legacy-scan fallback get separate equal budgets, so one request can run up to twice the cap in total. Default 64. | | `GITLAWB_IPFS_MAX_REPO_VISITS` | Ceiling on repos one `/ipfs/{cid}` request may visit (acquire + probe) past the visibility gate. Also the worst-case per-request Tigris fetch count. On exhaustion the scan stops with a retryable 503. Default 1024. | | `GITLAWB_IPFS_REQUEST_BUDGET_SECS` | Absolute wall-clock budget for one admitted `/ipfs/{cid}` request's acquire+walk lifetime. Per-stage clamps bound the acquire and walk stages to the remaining budget, and no stage starts once it is exhausted; the scan then stops with a retryable 503. The object-type probe and content-read `cat-file` subprocesses are budget-checked before starting and each also run under their own deadline (the lesser of `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` and the remaining budget), reaped via process-group teardown, so a hung `cat-file` cannot hold the request's walk slot past it. One hang path is still unbounded: the probe's object-store readability check is a plain filesystem sweep with nothing to reap, so a wedged filesystem can hold the slot past the deadline. Default 600. Accepted range is 1 to 3153600000 (100 years), since the node derives a deadline from this value and a larger one cannot be represented. | +| `GITLAWB_IPFS_RESOLVE_BUDGET_SECS` | Shorter budget for the pre-walk CID resolve inside an admitted `/ipfs/{cid}` request: the lookup that maps the requested CID to its git oid(s), which runs while the scarce walk admission is already held. A well-formed CID with no pin row does no probe and no walk work, so without this it could hold a walk slot for the whole request budget while nothing walked, and enough such requests shed every real retrieval at admission. The effective deadline is the lesser of this and the remaining request budget, so a value above `GITLAWB_IPFS_REQUEST_BUDGET_SECS` degrades to the request budget. Only the resolve is on this clock; walk and probe work stay on the request budget, so a slow but progressing scan is never shed by it. Default 10. Accepted range is 1 to 3153600000 (100 years). | | `GITLAWB_IPFS_RATE_LIMIT` | Max `/ipfs/{cid}` requests per client IP per hour (route flood brake). 0 disables. Default 600. | | `GITLAWB_TIGRIS_BUCKET` | Optional S3/Tigris shared repo storage bucket. | | `GITLAWB_PINATA_JWT` | Optional Pinata/IPFS warm-storage pinning. | @@ -362,6 +412,8 @@ Important node settings: Production note: change the default Postgres password before exposing a node publicly. +Legacy-pin window: releases before the CID-resolver work stored the provider CID (Kubo dag-pb / Pinata) as a pinned object's resolver key. The `/ipfs/{cid}` resolver now recomputes the raw-content CID from the object bytes and refuses to serve a key that does not match, so `GET /api/v1/ipfs/pins` can still advertise an unrepaired legacy CID that 404s. Such a row is repaired opportunistically the next time a push carries the object again (its key is rewritten to the raw CID, the old value kept in `legacy_provider_cid`), but git negotiation omits objects the node already has, so most legacy rows never re-enter a push delta. A deferred one-shot startup sweep, not this opportunistic path, is what fully retires the advertise-then-404 window. Rows whose object bytes are gone stay withheld. + --- ## Optional node staking diff --git a/crates/git-remote-gitlawb/Cargo.toml b/crates/git-remote-gitlawb/Cargo.toml index b6b9e76c..b704329e 100644 --- a/crates/git-remote-gitlawb/Cargo.toml +++ b/crates/git-remote-gitlawb/Cargo.toml @@ -11,7 +11,7 @@ name = "git-remote-gitlawb" path = "src/main.rs" [dependencies] -gitlawb-core = { path = "../gitlawb-core" } +gitlawb-core = { path = "../gitlawb-core", features = ["redirect"] } anyhow = { workspace = true } reqwest = { workspace = true } tracing = { workspace = true } diff --git a/crates/git-remote-gitlawb/src/main.rs b/crates/git-remote-gitlawb/src/main.rs index 02e39c3e..738839c1 100644 --- a/crates/git-remote-gitlawb/src/main.rs +++ b/crates/git-remote-gitlawb/src/main.rs @@ -199,9 +199,7 @@ fn handle_connect( other => bail!("unsupported git service: {other}"), } - let client = reqwest::blocking::Client::builder() - .timeout(std::time::Duration::from_secs(300)) - .build()?; + let client = build_http_client()?; // ── Phase 1: ref advertisement (GET /info/refs?service=) ───────── // @@ -320,6 +318,58 @@ fn handle_connect( ) } +// ── HTTP client ─────────────────────────────────────────────────────────────── + +/// Total request timeout. A pack transfer can be large and slow, so this is far +/// wider than the CLI's. +const HTTP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(300); + +/// The one client both phases use, with the redirect policy the signing surfaces +/// need. +/// +/// Every request this client sends can carry RFC 9421 `Signature` and +/// `Signature-Input` headers: a push signs from the first request, and a fetch of a +/// private repo signs on the retry. reqwest strips only `Authorization`, `Cookie`, +/// `Proxy-Authorization` and `WWW-Authenticate` across hosts, so under the default +/// `Policy::limited(10)` those signature headers rode a 302 to whatever origin the +/// node named, and on a 307/308 the pack body went with them. Scope the follow to the +/// origin that issued the redirect AND to an identical request-target, which is the +/// same predicate `gl` uses. +fn build_http_client() -> Result { + Ok(reqwest::blocking::Client::builder() + .timeout(HTTP_TIMEOUT) + .redirect(reqwest::redirect::Policy::custom(same_origin_redirect)) + .build()?) +} + +/// Refuse any redirect that leaves the issuing origin or rewrites the request-target, +/// and bound the chain. +/// +/// Refusal is `stop`, not `error`: the 3xx comes back as an ordinary response and +/// the caller reports it through the status path it already has. +/// +/// `Policy::custom` replaces reqwest's built-in limit, so the chain bound is +/// restated. It is not what makes the request finite (`HTTP_TIMEOUT` covers the whole +/// chain); it is what keeps a node that redirects to itself from costing a request +/// per round trip until that timeout. `>` and not `>=` because reqwest pushes the +/// redirecting URL onto `previous` before consulting the policy, which is how +/// `Policy::limited` reads the same counter. +fn same_origin_redirect(attempt: reqwest::redirect::Attempt<'_>) -> reqwest::redirect::Action { + let Some(previous) = attempt.previous().last() else { + // Unreachable through reqwest, which pushes the redirecting URL first, but + // the safe reading of "cannot prove same-origin" is to refuse. + return attempt.stop(); + }; + if attempt.previous().len() > gitlawb_core::redirect::MAX_REDIRECTS { + return attempt.stop(); + } + if gitlawb_core::redirect::may_follow(previous, attempt.url()) { + attempt.follow() + } else { + attempt.stop() + } +} + // ── Smart-protocol request builders ─────────────────────────────────────────── const USER_AGENT: &str = "git/2.0 git-remote-gitlawb/0.1.0"; @@ -858,6 +908,438 @@ mod tests { String::from_utf8_lossy(&buf).into_owned() } + /// The signed headers must not survive a redirect off the node's origin, on + /// EITHER phase. + /// + /// This is the binary git runs for `clone`, `fetch` and `push`. It built its + /// client with a timeout and nothing else, so it ran reqwest's default + /// `Policy::limited(10)`, and reqwest's cross-host header stripping covers + /// `Authorization`, `Cookie`, `Proxy-Authorization` and `WWW-Authenticate` only. + /// `Signature` and `Signature-Input` came straight through. A push signs from the + /// first request, so a hostile node answering 302 was handed a working credential, + /// and a 307/308 would have taken the pack body along. + /// + /// Two mockito servers are two ports on one host, which is the boundary this + /// policy draws. The second server answers everything and expects nothing, with a + /// second mock matching on the `signature` header also at zero, so a followed + /// redirect fails whether or not the signature came with it. Phase 1 is driven + /// with a 302 and Phase 2 with a 308, the status that would carry the body. + /// + /// MUTATION (RED): drop the `.redirect(...)` line from `build_http_client`. + #[test] + fn signed_requests_do_not_follow_a_redirect_off_the_node_origin() { + let kp = Keypair::generate(); + let client = build_http_client().unwrap(); + + let mut elsewhere = mockito::Server::new(); + let never = elsewhere + .mock("GET", mockito::Matcher::Any) + .with_status(200) + .with_body("bytes from the redirect target") + .expect(0) + .create(); + let never_post = elsewhere + .mock("POST", mockito::Matcher::Any) + .with_status(200) + .expect(0) + .create(); + let signature_seen_get = elsewhere + .mock("GET", mockito::Matcher::Any) + .match_header("signature", mockito::Matcher::Any) + .with_status(200) + .expect(0) + .create(); + let signature_seen_post = elsewhere + .mock("POST", mockito::Matcher::Any) + .match_header("signature", mockito::Matcher::Any) + .with_status(200) + .expect(0) + .create(); + + let mut node = mockito::Server::new(); + let repo_base = format!("{}/zOwner/myrepo", node.url()); + let bounce_get = node + .mock("GET", mockito::Matcher::Regex(r"/info/refs".to_string())) + .with_status(302) + .with_header("location", &format!("{}/info/refs", elsewhere.url())) + .expect(1) + .create(); + let bounce_post = node + .mock( + "POST", + mockito::Matcher::Regex(r"/git-receive-pack$".to_string()), + ) + .with_status(308) + .with_header("location", &format!("{}/git-receive-pack", elsewhere.url())) + .expect(1) + .create(); + + let refs_url = format!("{repo_base}/info/refs?service=git-receive-pack"); + let advertisement = build_advertisement_request(&client, &refs_url, Some(&kp)) + .send() + .unwrap(); + + let body = b"0009done\n".to_vec(); + let post_url = format!("{repo_base}/git-receive-pack"); + let pack_post = + build_pack_post_request(&client, &post_url, "git-receive-pack", &body, Some(&kp)) + .body(body.clone()) + .send() + .unwrap(); + + // The other origin first: it is the assertion that names the leak, and it must + // be the one that speaks when a followed redirect makes every one of these + // fail at once. + never.assert(); + never_post.assert(); + signature_seen_get.assert(); + signature_seen_post.assert(); + bounce_get.assert(); + bounce_post.assert(); + + assert_eq!( + advertisement.status(), + 302, + "a refused redirect stops rather than errors, so the caller sees the 3xx" + ); + assert!( + !advertisement + .text() + .unwrap() + .contains("bytes from the redirect target"), + "the redirect target's bytes must never reach the caller" + ); + assert_eq!( + pack_post.status(), + 308, + "the pack POST stops at the redirect too" + ); + } + + /// A same-origin hop that REWRITES the request-target is refused, even though the + /// origin never changes. + /// + /// The signature binds `@path` as the literal path-and-query the helper signed, and + /// the node rebuilds it from the URI it received. An `info/refs` to `info/refs/` + /// bounce therefore presents a signature over a target the node never saw, so a + /// clone or push behind such a proxy 401s. Refusing the hop hands the caller the + /// 3xx that names what happened instead. + /// + /// The target mock expects zero hits and is asserted: mockito only checks + /// `.expect(N)` when `.assert()` runs, so an unbound or unasserted mock passes + /// vacuously. + #[test] + fn a_same_origin_path_changing_redirect_is_refused() { + let kp = Keypair::generate(); + let client = build_http_client().unwrap(); + + let mut node = mockito::Server::new(); + let bounce = node + .mock("GET", "/zOwner/myrepo/info/refs") + .with_status(301) + .with_header("location", "/zOwner/myrepo/info/refs/") + .expect(1) + .create(); + let target = node + .mock("GET", "/zOwner/myrepo/info/refs/") + .with_status(200) + .with_body("bytes from the rewritten target") + .expect(0) + .create(); + + let refs_url = format!("{}/zOwner/myrepo/info/refs", node.url()); + let resp = build_advertisement_request(&client, &refs_url, Some(&kp)) + .send() + .unwrap(); + + assert_eq!( + resp.status(), + 301, + "a refused redirect stops rather than errors, so the caller sees the 3xx" + ); + assert!( + !resp + .text() + .unwrap() + .contains("bytes from the rewritten target"), + "the rewritten target's bytes must never reach the caller" + ); + bounce.assert(); + target.assert(); + } + + /// The other direction, and the only same-origin hop still followed: a redirect + /// that re-issues the IDENTICAL request-target. + /// + /// Without this the policy could be tightened to `Policy::none()` and nothing in + /// this crate would notice. It is also the first executed coverage of the chain + /// bound in `build_http_client`'s policy closure: the route answers 301 pointing at + /// itself, so bounded it is hit once for the original request plus `MAX_REDIRECTS` + /// follows and the call returns the 301 (a refused hop stops rather than errors). + /// Unbounded it would spin until `HTTP_TIMEOUT`, costing the node a request per + /// round trip. Its twin is `a_self_redirect_stops_at_the_chain_bound` in + /// `crates/gl/src/http.rs`. + #[test] + fn an_identical_target_redirect_is_still_followed_up_to_the_chain_bound() { + let kp = Keypair::generate(); + let client = build_http_client().unwrap(); + + let mut node = mockito::Server::new(); + let loop_route = node + .mock("GET", "/zOwner/myrepo/info/refs") + .with_status(301) + .with_header("location", "/zOwner/myrepo/info/refs") + .expect(gitlawb_core::redirect::MAX_REDIRECTS + 1) + .create(); + + let refs_url = format!("{}/zOwner/myrepo/info/refs", node.url()); + let resp = build_advertisement_request(&client, &refs_url, Some(&kp)) + .send() + .expect("the bound must end the chain, not the timeout"); + + assert_eq!( + resp.status(), + 301, + "the chain ends by refusing the next hop, so the last 3xx is what comes back" + ); + loop_route.assert(); + } + + // ── the node's own verification, run over what the helper actually sent ── + + /// What the verifying mock made of a request that reached it. + /// + /// The empty slot (`None`) is its own state and means the mock was never hit at + /// all, which is what the refusal test asserts. It must stay distinguishable from + /// [`Verdict::WrongIdentity`], because "nobody verified anything" and "something + /// verified against the wrong key" are opposite findings. + /// + /// The payloads are read through `Debug` in the assertion messages and nowhere + /// else, which the dead-code pass does not count; they carry the detail that makes + /// a failure legible, so they stay. + #[derive(Debug)] + #[allow(dead_code)] + enum Verdict { + /// The chain accepted the signature AND the key it resolved is the test's DID. + Accepted, + /// The chain refused it. Carries the error so a failure reads as the actual + /// rejection rather than a bare hit count. + Rejected(String), + /// The chain accepted a signature made by somebody else. A key resolved from + /// the parsed `key_id` is read out of the artifact under verification, so an + /// accept on it alone proves consistency, never authenticity. + WrongIdentity { expected: String, got: String }, + } + + /// The node's `require_signature` verification, over a request this crate did not + /// necessarily build: parse the headers, recompute the content-digest from the + /// body, rebuild the signing string over `@method`/`@path`/`content-digest`, + /// Ed25519-verify. Returns the DID the signature resolved to, so a caller can pin + /// the identity. + /// + /// Its twin is the hand-copy in `crates/gl/src/http.rs`, which cannot import this + /// module (this is a binary crate's test module). Keep the two textually identical + /// apart from the mockito seam around them, so an edit to one is visibly an edit to + /// both. + /// + /// The production verifier both copies mirror is `crate::auth::require_signature` in + /// `crates/gitlawb-node/src/auth/mod.rs`. This is a re-implementation, not a call, so + /// an edit to that middleware has to land here too: otherwise the copies drift and + /// this test keeps passing against a rule the node has stopped applying. + /// + /// It asserts internally, which is deliberate but constrains its callers: inside + /// `with_body_from_request` those assertions fire on the server thread and reach + /// the client as a transport error, not as a recorded verdict. So the identity + /// check lives in the caller as a [`Verdict`] variant, never as an assert in here. + fn node_verifies( + method: &str, + path_and_query: &str, + body: &[u8], + sig_input: &str, + sig_header: &str, + content_digest: &str, + ) -> anyhow::Result { + use gitlawb_core::http_sig::{build_signing_string, compute_content_digest, HttpSignature}; + use gitlawb_core::identity::verify; + use std::collections::HashMap; + + let sig = HttpSignature::parse(sig_input, sig_header)?; + sig.check_created()?; + assert!( + sig.missing_components().is_empty(), + "signature must cover all required components" + ); + assert_eq!(sig.alg, "ed25519"); + assert_eq!( + content_digest, + compute_content_digest(body), + "content-digest must match the body" + ); + let vk = sig.key_id.to_verifying_key()?; + let mut values = HashMap::new(); + values.insert("@method".to_string(), method.to_uppercase()); + values.insert("@path".to_string(), path_and_query.to_string()); + values.insert("content-digest".to_string(), content_digest.to_string()); + let sig_params_value = sig_input.strip_prefix("sig1=").unwrap_or(sig_input); + let components: Vec<&str> = sig.components.iter().map(String::as_str).collect(); + let signing_string = build_signing_string(&components, sig_params_value, &values)?; + let sig_array: [u8; 64] = sig.signature_bytes.as_slice().try_into()?; + verify(&vk, signing_string.as_bytes(), &sig_array)?; + Ok(sig.key_id.to_string()) + } + + /// Pull a header value off a received mockito request, or explain which one the + /// helper failed to send. + fn received_header(req: &mockito::Request, name: &str) -> String { + req.header(name) + .first() + .unwrap_or_else(|| panic!("the helper sent no {name} header")) + .to_str() + .unwrap() + .to_string() + } + + /// Run [`node_verifies`] over a GET that arrived at the mock and record what the + /// node would have made of it, pinned to `expected_did`. + fn record_get_verdict( + req: &mockito::Request, + expected_did: &str, + slot: &std::sync::Arc>>, + ) { + let verdict = match node_verifies( + "GET", + req.path_and_query(), + b"", + &received_header(req, "signature-input"), + &received_header(req, "signature"), + &received_header(req, "content-digest"), + ) { + Ok(did) if did == expected_did => Verdict::Accepted, + Ok(did) => Verdict::WrongIdentity { + expected: expected_did.to_string(), + got: did, + }, + Err(e) => Verdict::Rejected(e.to_string()), + }; + *slot.lock().unwrap() = Some(verdict); + } + + /// The finding's repro, now a guard: a rewritten same-origin target must never + /// receive the advertisement's signature, and the proof is the node's own + /// verification, not a hit count. + /// + /// Post-fix the hop is refused, so the slot stays empty. Pre-fix the hop is + /// followed and the slot records the Ed25519 rejection of a signature made over + /// `.../info/refs?service=git-upload-pack` and presented at `.../info/refs/...`, + /// which is the 401 an operator behind such a proxy actually sees. The verdict is + /// asserted first, so a failure speaks about verification rather than about + /// reachability. + /// + /// Its paired positive control is + /// `a_direct_signed_advertisement_verifies_under_the_node_verifier`: without it, an + /// empty slot would be satisfied just as well by a harness that can never record + /// anything. + #[test] + fn a_rewritten_target_never_receives_the_signature() { + let kp = Keypair::generate(); + let expected_did = kp.did().to_string(); + let client = build_http_client().unwrap(); + let slot = std::sync::Arc::new(std::sync::Mutex::new(None::)); + + let mut node = mockito::Server::new(); + let bounce = node + .mock("GET", "/zOwner/myrepo/info/refs?service=git-upload-pack") + .with_status(301) + .with_header( + "location", + "/zOwner/myrepo/info/refs/?service=git-upload-pack", + ) + .expect(1) + .create(); + let recorder = slot.clone(); + let did_for_target = expected_did.clone(); + let target = node + .mock("GET", "/zOwner/myrepo/info/refs/?service=git-upload-pack") + .with_status(200) + .with_body_from_request(move |req| { + record_get_verdict(req, &did_for_target, &recorder); + Vec::new() + }) + .expect(0) + .create(); + + let refs_url = format!( + "{}/zOwner/myrepo/info/refs?service=git-upload-pack", + node.url() + ); + let resp = build_advertisement_request(&client, &refs_url, Some(&kp)) + .send() + .unwrap(); + + let verdict = slot.lock().unwrap().take(); + assert!( + verdict.is_none(), + "the node's own verifier must never see this request: the signature covers \ + /zOwner/myrepo/info/refs?service=git-upload-pack and the rewritten target \ + adds a trailing slash, so what arrives there is a stale request-target; \ + recorded verdict: {verdict:?}" + ); + assert_eq!( + resp.status(), + 301, + "the caller sees the 3xx, not the rewritten target's answer" + ); + bounce.assert(); + target.assert(); + } + + /// The positive control for the test above, and the proof that the helper signs the + /// query it sends. + /// + /// A direct signed advertisement GET, no redirect anywhere, through the same + /// verifying mock. The verdict must be `Accepted`, which is what makes the refusal + /// test's empty slot attributable to the refusal rather than to a harness that + /// cannot record. The advertisement URL carries `?service=`, so a helper that + /// signed the bare path would land here as `Rejected`. + #[test] + fn a_direct_signed_advertisement_verifies_under_the_node_verifier() { + let kp = Keypair::generate(); + let expected_did = kp.did().to_string(); + let client = build_http_client().unwrap(); + let slot = std::sync::Arc::new(std::sync::Mutex::new(None::)); + + let mut node = mockito::Server::new(); + let recorder = slot.clone(); + let did_for_target = expected_did.clone(); + let route = node + .mock("GET", "/zOwner/myrepo/info/refs?service=git-upload-pack") + .with_status(200) + .with_body_from_request(move |req| { + record_get_verdict(req, &did_for_target, &recorder); + Vec::new() + }) + .expect(1) + .create(); + + let refs_url = format!( + "{}/zOwner/myrepo/info/refs?service=git-upload-pack", + node.url() + ); + let resp = build_advertisement_request(&client, &refs_url, Some(&kp)) + .send() + .unwrap(); + assert_eq!(resp.status(), 200); + + let verdict = slot.lock().unwrap().take(); + assert!( + matches!(verdict, Some(Verdict::Accepted)), + "a direct signed advertisement must verify under the node's own chain and \ + resolve to {expected_did}, or the refusal test's empty slot proves \ + nothing; recorded verdict: {verdict:?}" + ); + route.assert(); + } + /// The regression that round-1 missed: the Phase-2 `git-upload-pack` POST was /// left unsigned, so an owner's fetch of a private repo cleared the (now signed) /// advertisement and then 404'd on the pack POST. Drive BOTH request builders @@ -1137,49 +1619,13 @@ mod tests { /// to end (sign here, verify with the node's verifier), not reasoned. #[test] fn client_signature_verifies_under_node_verification_for_both_services() { - use gitlawb_core::http_sig::{build_signing_string, compute_content_digest, HttpSignature}; - use gitlawb_core::identity::verify; - use std::collections::HashMap; - let kp = Keypair::generate(); let client = reqwest::blocking::Client::new(); let body = b"0009done\n".to_vec(); - // Re-implements the node's require_signature verification (auth/mod.rs): - // parse headers, recompute content-digest from the body, rebuild the signing - // string over @method/@path/content-digest, Ed25519-verify. Ok iff the node - // would accept it. - let node_verifies = |method: &str, - path_and_query: &str, - body: &[u8], - sig_input: &str, - sig_header: &str, - content_digest: &str| - -> anyhow::Result<()> { - let sig = HttpSignature::parse(sig_input, sig_header)?; - sig.check_created()?; - assert!( - sig.missing_components().is_empty(), - "signature must cover all required components" - ); - assert_eq!(sig.alg, "ed25519"); - assert_eq!( - content_digest, - compute_content_digest(body), - "content-digest must match the body" - ); - let vk = sig.key_id.to_verifying_key()?; - let mut values = HashMap::new(); - values.insert("@method".to_string(), method.to_uppercase()); - values.insert("@path".to_string(), path_and_query.to_string()); - values.insert("content-digest".to_string(), content_digest.to_string()); - let sig_params_value = sig_input.strip_prefix("sig1=").unwrap_or(sig_input); - let components: Vec<&str> = sig.components.iter().map(String::as_str).collect(); - let signing_string = build_signing_string(&components, sig_params_value, &values)?; - let sig_array: [u8; 64] = sig.signature_bytes.as_slice().try_into()?; - verify(&vk, signing_string.as_bytes(), &sig_array)?; - Ok(()) - }; + // The verification chain is [`node_verifies`], the test-module helper the + // redirect verdict tests share. Same primitives the node's require_signature + // runs, over the request-target this crate transmits. // @path exactly as the node reconstructs it from the request it receives. let path_and_query = |req: &reqwest::blocking::Request| match req.url().query() { Some(q) => format!("{}?{}", req.url().path(), q), diff --git a/crates/gitlawb-core/Cargo.toml b/crates/gitlawb-core/Cargo.toml index 3ba05f1c..67e9ed9c 100644 --- a/crates/gitlawb-core/Cargo.toml +++ b/crates/gitlawb-core/Cargo.toml @@ -22,6 +22,7 @@ multihash-codetable = { workspace = true } cid = { workspace = true } chrono = { workspace = true } uuid = { workspace = true } +url = { workspace = true, optional = true } zeroize = { version = "1", features = ["derive"] } pkcs8 = { version = "0.10", features = ["pem", "std"] } curve25519-dalek = "4" @@ -30,3 +31,9 @@ chacha20poly1305 = "0.10" [dev-dependencies] tokio = { workspace = true } +# Not optional here: the redirect matrix must run under a bare +# `cargo test -p gitlawb-core`, with no feature selected. +url = { workspace = true } + +[features] +redirect = ["dep:url"] diff --git a/crates/gitlawb-core/src/cid.rs b/crates/gitlawb-core/src/cid.rs index b7993cc4..2071d478 100644 --- a/crates/gitlawb-core/src/cid.rs +++ b/crates/gitlawb-core/src/cid.rs @@ -64,6 +64,19 @@ impl Cid { } } +/// True when `s` parses as a CIDv1 with the raw codec — the exact shape +/// [`Cid::from_git_object_bytes`] produces and the `/ipfs` resolver looks up. +/// A legacy provider CID (Kubo dag-pb, Pinata CIDv0) parses to a different +/// version or codec and returns `false`, marking it an opportunistic-repair +/// candidate. Decidable from the string alone (no object bytes), so the pin path +/// can gate the byte-read/recompute cost on it and leave non-legacy rows at the +/// existing DB-only skip cost. An unparseable string is non-canonical (`false`). +pub fn is_raw_cidv1(s: &str) -> bool { + s.parse::>() + .map(|c| c.version() == cid::Version::V1 && c.codec() == RAW) + .unwrap_or(false) +} + impl fmt::Display for Cid { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self.0) @@ -177,6 +190,38 @@ mod tests { assert!(result.is_err()); } + #[test] + fn is_raw_cidv1_classifies_codec_from_string() { + // The canonical resolver key: CIDv1 + raw codec → not a repair candidate. + let raw = Cid::from_git_object_bytes(b"blob 5\0hello"); + assert!( + is_raw_cidv1(raw.as_str()), + "from_git_object_bytes output is CIDv1/raw" + ); + + // A CIDv0 (Pinata dag-pb legacy shape) → repair candidate. + assert!( + !is_raw_cidv1("QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG"), + "a CIDv0 dag-pb value is a legacy-repair candidate" + ); + + // A CIDv1 with the dag-pb codec (the Kubo above-block-size root) over the + // same multihash → still a repair candidate (codec, not just version). + let parsed = raw.as_str().parse::>().unwrap(); + const DAG_PB: u64 = 0x70; + let dagpb = CidGeneric::<64>::new_v1(DAG_PB, *parsed.hash()).to_string(); + assert!( + !is_raw_cidv1(&dagpb), + "a CIDv1 dag-pb value is a legacy-repair candidate" + ); + + // Garbage is non-canonical. + assert!( + !is_raw_cidv1("not-a-cid"), + "an unparseable string is non-canonical" + ); + } + #[test] fn sha256_hex_of_empty_input_is_well_known() { // SHA-256("") is a fixed constant; verifies the hasher is wired correctly. diff --git a/crates/gitlawb-core/src/lib.rs b/crates/gitlawb-core/src/lib.rs index efa99897..d0edec0a 100644 --- a/crates/gitlawb-core/src/lib.rs +++ b/crates/gitlawb-core/src/lib.rs @@ -5,7 +5,16 @@ pub mod encrypt; pub mod error; pub mod http_sig; pub mod identity; +// `url` is the one dependency here that drags a tail (idna, then the icu +// crates), and gitlawb-core is allowlisted to stay embeddable. Every client that +// needs this predicate already parses URLs, so they opt in and nothing else +// pays. `test` is in the cfg so `cargo test -p gitlawb-core` still compiles and +// runs the matrix below with no feature selected; without it the tests would +// silently not run, which is the failure this module exists to prevent. +#[cfg(any(feature = "redirect", test))] +pub mod redirect; pub mod sanitize; +pub mod scan_token; pub mod ucan; pub use error::Error; diff --git a/crates/gitlawb-core/src/redirect.rs b/crates/gitlawb-core/src/redirect.rs new file mode 100644 index 00000000..06460c29 --- /dev/null +++ b/crates/gitlawb-core/src/redirect.rs @@ -0,0 +1,241 @@ +//! Redirect policy shared by every gitlawb HTTP client that signs its requests. +//! +//! Both `gl` (async) and `git-remote-gitlawb` (blocking) attach RFC 9421 +//! `Signature` and `Signature-Input` headers, and reqwest strips only +//! `Authorization`, `Cookie`, `Proxy-Authorization` and `WWW-Authenticate` when a +//! redirect crosses hosts. A signature would survive that hop, and it binds +//! `@method`, `@path` and `content-digest` with no authority component, so a node +//! answering 302 could hand a working credential to a host of its choosing and read +//! as the caller anywhere until the clock-skew window closes. On a 307/308 the +//! request body goes along with it, which for the remote helper is the pack. +//! +//! That same missing authority component is why the predicate also pins the +//! request-target: `@path` is signed as the client sent it and verified as the node +//! received it, so a same-origin hop that rewrites the path or the query (a +//! trailing-slash or query normalization) makes the signature cover a target the +//! node never saw and the read 401s. Only a hop that re-issues the identical target, +//! an http-to-https upgrade being the one that matters in practice, is followed. +//! +//! The decision lives here rather than in either client because the two used to +//! disagree: `gl` was scoped to the origin while the remote helper, the binary that +//! actually runs `git clone gitlawb://`, still ran reqwest's default and followed +//! anywhere. One predicate is what keeps a future third client from repeating that. +//! +//! The type is `url::Url`, which is what `reqwest::Url` re-exports, so both clients +//! pass their attempt URLs straight in. + +/// Longest redirect chain followed. `reqwest::redirect::Policy::custom` replaces +/// reqwest's built-in limit, so the bound has to be restated by every client that +/// installs a custom policy; the value is reqwest's own default. +/// +/// This counts FOLLOWS, matching `Policy::limited`: reqwest pushes the redirecting +/// URL onto `previous` before consulting the policy, and `Limit(max)` refuses once +/// `previous.len() > max`, so a caller comparing against this constant must use `>` +/// too or it permits one hop fewer than it says. +pub const MAX_REDIRECTS: usize = 10; + +/// Follow a redirect only when it stays on the origin that issued it AND re-issues +/// the identical request-target. +/// +/// `Policy::none()` would have been the simpler answer, but one same-origin redirect +/// shape is legitimate here: a node fronted by a proxy that upgrades http to https, +/// or otherwise re-issues the same path and query, so the policy is scoped rather +/// than switched off. +/// +/// Path and query must match exactly, and that is the request-target clause rather +/// than an origin one. `@path` is signed as the client sent it and verified as the +/// node received it, so a hop that rewrites either half leaves a signature covering +/// a target nobody asked for and the node answers 401. Refusing the hop turns a +/// confusing 401 into the 3xx that names what actually happened. One policy covers +/// signed and unsigned callers alike, for the same reason the predicate is shared: +/// two rules would drift. +/// +/// The clause pins `@path`, and only `@path`. A gitlawb signature also covers +/// `@method` and `content-digest`, and both of those are still open on a followed +/// hop: on a 301, 302 or 303, reqwest 0.12.28 delegates to tower-http's +/// `FollowRedirect`, which rewrites a POST to a GET and empties the body +/// (tower-http-0.6.8 `src/follow_redirect/mod.rs:273-285`), while its +/// `drop_payload_headers` removes only `Content-Type`, `Content-Length`, +/// `Content-Encoding` and `Transfer-Encoding`. So `Signature`, `Signature-Input` and +/// `Content-Digest` ride along on a request that no longer has the method or the body +/// they were computed over. Only 307 and 308 preserve both. This predicate returning +/// true therefore makes a GET-shaped hop safe to replay against the node's verifier +/// and says nothing about a bodied one: a signed write must not rely on it alone. +/// +/// `Url::query` is `None` for `/a` and `Some("")` for `/a?`, and those are two +/// different request-targets on the node side too, so the comparison is strict and +/// needs no special case. +/// +/// Host and port must match exactly. Port is compared as `Url::port`, which is +/// `None` for a scheme's default port, so http -> https on the same host compares +/// equal while http -> http on a different port does not. A downgrade from https to +/// http is refused as well: the target is the same host, but the signature would go +/// out in cleartext, which is the same credential leak by a slower route. +/// +/// Host comparison rides on `url`'s parse-time normalization (lowercasing and IDN +/// -> punycode), so the spellings an attacker reaches for do not open a gap. That +/// is a property of the parsed `Url`, not of this function, which is why the test +/// matrix pins it: a move to raw string comparison would silently lose it. +pub fn may_follow(previous: &url::Url, next: &url::Url) -> bool { + let same_origin = next.host_str() == previous.host_str() && next.port() == previous.port(); + let same_target = next.path() == previous.path() && next.query() == previous.query(); + let downgraded = previous.scheme() == "https" && next.scheme() != "https"; + same_origin && same_target && !downgraded +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Every branch of the decision, both ways, plus the spellings that would slip + /// past a comparison less careful than `Url`'s own normalization. + #[test] + fn may_follow_covers_each_origin_branch() { + let url = |s: &str| url::Url::parse(s).unwrap(); + let cases: &[(&str, &str, bool, &str)] = &[ + ( + "http://node.example/a", + "http://node.example/b", + false, + "same origin but the path changed: @path is signed as sent and verified as received", + ), + ( + "http://node.example/a", + "https://node.example/a", + true, + "http to https on one host with an identical request-target: both ports are \ + the scheme default, so the proxy upgrade is still followed", + ), + ( + "https://node.example/a", + "https://node.example/a/", + false, + "trailing-slash normalization changes the request-target, so it is refused", + ), + ( + "https://node.example:8443/a", + "https://node.example:8443/a", + true, + "same explicit port", + ), + ( + "http://node.example/a", + "http://attacker.example/a", + false, + "different host", + ), + ( + "http://node.example/a", + "http://node.example:8080/a", + false, + "same host, different port", + ), + ( + "https://node.example/a", + "http://node.example/a", + false, + "https downgraded to cleartext on the same host", + ), + ( + "https://node.example/a", + "http://node.example:443/a", + false, + "a downgrade dressed up as the https port", + ), + // The rows below pass today because `Url::parse` normalizes the host, not + // because anything here compares case-insensitively or decodes IDN. They + // are the variants an attacker reaches for, so they are pinned: swapping + // this predicate for a raw string comparison must break the suite. + // Each of these pairs an IDENTICAL path on both sides, deliberately. The + // request-target clause below would make every one of them false on the + // path alone, and a row that is false for two reasons has stopped pinning + // either. With the paths equal, the host or port comparison is the only + // thing left that can decide them. + ( + "https://node.example/a", + "https://NODE.EXAMPLE/a", + true, + "same host in a different case: parse lowercases it", + ), + ( + "https://node.example/a", + "https://node.example./a", + false, + "a trailing dot is a different host to url, so the redirect is refused", + ), + ( + "https://exämple.test/a", + "https://xn--exmple-cua.test/a", + true, + "unicode host and its punycode spelling are one host after parse", + ), + ( + "https://node.example/a", + "https://user:pw@node.example/a", + true, + "userinfo is not part of the origin: same host, still followed", + ), + ( + "https://node.example/a", + "https://node.example@attacker.example/a", + false, + "the node's name smuggled into userinfo: the host is the attacker's", + ), + ( + "https://node.example/a", + "https://attacker.example/a#node.example", + false, + "the node's name pushed into the fragment: the host is the attacker's", + ), + // The request-target clause, both directions. `@path` is the only thing a + // gitlawb signature binds the request to, so a hop that rewrites it hands + // the node a signature over a target it never received. + ( + "https://node.example/a?x=1", + "https://node.example/a?x=1", + true, + "identical request-target: the same-origin hop that is still followed", + ), + ( + "https://node.example/a?x=1", + "https://node.example/a?x=2", + false, + "same path but a different query: the request-target covers the query too", + ), + ( + "https://node.example/a", + "https://node.example/a?x=1", + false, + "a query added where there was none", + ), + ( + "http://node.example/a", + "http://node.example/a?", + false, + "an empty query added where there was none: a missing query and an empty \ + one are different request-targets", + ), + ( + "https://node.example/a?x=1", + "https://node.example/a", + false, + "the query dropped where there was one: the comparison is symmetric, and \ + nothing else in the matrix pins that direction", + ), + ( + "https://node.example/a#x", + "https://node.example/a#y", + true, + "a fragment-only difference is still followed: a fragment never reaches \ + the wire, so it is no part of the request-target the node verifies", + ), + ]; + for (previous, next, expected, why) in cases { + assert_eq!( + may_follow(&url(previous), &url(next)), + *expected, + "{previous} -> {next} ({why})" + ); + } + } +} diff --git a/crates/gitlawb-core/src/scan_token.rs b/crates/gitlawb-core/src/scan_token.rs new file mode 100644 index 00000000..f1a6ac81 --- /dev/null +++ b/crates/gitlawb-core/src/scan_token.rs @@ -0,0 +1,499 @@ +//! Sealed continuation token for the node's bounded legacy CID scan (INV-13). +//! +//! The `/ipfs/{cid}` resolver's legacy scan stops at a row ceiling and sheds a +//! retryable 503. To let a holder buried past that ceiling still be reached, the +//! shed carries the scan position so the caller can echo it back and resume. The +//! whole point of the design is that the node keeps NO server-side scan state: the +//! position rides in the caller's token. +//! +//! That makes the token an EMITTED continuation derived from a FETCHED row, and on +//! a scan that served nothing every fetched row is by construction a private or +//! quarantined repo the caller may not read. The row's `created_at` leaks its +//! creation time and its `id` carries the owner's DID, so both halves are withheld +//! fields and the token must be CONFIDENTIAL, not merely tamper-evident: +//! +//! * AEAD-sealed (XChaCha20-Poly1305), never base64-of-plaintext and never +//! signed plaintext. Integrity is not confidentiality. +//! * A fresh `OsRng` nonce on EVERY seal. Under a stream cipher a repeated nonce +//! means repeated keystream, and an attacker who can force the node to seal a +//! position whose plaintext they know XORs two tokens and recovers a withheld +//! row's fields in full, strictly worse than emitting plaintext. +//! * FIXED-WIDTH plaintext. AEAD ciphertext is plaintext-length plus the tag, and +//! every field of a scan position varies in length, so a variable encoding would +//! make token LENGTH a side channel for the sealed row (a short name under a +//! short owner vs a long one) and for the candidate oid's width (40 hex on a +//! sha1 repo, 64 on a sha256 one). Every token this module mints is byte-identical +//! in length. Each field is padded to its own separate width, which is a per-field +//! constant and so still leaks nothing about a given row or candidate. +//! * The canonical CID as associated data, so a token minted while scanning for +//! one CID does not authenticate when replayed against another. +//! +//! Every failure to open (wrong key, tampered bytes, wrong CID, expired, malformed) +//! returns the same `None`. The caller treats that as "no token" and starts at the +//! front, so no failure class is distinguishable and the token is no oracle. + +use base64::{engine::general_purpose::URL_SAFE_NO_PAD as B64URL, Engine}; +use chacha20poly1305::{ + aead::{Aead, KeyInit, OsRng, Payload}, + XChaCha20Poly1305, XNonce, +}; +use rand::RngCore; + +/// Keyset position of the last row a truncated scan fetched. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ScanPosition { + /// The row's raw stored `created_at` text, the first half of the keyset cursor. + pub created_at_key: String, + /// The row's `id`, the tiebreaking half of the keyset cursor. + pub id: String, + /// The oid hex of the CANDIDATE this position resumes. One CID can map to several + /// git oids, so a bare row cursor names a row without naming whose walk it belongs + /// to. The candidate is named by IDENTITY rather than by position in the candidate + /// list: that list is ordered by hex and mutates between rungs, so an index would + /// name a different candidate the moment anything is pinned or unpinned. + pub sha256_hex: String, +} + +/// Plaintext version byte, so a future layout change is a clean open-failure +/// (treated as absent) rather than a misparse. +/// +/// Bumped to 2 when the two halves stopped sharing one width (see [`ID_WIDTH`]), and +/// to 3 when the position gained the candidate oid it resumes (see [`OID_WIDTH`]). +/// A token minted under an earlier layout is a different length and a different +/// framing, so it opens to `None` and the caller restarts at the front, which is the +/// safe direction: a misparse would resume at a fabricated row and skip coverage. +const VERSION: u8 = 3; + +/// Byte width the `created_at` half is padded to. Every value stored here is a +/// serialized timestamp, about 30 bytes, so 64 is roomy for the field's whole domain. +/// It is deliberately NOT widened to match [`ID_WIDTH`]: padding both halves to the id +/// width would nearly double every token for a field that can never use the space. +const CREATED_WIDTH: usize = 64; + +/// Byte width the `id` half is padded to. +/// +/// The bound is set by the WRITERS, not by what a typical id happens to look like. +/// `upsert_mirror_repo` builds `repos.id` as `{owner}/{name}`, and the slug validators +/// in the node's `repo_store` admit an owner of up to 255 bytes and a name of up to +/// 100, so 356 bytes is reachable through the ordinary write path and repo names are +/// peer-controllable. 384 clears that with margin. +/// +/// Under-sizing this is not a cosmetic bug. A row at a truncation boundary whose id +/// exceeds the width fails the seal, the handler sheds a 503 with no continuation, and +/// a tokenless shed is byte-identical to the wrapped-scan response whose contract is +/// "the absence of a token means the ladder is over". The boundary row is deterministic +/// for a stable inventory, so every retry reproduces it and every row past it becomes +/// permanently unreachable. Anything past the width still fails loudly rather than +/// silently truncating a cursor into one that resumes at the wrong row. +const ID_WIDTH: usize = 384; + +/// Byte width the candidate oid half is padded to. +/// +/// Git mints exactly two oid widths and this field carries BOTH. A production repo is +/// created by `store::init_bare` with `git init --bare --object-format=sha1`, so its +/// oids are 40 hex; only the sha256 test fixtures mint 64. The field is therefore +/// length-prefixed like the two row halves rather than a bare fixed 64: a 64-only +/// field would fail every seal on a real deployment, and a failed seal sheds a +/// tokenless 503 that is byte-identical to "your ladder is over". +/// +/// The padding to 64 is what keeps the WIDTH off the wire. Without it a 40-hex token +/// is 24 bytes shorter than a 64-hex one, and token length would say which object +/// format the holder's repo uses. +const OID_WIDTH: usize = 64; + +/// `version | created_len:u16 | created[CREATED_WIDTH] | id_len:u16 | id[ID_WIDTH] | oid_len:u16 | oid[OID_WIDTH] | expires:i64` +const PLAINTEXT_LEN: usize = 1 + 2 + CREATED_WIDTH + 2 + ID_WIDTH + 2 + OID_WIDTH + 8; + +/// Nonce width for XChaCha20-Poly1305. +const NONCE_LEN: usize = 24; + +/// A fresh random 32-byte sealing key from the OS CSPRNG. +pub fn new_key() -> [u8; 32] { + let mut key = [0u8; 32]; + OsRng.fill_bytes(&mut key); + key +} + +// The two halves below are deliberately separate and adjacent. The framing pair owns +// "every token is the same length"; the AEAD pair owns "the contents are confidential +// and CID-bound". Keeping them apart is what lets each property be exercised (and +// broken) without disturbing the other. + +/// Encode a position into the FIXED-WIDTH plaintext: +/// `version | created_len:u16 | created[CREATED_WIDTH] | id_len:u16 | id[ID_WIDTH] | oid_len:u16 | oid[OID_WIDTH] | expires:i64` +/// +/// The padding is the point. AEAD ciphertext is plaintext-length plus the tag, and every +/// field of a scan position varies in length, so a length-prefixed encoding with no +/// padding would make token LENGTH a side channel for the sealed row and for the +/// candidate's object format. Each field is padded to its OWN fixed width, which keeps +/// every minted token the same length while letting the id half carry the range the +/// write path actually admits. +fn encode_position(pos: &ScanPosition, expires_at_unix: i64) -> anyhow::Result> { + let mut out = vec![0u8; PLAINTEXT_LEN]; + out[0] = VERSION; + let mut at = 1; + for (field, width) in [ + (pos.created_at_key.as_bytes(), CREATED_WIDTH), + (pos.id.as_bytes(), ID_WIDTH), + (pos.sha256_hex.as_bytes(), OID_WIDTH), + ] { + if field.len() > width { + // Loud rather than truncating: a clipped cursor resumes at the wrong row and + // silently skips coverage, which is the availability half of the bug this + // token exists to fix. + anyhow::bail!( + "scan token field is {} bytes, over the {width}-byte fixed width", + field.len() + ); + } + out[at..at + 2].copy_from_slice(&(field.len() as u16).to_le_bytes()); + at += 2; + out[at..at + field.len()].copy_from_slice(field); + at += width; + } + out[at..at + 8].copy_from_slice(&expires_at_unix.to_le_bytes()); + Ok(out) +} + +/// Decode what [`encode_position`] wrote. `None` on any structural mismatch. +fn decode_position(bytes: &[u8]) -> Option<(ScanPosition, i64)> { + if bytes.len() != PLAINTEXT_LEN || bytes[0] != VERSION { + return None; + } + let mut at = 1; + let mut fields = [const { String::new() }; 3]; + for (slot, width) in fields.iter_mut().zip([CREATED_WIDTH, ID_WIDTH, OID_WIDTH]) { + let len = u16::from_le_bytes([bytes[at], bytes[at + 1]]) as usize; + at += 2; + if len > width { + return None; + } + *slot = String::from_utf8(bytes[at..at + len].to_vec()).ok()?; + at += width; + } + let expires_at = i64::from_le_bytes(bytes[at..at + 8].try_into().ok()?); + let [created_at_key, id, sha256_hex] = fields; + // A zero-length candidate is a third state the encoder never mints. The front-of-table + // sentinel is empty ROW halves with a real oid, so an empty oid would hand the resume + // path a candidate that names nothing; refuse it like every other malformed frame. + if sha256_hex.is_empty() { + return None; + } + Some(( + ScanPosition { + created_at_key, + id, + sha256_hex, + }, + expires_at, + )) +} + +/// AEAD-seal `plaintext` under `key`, bound to `cid`, framed as `nonce || ciphertext`. +fn seal_bytes(key: &[u8; 32], cid: &str, plaintext: &[u8]) -> anyhow::Result> { + let cipher = XChaCha20Poly1305::new_from_slice(key) + .map_err(|e| anyhow::anyhow!("scan token key: {e}"))?; + // A FRESH nonce per seal, from the OS CSPRNG. Under a stream cipher a repeated nonce + // repeats the keystream, and two tokens sealed under one nonce XOR to the difference + // of their plaintexts, which recovers a withheld row in full when the attacker can + // force one of the two positions. This draw is the property the whole confidentiality + // claim rests on. + let mut nonce = [0u8; NONCE_LEN]; + OsRng.fill_bytes(&mut nonce); + let sealed = cipher + .encrypt( + XNonce::from_slice(&nonce), + Payload { + msg: plaintext, + // The canonical CID as associated data: a token minted while scanning for + // one CID does not authenticate against another, so it cannot be replayed + // to seed a different scan. + aad: cid.as_bytes(), + }, + ) + .map_err(|e| anyhow::anyhow!("scan token seal: {e}"))?; + let mut out = Vec::with_capacity(NONCE_LEN + sealed.len()); + out.extend_from_slice(&nonce); + out.extend_from_slice(&sealed); + Ok(out) +} + +/// Open what [`seal_bytes`] framed. `None` on any failure, including a wrong `cid`. +fn open_bytes(key: &[u8; 32], cid: &str, raw: &[u8]) -> Option> { + if raw.len() <= NONCE_LEN + 16 { + return None; + } + let (nonce, sealed) = raw.split_at(NONCE_LEN); + let cipher = XChaCha20Poly1305::new_from_slice(key).ok()?; + cipher + .decrypt( + XNonce::from_slice(nonce), + Payload { + msg: sealed, + aad: cid.as_bytes(), + }, + ) + .ok() +} + +/// Seal `pos` under `key`, bound to `cid`, expiring at `expires_at_unix`. +/// +/// Returns the base64url (no pad) token. Errors only when a field exceeds +/// its own fixed width ([`CREATED_WIDTH`], [`ID_WIDTH`], [`OID_WIDTH`]) or the AEAD +/// itself fails, never silently truncates. +pub fn seal_scan_token( + key: &[u8; 32], + cid: &str, + pos: &ScanPosition, + expires_at_unix: i64, +) -> anyhow::Result { + let plaintext = encode_position(pos, expires_at_unix)?; + Ok(B64URL.encode(seal_bytes(key, cid, &plaintext)?)) +} + +/// Open a token minted by [`seal_scan_token`] under the same key and CID. +/// +/// `None` for every failure class alike (wrong key, tampered, foreign CID, expired, +/// malformed, wrong version), so the caller can treat all of them as "absent" without +/// leaking which one occurred. +pub fn open_scan_token( + key: &[u8; 32], + cid: &str, + token: &str, + now_unix: i64, +) -> Option { + let raw = B64URL.decode(token).ok()?; + let plaintext = open_bytes(key, cid, &raw)?; + let (pos, expires_at) = decode_position(&plaintext)?; + if now_unix >= expires_at { + return None; + } + Some(pos) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A production-shaped candidate: `git init --bare --object-format=sha1`, so 40 hex. + const OID_40: &str = "a1b2c3d4e5f60718293a4b5c6d7e8f9012345678"; + /// A test-fixture-shaped candidate: the sha256 repos the suite creates mint 64 hex. + const OID_64: &str = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + + fn pos(created: &str, id: &str) -> ScanPosition { + pos_for(created, id, OID_40) + } + + fn pos_for(created: &str, id: &str, sha256_hex: &str) -> ScanPosition { + ScanPosition { + created_at_key: created.to_string(), + id: id.to_string(), + sha256_hex: sha256_hex.to_string(), + } + } + + /// Seal a hand-built plaintext through the AEAD half, so a test can frame bytes the + /// encoder would never mint (an old version byte, a zero-length oid) and still exercise + /// the real open path. + fn seal_raw(key: &[u8; 32], cid: &str, plaintext: &[u8]) -> String { + B64URL.encode(seal_bytes(key, cid, plaintext).unwrap()) + } + + /// Byte offset of the oid length prefix inside the plaintext, derived from the widths + /// rather than hardcoded so a width change moves it with the layout. + const OID_LEN_AT: usize = 1 + 2 + CREATED_WIDTH + 2 + ID_WIDTH; + + /// Scenario 1: both of git's oid widths round trip. The 40-hex case is the PRODUCTION + /// shape (`git init --bare --object-format=sha1`), and an all-64 fixture suite would + /// never exercise it, which is exactly how a fixed-64 field would ship broken. + #[test] + fn round_trips_at_both_oid_widths() { + let key = new_key(); + for hex in [OID_40, OID_64] { + let p = pos_for("2020-01-01T00:00:03+00:00", "z6MkOwner/private-repo", hex); + let t = + seal_scan_token(&key, "bafkcid", &p, 1 << 40).expect("both oid widths must seal"); + assert_eq!( + open_scan_token(&key, "bafkcid", &t, 0), + Some(p), + "a {}-hex candidate must open to the identical position", + hex.len() + ); + } + } + + /// Scenario 2: a token framed under the OLD version opens to `None`, never a misparse. + /// Both legs matter: the version-2 plaintext was a different LENGTH, and a future + /// same-length layout would only be caught by the version byte itself. + #[test] + fn a_prior_layout_version_opens_to_none() { + let key = new_key(); + + // The version-2 layout verbatim: no oid field, so 461 bytes. + let mut old = vec![0u8; 1 + 2 + CREATED_WIDTH + 2 + ID_WIDTH + 8]; + old[0] = 2; + assert_eq!(old.len(), 461, "the version-2 plaintext was 461 bytes"); + assert_eq!( + open_scan_token(&key, "bafkcid", &seal_raw(&key, "bafkcid", &old), 0), + None, + "a version-2 token must open to None so the caller restarts at the front" + ); + + // Same length, stale version byte: only the version check can refuse this one. + let mut stamped = encode_position( + &pos("2020-01-01T00:00:03+00:00", "z6MkOwner/private-repo"), + 1 << 40, + ) + .unwrap(); + stamped[0] = VERSION - 1; + assert_eq!( + open_scan_token(&key, "bafkcid", &seal_raw(&key, "bafkcid", &stamped), 0), + None, + "a stale version byte must open to None even at the current width" + ); + } + + /// Scenario 3: token length is invariant across candidate VALUE and candidate WIDTH. + /// The padding is what keeps the oid width off the wire: without it a 40-hex token is + /// 24 bytes shorter than a 64-hex one and the length says which repo format the + /// holder uses. + #[test] + fn token_length_is_invariant_across_oid_widths() { + let key = new_key(); + let created = "2020-01-01T00:00:00+00:00"; + let id = "z6MkOwner/private-repo"; + let short = + seal_scan_token(&key, "bafkcid", &pos_for(created, id, OID_40), 1 << 40).unwrap(); + let long = + seal_scan_token(&key, "bafkcid", &pos_for(created, id, OID_64), 1 << 40).unwrap(); + assert_eq!( + short.len(), + long.len(), + "a 40-hex and a 64-hex candidate must mint tokens of identical length, or the \ + oid width is a side channel" + ); + + // The absolute width, pinned by execution rather than by arithmetic on paper. The + // gl client's mock fixtures hardcode this number (`TOKEN_LEN` in + // crates/gl/src/ipfs_cmd.rs), and nothing in that crate seals a real token, so this + // assertion is the only executable check that the two agree. + assert_eq!( + short.len(), + 756, + "24 nonce + {PLAINTEXT_LEN} plaintext + 16 tag, base64url no pad" + ); + } + + /// Scenario 4: the front-of-table sentinel. Empty row halves with a REAL candidate + /// round trip, which is what lets a seal say "this candidate, no row cursor yet". + #[test] + fn empty_row_fields_round_trip_with_a_real_candidate() { + let key = new_key(); + let p = pos_for("", "", OID_40); + let t = + seal_scan_token(&key, "bafkcid", &p, 1 << 40).expect("the front sentinel must seal"); + assert_eq!(open_scan_token(&key, "bafkcid", &t, 0), Some(p)); + } + + /// Scenario 6, third leg: a zero-length oid is a distinguishable third state that the + /// encoder never mints, and accepting it would hand the sentinel machinery a candidate + /// naming nothing. The decode path refuses it. + #[test] + fn a_zero_length_candidate_opens_to_none() { + let key = new_key(); + let mut plaintext = encode_position( + &pos("2020-01-01T00:00:03+00:00", "z6MkOwner/private-repo"), + 1 << 40, + ) + .unwrap(); + plaintext[OID_LEN_AT..OID_LEN_AT + 2].copy_from_slice(&0u16.to_le_bytes()); + assert_eq!( + open_scan_token(&key, "bafkcid", &seal_raw(&key, "bafkcid", &plaintext), 0), + None, + "a zero-length candidate must open to None, not to a position naming nothing" + ); + } + + /// Scenario 6, second leg: an oid past the 64-byte width fails the seal loudly rather + /// than being clipped into a hex that names a different candidate. + #[test] + fn an_oid_over_the_fixed_width_fails_loudly() { + let key = new_key(); + let p = pos_for( + "2020-01-01T00:00:03+00:00", + "z6MkOwner/private-repo", + &"a".repeat(OID_WIDTH + 1), + ); + assert!( + seal_scan_token(&key, "bafkcid", &p, 1 << 40).is_err(), + "an over-wide candidate must fail the seal, never be truncated into a hex that \ + resumes the wrong candidate" + ); + } + + #[test] + fn round_trips_under_the_same_key_and_cid() { + let key = new_key(); + let p = pos("2020-01-01T00:00:03+00:00", "z6MkOwner/private-repo"); + let t = seal_scan_token(&key, "bafkcid", &p, 1 << 40).unwrap(); + assert_eq!(open_scan_token(&key, "bafkcid", &t, 0), Some(p)); + } + + #[test] + fn every_failure_class_opens_to_none() { + let key = new_key(); + let other = new_key(); + let p = pos("2020-01-01T00:00:03+00:00", "z6MkOwner/private-repo"); + let t = seal_scan_token(&key, "bafkcid", &p, 1 << 40).unwrap(); + + assert_eq!(open_scan_token(&other, "bafkcid", &t, 0), None, "wrong key"); + assert_eq!( + open_scan_token(&key, "bafkOTHER", &t, 0), + None, + "foreign CID" + ); + assert_eq!( + open_scan_token(&key, "bafkcid", &t, 1 << 41), + None, + "expired" + ); + assert_eq!(open_scan_token(&key, "bafkcid", "!!not b64", 0), None); + assert_eq!(open_scan_token(&key, "bafkcid", "", 0), None); + let mut flipped: Vec = t.bytes().collect(); + let last = flipped.len() - 1; + flipped[last] = if flipped[last] == b'A' { b'B' } else { b'A' }; + assert_eq!( + open_scan_token(&key, "bafkcid", &String::from_utf8(flipped).unwrap(), 0), + None, + "tampered" + ); + } + + /// The id half must clear the LARGEST repo id the node's own write path admits, + /// not merely a typical one. `upsert_mirror_repo` builds `repos.id` as + /// `{owner}/{name}`, and the slug validators in `repo_store` admit 255 bytes of + /// owner and 100 of name, so 356 is reachable. A width under that turns the + /// boundary row into a seal failure, which sheds a tokenless 503 that is + /// byte-identical to "your ladder is over" and strands every row past it forever. + #[test] + fn round_trips_a_repo_id_at_the_write_paths_maximum() { + let key = new_key(); + let id = format!("{}/{}", "o".repeat(255), "n".repeat(100)); + assert_eq!(id.len(), 356, "255 owner + '/' + 100 name"); + let p = pos("2020-01-01T00:00:03+00:00", &id); + let t = seal_scan_token(&key, "bafkcid", &p, 1 << 40) + .expect("a repo id the write path admits must seal, never fail the width"); + assert_eq!(open_scan_token(&key, "bafkcid", &t, 0), Some(p)); + } + + #[test] + fn a_field_over_the_fixed_width_fails_loudly() { + let key = new_key(); + let p = pos("2020-01-01T00:00:03+00:00", &"x".repeat(ID_WIDTH + 1)); + assert!( + seal_scan_token(&key, "bafkcid", &p, 1 << 40).is_err(), + "an over-wide field must fail the seal, never be truncated into a cursor \ + that resumes at the wrong row" + ); + } +} diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index df7a42db..92d12980 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -1,14 +1,17 @@ //! GET /ipfs/{cid} — content-addressed retrieval of git objects by CIDv1. //! -//! Every git object stored on this node is addressable by its IPFS CIDv1. +//! Every git object pinned on this node is addressable by its IPFS CIDv1. //! The CID is computed as: //! //! CIDv1(codec=raw, multihash=sha2-256(content_bytes)) //! //! where `content_bytes` is the raw object content as returned by -//! `git cat-file ` (i.e. without the git framing header). -//! This is consistent with how `gitlawb_core::cid::Cid::from_git_object_bytes` -//! computes CIDs when objects are pushed. +//! `git cat-file ` (i.e. without the git framing header) — the +//! same bytes `gitlawb_core::cid::Cid::from_git_object_bytes` hashes when the +//! object is pinned. That digest is NOT the object's git oid: git frames the +//! content with a `" \0"` header before hashing, so `sha2-256(content)` +//! and the git oid differ. The handler therefore maps the CID back to its oid via +//! the `pinned_cids` table rather than treating the digest as an oid (#173). //! //! Serving is access-controlled: an object is returned only from a repo row the //! requesting caller is permitted to read (per-caller path-scoped visibility, @@ -27,14 +30,371 @@ use std::str::FromStr; use crate::auth::AuthenticatedDid; use crate::error::{AppError, Result}; use crate::git::store; -use crate::git::visibility_pack::{allowed_blob_set_for_caller_bounded, has_path_scoped_rule}; +use crate::git::visibility_pack::{ + allowed_blob_set_for_caller_bounded, allowed_tree_set_for_caller_bounded, has_path_scoped_rule, + reachable_commit_tag_oids_bounded, +}; use crate::state::AppState; use crate::visibility::{visibility_check, Decision}; +/// Hard ceiling on the number of full-history reachability walks a single +/// `GET /ipfs/{cid}` request may spawn. The route brake (`ipfs_rate_limiter`, charged +/// once per request by the middleware) caps request RATE, and the per-walk charge on +/// the separate `ipfs_work_rate_limiter` bounds the walk work across requests, but +/// within ONE request the object can exist under path-scoped rules in many repos, and +/// each distinct repo pays its own `spawn_blocking` walk (the memo only dedups the same +/// repo). Without a ceiling a single request fans out to O(repos) walks — an +/// amplification sink (INV-10). Once this many walks have run IN A PHASE, no further +/// walk is spawned for that phase: any remaining candidate there that still needs +/// a walk is skipped (and, with nothing else readable, the request falls through +/// to the opaque 404). The bound is deliberately generous: a legitimate caller +/// serves on the first repo that grants them, so reaching it requires being +/// denied by this many path-scoped repos first, which real traffic effectively +/// never does. Tunable if that assumption stops holding. +/// +/// Kept at `MAX_PIN_SOURCES + 1` so the ceiling can never truncate a request +/// BEFORE its whole bounded provenance source set (first-pinner + up to +/// `MAX_PIN_SOURCES` additional) has been tried: an authorizing public source that +/// sorts after `MAX_PIN_SOURCES` path-scoped denials must still be reached and +/// served, not falsely 503'd as a truncated search. The legacy scan's fan-out is +/// separately bounded by `MAX_LEGACY_PROBES_PER_REQUEST`, so widening this by one +/// does not loosen that path. +/// +/// The ceiling is charged PER PHASE (#173 round 13, F3), and that is what makes the +/// paragraph above hold for the fallback too: the legacy-scan fallback gets its own +/// equal budget rather than the provenance phase's remainder, so one request can spawn +/// up to `2 * walk_cap` walks in total and no more. Without the split, a source set of +/// root-readable but path-scoped denials spends the whole ceiling reaching its denials, +/// and the fallback armed to find the PUBLIC source `record_pin_source` silently +/// dropped cannot walk to it, a deterministic 503 on every retry for an object that is +/// public. +pub(crate) const MAX_HISTORY_WALKS_PER_REQUEST: u32 = crate::db::MAX_PIN_SOURCES as u32 + 1; + +/// Hard per-request ceiling on how many legacy (NULL-provenance) repositories +/// the CID resolver's scan fallback may PROBE (`acquire` + `git cat-file -t`). +/// The provenance path targets one repo; the legacy scan, absent this bound, +/// fans one anonymous request out to O(repos) subprocess spawns and cold-cache +/// Tigris fetches for a CID enumerable from the public pins index (#173 round 3, +/// F1, INV-10). Deliberately generous: a normal node has far fewer repos than +/// this, so a genuine miss still completes the whole scan and returns a truthful +/// 404; only a node larger than the cap truncates, and a truncated search +/// surfaces as a retryable 503 (never a false "absent"). Legacy pins are a +/// shrinking set — each re-pin backfills provenance — so this fallback is a +/// transitional path, not the steady state. Tunable via `AppState`. +pub(crate) const MAX_LEGACY_PROBES_PER_REQUEST: u32 = 256; + +/// How many repo rows the legacy scan pulls from the database per keyset page +/// (#173, jatmn, INV-10). The probe ceiling above bounds the EXPENSIVE work, but +/// it only starts counting once a probe runs; before that, loading the node's whole +/// repo inventory and every matching visibility rule is itself work proportional to +/// the node's size, bought by one anonymous GET while the scarce walk permits are +/// held. Paging makes the database-facing selection bounded too: the scan reads one +/// page, gates it, and asks for another only while its probe and visit budgets have +/// room. Not an operator knob — sized so a full default-budget scan (256 probes) +/// costs two pages, and a field on `AppState` for the same test-seam reason as the +/// sibling caps. +pub(crate) const LEGACY_SCAN_PAGE_ROWS: usize = 128; + +/// Hard per-request ceiling on how many repo ROWS the legacy scan's pager may fetch +/// (#173 round 13, F2, INV-10). The probe ceiling above only starts counting once a +/// probe runs, and the two denial classes that dominate a hostile inventory +/// (quarantine and a root-scope visibility deny) return before either `walk.probes` +/// or `walk.visits` increments. So an all-quarantined or all-root-denying node paged +/// through its ENTIRE repo table at zero probes, anonymously, retaining every row and +/// rule set, while holding one of the scarce global walk permits for up to the whole +/// request budget. This ceiling is what the DB-facing selection actually stops on. +/// +/// Reaching a holder buried past the ceiling costs `ceil(repos / ceiling) + 1` +/// token-echoing retries: a truncated scan sheds the retryable 503 with a sealed +/// continuation (`ScanPosition`), and the caller echoes it as `?scan=` to resume +/// exactly where the previous page stopped. No server-side scan state exists, so +/// concurrent callers cannot advance or reset each other's ladder. +/// +/// Above roughly `ceiling * (work-budget page term)` rows the bound's total page cost +/// exceeds one work-budget window, so a caller laddering a very large inventory will +/// meet the per-IP page toll before the end and resume after their bucket refills. +/// +/// Tuning DOWN has a cost worth stating: token presence is a coarse inventory-size +/// oracle. A ceiling truncation emits a token; a wrapped scan does not, so laddering +/// until the `scan-wrapped` taint tells an anonymous caller the node's TOTAL repo +/// count (private and quarantined rows included) to within one ceiling. At the 2048 +/// default that is tolled and coarse; it sharpens as the ceiling is lowered. +/// +/// Tunable via `GITLAWB_IPFS_MAX_LEGACY_SCAN_ROWS` / `AppState`. +pub(crate) const MAX_LEGACY_SCAN_ROWS_PER_REQUEST: usize = 2048; + +/// Hard per-request ceiling on the BYTES of visibility rules the legacy scan's pager may +/// retain (#173 round 13, F2, INV-10). The row ceiling bounds the row count but not the +/// memory each row drags in: `fetch_next_page` keeps every fetched page's rules in +/// `LegacyScanPager::rules` for the whole request (a later oid candidate re-reads them +/// rather than re-querying), so a node whose repos each carry many path-scoped rules is +/// retained-memory-unbounded at a row count well under the row ceiling. +/// +/// Bytes, not a rule count, because a count is the wrong unit for a memory bound: an +/// owner controls how many rules their repos carry AND how long each rule's +/// `reader_dids` list is, so a handful of rules can retain as much as thousands. +/// +/// Enforced IN THE QUERY (`Db::list_visibility_rules_for_repos_bounded`), not by summing +/// the page once it has landed. A post-fetch sum truncates the request but leaves the +/// transfer and the allocation already paid, so it bounds the result and not the work, +/// which is the wrong half of INV-10 on an anonymously reachable route. The query cuts on +/// a repo boundary and reports where; `fetch_next_page` drops the page's tail there and +/// mints a continuation, so the page that would have blown the budget truncates the +/// request that bought it without ever being materialized. +/// +/// Not an operator knob: it is a memory guard, not a reach/coverage tradeoff. 4 MiB is +/// about 2 KiB per row at the default row ceiling, which is a generous rule set per repo +/// and still a bounded allocation for one anonymous GET. +pub(crate) const MAX_LEGACY_SCAN_RULE_BYTES_PER_REQUEST: usize = 4 * 1024 * 1024; + +/// Hard ceiling on the byte size of an object `GET /ipfs/{cid}` buffers and serves +/// (#173 round 8, F6, INV-10). The serve reads via a blocking `git cat-file` and +/// buffers the whole object; unbounded, a large public blob (enumerable from the pins +/// index) could exhaust memory or block a runtime worker. A content-addressed serve +/// must verify the whole object hashes to the requested CID before any byte egresses +/// (F2), so it cannot stream — it buffers up to this cap and withholds anything larger +/// (raise the cap if a class of legitimate objects legitimately exceeds it; never +/// stream unverified). 32 MiB is generous for git blobs/trees/commits. Tunable via +/// `AppState` for the test seam, like the sibling caps. +pub(crate) const MAX_SERVED_OBJECT_BYTES: u64 = 32 * 1024 * 1024; + +/// Keyset pager for the legacy (NULL-provenance) scan fallback in `get_by_cid`. +/// +/// Replaces the old "load every repo, every matching rule, and the whole node's +/// quarantine set up front" preload (#173, jatmn, INV-10). That preload ran before +/// the probe ceiling had spent a single probe, so an anonymous GET for a CID +/// enumerable from the public pins index bought allocation and queries proportional +/// to the node's entire repo and rule inventory, with the scarce walk permits held +/// throughout. Here the scan reads one bounded page at a time and asks for another +/// only while its probe and visit budgets have room. +/// +/// Per REQUEST, not per oid candidate. `get_by_cid` may try several oids under one +/// CID, and a pager that reset between them would restore the full fan-out; instead +/// the cursor, the fetched rows, and their rules persist across the whole request, +/// so a later candidate re-reads the pages already paid for and only ever extends +/// the cursor forward. +#[derive(Default)] +struct LegacyScanPager { + /// Rows fetched so far this request, in `(created_at, id)` ASC order. Bounded by + /// the budgets that gate the next fetch, never by the node's repo count. + rows: Vec, + /// Visibility rules for the fetched rows only, keyed by repo id. + rules: HashMap>, + /// Keyset cursor: the `(created_at, id)` of the last row fetched, `None` before + /// the first page. Both halves are immutable columns, so paging is exact. + cursor: Option<(String, String)>, + /// Set once a short page proves no rows remain after the cursor. + exhausted: bool, + /// True when `cursor` was seeded from a caller-supplied continuation token rather + /// than starting at the front. Half of the `"scan-wrapped"` condition: absence is + /// only ever proven over `[start, end)`, so a resumed scan that runs off the end + /// has NOT covered `[front, start)` and must never reach the definitive 404. + resumed: bool, + /// Rows fetched THIS request, the quantity the row ceiling bounds. Distinct from + /// `rows.len()`, which is the same number today but would silently stop tracking + /// the DB-facing cost if the pager ever dropped gated rows. + fetched_rows: usize, + /// Bytes of visibility rules retained this request, the quantity the rules ceiling + /// bounds. + fetched_rule_bytes: usize, + /// Set by `fetch_next_page` when the rules query CUT the page it just fetched, that + /// is when the byte budget stopped the query part-way through the page's repos. The + /// flag exists because the decision has to happen where the fetch happens: measuring + /// only when another page is contemplated lets the page that actually blew the budget + /// go unnoticed on a scan that ends there, and measuring after the fetch bounds the + /// result rather than the work. + rule_bytes_exceeded: bool, +} + +/// Retained size of one visibility rule, in bytes. +/// +/// The heap the pager holds for a rule is its owned strings, and the one an owner can +/// grow without limit is `reader_dids` (there is no per-repo rule cap and no per-rule +/// reader cap). Counting the strings rather than the struct is what makes this track +/// the thing that can actually get large; the fixed fields are noise beside a long +/// reader list. +fn rule_retained_bytes(rule: &crate::db::VisibilityRule) -> usize { + rule.id.len() + + rule.repo_id.len() + + rule.path_glob.len() + + rule.created_by.len() + + rule.reader_dids.iter().map(String::len).sum::() +} + +impl LegacyScanPager { + /// Fetch the next page and its rules, appending both. + /// + /// INV-22: these awaits happen while the scarce walk admission is held and the + /// pool sets no `statement_timeout`, so each is clamped to the remaining request + /// budget exactly as the old preload's queries were. A timeout on the rules query + /// FAILS CLOSED — it returns the retryable budget 503 rather than letting the scan + /// continue against an empty rule map and serve a path-scoped object to a caller + /// the rules would have denied. + async fn fetch_next_page( + &mut self, + state: &AppState, + request_deadline: std::time::Instant, + cid_str: &str, + ) -> Result<()> { + #[cfg(test)] + bump_preload_queries(); + let budget_secs = state.config.ipfs_request_budget_secs; + let budget_shed = || { + AppError::Overloaded(format!( + "ipfs scan incomplete (budget) for CID {cid_str}; retry shortly" + )) + }; + let after = self + .cursor + .as_ref() + .map(|(created_at, id)| (created_at.as_str(), id.as_str())); + // The row ceiling bounds what this REQUEST costs the database, so the ask is the + // smaller of one page and what is left of the budget. Capped in the LIMIT rather + // than by trimming the page once it has landed, because a trim bounds the result + // and leaves the selection, the transfer and the allocation already paid, which is + // the wrong half of the guarantee on an anonymously reachable route. + let remaining = state + .ipfs_max_legacy_scan_rows + .saturating_sub(self.fetched_rows); + let limit = state.ipfs_legacy_scan_page_rows.min(remaining); + // The caller's arm ordering is what guarantees this: `get_by_cid`'s row-ceiling + // arm breaks and mints a continuation before reaching this fetch once + // `fetched_rows >= ipfs_max_legacy_scan_rows`, so the budget always has room here. + debug_assert!( + limit >= 1, + "legacy scan LIMIT must ask for at least one row" + ); + // Record the DB-facing ask before it is made. It sits above the timeout opener + // because the committed guard that checks this query is deadline-wrapped reads a + // fixed lookback from the query call, and anything inserted inside that window + // eats its margin. + #[cfg(test)] + note_scan_limit(limit); + let page = match tokio::time::timeout( + request_deadline.saturating_duration_since(std::time::Instant::now()), + state.db.list_repos_page_for_scan(after, limit as i64), + ) + .await + { + Ok(Ok(page)) => page, + Ok(Err(e)) => return Err(e.into()), + Err(_elapsed) => { + tracing::warn!( + budget_secs, + "/ipfs list_repos_page_for_scan exceeded the request budget \ + (GITLAWB_IPFS_REQUEST_BUDGET_SECS); shedding a retryable 503 and freeing the walk permit" + ); + return Err(budget_shed()); + } + }; + #[cfg(test)] + note_scan_rows(page.len()); + self.fetched_rows += page.len(); + // Measured on the FULL page the query returned, before any rules cut shortens it: + // this is the DB-facing row cost the row ceiling bounds, and a page that is short + // is a page with nothing behind it whatever the rules do. Compared against the + // limit ACTUALLY sent, not the page size: once the budget can shorten the ask, a + // page shorter than a full page proves nothing about the table, and marking the + // scan exhausted there breaks at the top-of-loop arm that sits ahead of every + // ceiling arm, taints nothing and mints no token, so existing content returns a + // false definitive 404. + if page.len() < limit { + self.exhausted = true; + } + if page.is_empty() { + return Ok(()); + } + let mut page = page; + let repo_ids: Vec = page.iter().map(|r| r.repo.id.clone()).collect(); + // The budget is per REQUEST, so what this page may spend is what is left of it. + // A cut ends the scan, so the remaining budget is only ever zero on a page bought + // after the always-admit escape overshot, and zero still admits one repo. + let budget_left = state + .ipfs_max_legacy_scan_rule_bytes + .saturating_sub(self.fetched_rule_bytes); + let (rules, cut_at) = match tokio::time::timeout( + request_deadline.saturating_duration_since(std::time::Instant::now()), + state + .db + .list_visibility_rules_for_repos_bounded(&repo_ids, budget_left), + ) + .await + { + Ok(Ok(rules)) => rules, + Ok(Err(e)) => return Err(e.into()), + Err(_elapsed) => { + tracing::warn!( + budget_secs, + "/ipfs list_visibility_rules_for_repos exceeded the request budget \ + (GITLAWB_IPFS_REQUEST_BUDGET_SECS); denying (fail closed) and freeing the walk permit" + ); + return Err(budget_shed()); + } + }; + #[cfg(test)] + note_scan_rule_rows(rules.values().map(Vec::len).sum()); + // The bound lives in the QUERY, not in a sum taken once the page has landed. A + // rules query answers with whatever the matched repos carry, and nothing caps + // that per repo: a post-fetch sum truncated the request but left the transfer and + // the allocation already paid, which bounds the RESULT rather than the WORK. So + // the cut comes back from the database and the oversized tail is never + // materialized at all. Bytes rather than a rule count for the same reason as + // before: the quantity an owner can grow is the length of each `reader_dids` + // list, not the number of rows in `visibility_rules`. + if let Some(cut) = cut_at { + // The rows from the cut onward were never rule-loaded. Gating them against an + // empty rule map would read as "no restrictions" and FAIL OPEN, so they are + // dropped from the page entirely and the cursor stops in front of them. + // + // `max(1)` is belt and braces over the query's own guarantee that the first + // rule-carrying repo is always admitted. A cut at 0 would leave the cursor + // where it was, the caller's next request would reproduce this page exactly, + // and the ladder would be wedged on a permanent 503. + page.truncate(cut.max(1)); + self.rule_bytes_exceeded = true; + // This page had rows behind the cut, so the table is NOT covered even if the + // page itself was short. This replaces the old `!exhausted` condition: the + // taint now keys on the query having left repos unloaded rather than on the + // page's length. A short final page whose rules all fit produces no cut, so a + // scan that genuinely covered the table is still the definitive 404 it was, + // and a short final page that IS cut is honestly incomplete and resumable. + self.exhausted = false; + } + let last = page.last().expect("the cut always leaves at least one row"); + self.cursor = Some((last.created_at_key.clone(), last.repo.id.clone())); + self.fetched_rule_bytes += rules + .values() + .flat_map(|v| v.iter()) + .map(rule_retained_bytes) + .sum::(); + self.rules.extend(rules); + self.rows.extend(page); + Ok(()) + } +} + +/// Query string of `GET /ipfs/{cid}`. +#[derive(serde::Deserialize)] +pub struct ScanQuery { + /// Sealed continuation from a previous truncated scan's 503 body. Opened with the + /// key derived from the node's persistent identity (`AppState::derive_scan_token_key`, + /// so a restart does not invalidate it) and the request's canonical CID as associated data; ANY + /// failure (undecryptable, tampered, expired, malformed, minted for another CID) is + /// treated as absent and the scan starts at the front, identically and silently, so + /// the token is no oracle. + scan: Option, +} + +/// How long a continuation stays usable. Long enough for a caller to walk a ladder at a +/// human pace and to ride out a work-bucket throttle; short enough that a leaked token +/// stops being a valid scan seed quickly. An expired token is simply absent. +const SCAN_TOKEN_TTL_SECS: i64 = 3600; + /// GET /ipfs/{cid} /// -/// Search all repos on the node for a git object whose SHA-256 hash matches -/// the given CIDv1, returning its raw content if the caller may read it. +/// Resolve the CIDv1 to its git oid via the `pinned_cids` table, then search all +/// repos on the node for that object, returning its raw content if the caller may +/// read it. /// /// Visibility (#110, #126): the object is served only from a repo row the /// caller passes. For each iterated row we gate against that row's OWN rules @@ -43,13 +403,16 @@ use crate::visibility::{visibility_check, Decision}; /// row than the one read (KTD2a). We check object existence via /// `store::object_type` *before* the expensive reachability walk so random-CID /// spray cannot trigger full-history git walks on repos that don't carry the -/// object. When the row carries path-scoped rules (KTD4) the served object -/// must be either a non-blob (trees/commits are structural; KTD3) OR a blob -/// in the caller's *reachable* allowed-set (`allowed_blob_set_for_caller`). -/// The reachable allowed-set excludes dangling blobs — a blob written via -/// `git hash-object -w` and never committed has no path to gate, so it is -/// fail-closed 404'd under path-scoped rules (#126). Denial and genuine -/// not-found both fall through to an opaque 404. +/// object. When the row carries path-scoped rules (KTD4) the served object is +/// gated by type: a `blob`/`tree` must be in the caller's *reachable* allowed-set +/// (`allowed_blob_set_for_caller` / `allowed_tree_set_for_caller`), and a +/// `commit`/`tag` must be in the repo's *reachable* commit/tag set +/// (`reachable_commit_tag_oids`, #173). A withheld subtree's tree object is denied +/// here exactly as `get_tree` denies its path, so its child names and oids cannot +/// leak by CID (#135). All these sets exclude dangling objects — a blob, tree, +/// commit, or tag written via plumbing and never referenced has no reachable path, +/// so it is fail-closed 404'd under path-scoped rules (#126, #173). Denial and +/// genuine not-found both fall through to an opaque 404. /// /// Scan completeness (F2): the 404 above is returned ONLY when every candidate /// repo reached a VERDICT — visibility deny, probe-says-absent, walk-gate deny, @@ -101,15 +464,14 @@ struct WalkAdmission { pub async fn get_by_cid( Path(cid_str): Path, + axum::extract::Query(scan_query): axum::extract::Query, State(state): State, - auth: Option>, - // Per-source keying for the walk concurrency sub-cap. Infallible extractors - // (mirror the git handlers in `repos.rs`): `PeerAddr` yields `None` under - // `oneshot` with no `ConnectInfo`, and the header map falls back per `client_key`. crate::rate_limit::PeerAddr(peer): crate::rate_limit::PeerAddr, - req_headers: HeaderMap, + headers: HeaderMap, + auth: Option>, ) -> Result { - // 1. Decode the CID and extract the SHA-256 digest + // 1. Decode and validate the CID (uniform 400 on a malformed / non-sha2-256 + // CID, before any DB or git work). let cid = CidGeneric::<64>::from_str(&cid_str) .map_err(|e| AppError::BadRequest(format!("invalid CID: {e}")))?; @@ -122,9 +484,13 @@ pub async fn get_by_cid( )); } - let sha256_hex = hex::encode(mh.digest()); - let caller = auth.as_ref().map(|e| e.0 .0.as_str()); - let caller_owned = caller.map(|c| c.to_string()); + // Canonicalize the CID for the pinned_cids lookup. Pins are stored under the + // canonical base32 `cid.to_string()`, but a client may send any equivalent + // multibase spelling (base58/base64) of the same CID; those parse and pass + // the sha2-256 check yet miss the canonical key, so they must be normalized + // before the DB lookup (#173). Response headers and error messages still echo + // the original `cid_str` the client sent. + let canonical_cid = cid.to_string(); // One absolute budget bounds this request's whole acquire+walk lifetime (F3), // captured before admission so the clock covers everything the walk permit @@ -145,13 +511,16 @@ pub async fn get_by_cid( // out concurrent walks past every git pool, exhausting the blocking pool + PIDs. // Acquire the global permit (and, for a resolvable source, the per-source // sub-permit) ONCE here and hold BOTH for the whole request — across every - // `spawn_blocking` walk in the loop below — so the slot reflects real blocking-thread + // `spawn_blocking` walk below — so the slot reflects real blocking-thread // occupancy (a tokio walk-timeout cannot free it while the blocking work still runs) - // and one request cannot open more than its share of concurrent walks. On - // unavailability shed a clean 503. The per-source key is the resolved source IP - // (`client_key`), never the DID (`/ipfs` admits any `did:key` unthrottled, so a DID - // key would be free to mint around); a `None` key (no trusted header, no peer) is - // bounded by the global pool only, never the per-source sub-cap. + // and one request cannot open more than its share of concurrent walks. Holding a + // slot across a walk is only safe because every walk child is duration-bounded + // (`*_bounded` + `run_bounded_git` teardown), so a hung git cannot pin the slot + // past `git_service_timeout_secs`. On unavailability shed a clean 503. The + // per-source key is the resolved source IP (`client_key`), never the DID (`/ipfs` + // admits any `did:key` unthrottled, so a DID key would be free to mint around); a + // `None` key (no trusted header, no peer) is bounded by the global pool only, + // never the per-source sub-cap. let global_permit = state .git_ipfs_walk_semaphore .clone() @@ -160,7 +529,7 @@ pub async fn get_by_cid( tracing::warn!("/ipfs walk concurrency cap reached; shedding request with 503"); AppError::Overloaded("ipfs service at capacity, retry shortly".into()) })?; - let source_key = crate::rate_limit::client_key(&req_headers, peer, state.push_limiter_trust); + let source_key = crate::rate_limit::client_key(&headers, peer, state.push_limiter_trust); let caller_permit = match &source_key { Some(ip) => Some(state.git_ipfs_walk_per_caller.try_acquire(ip).ok_or_else(|| { tracing::warn!(key = %ip, "/ipfs per-source walk cap reached; shedding request with 503"); @@ -185,343 +554,1397 @@ pub async fn get_by_cid( _per_source: caller_permit, }); - // 2. Search all repos for an object with this SHA-256. + // A SECOND, much shorter absolute clock, anchored here at admission, bounding only + // the pre-walk CID resolve below (#174 F4). The request budget alone is 600s by + // default, and a syntactically valid CID with no `pinned_cids` row runs zero probes + // and zero walks, so a resolve stalled in Postgres held these scarce permits for the + // whole 600s while nothing walked; enough distinct source keys doing that + // capacity-503 every real `/ipfs` retrieval at admission. Admission deliberately + // stays FIRST: resolving before taking it would let arbitrarily many unadmitted + // permissionless callers stack concurrent DB queries, trading one amplification for + // another, so the repair is a shorter deadline on the stage rather than a reorder. + let resolve_deadline = std::time::Instant::now() + + std::time::Duration::from_secs(state.config.ipfs_resolve_budget_secs); + + // Caller DID (owned): the `spawn_blocking` closures below cannot borrow the + // handler's `auth` extension, so resolve it once here. + let caller_owned = auth.as_ref().map(|e| e.0 .0.as_str().to_string()); + + // Every DB await from here on runs while the scarce walk permits are ALREADY + // held, and the pool sets no statement_timeout, so an unclamped query blocked in + // Postgres would pin those slots for the whole stall, past the request budget, + // and capacity-503 later requests from any unauthenticated caller (#174 F2). + // Each one is clamped to the request deadline; returning on the timeout arm + // RAII-drops `admission`, which is the whole mechanism, so no new state is + // needed. Defined once here so the clamp sites on the provenance path share one + // definition (the legacy-scan preload below keeps its own `budget_shed` inside + // its nested scope). + // + // Which of the two clocks each await runs on (#174 F4), enumerated once here: + // - `oids_for_cid` runs on the SHORT resolve budget (clamped by the request + // budget). It is the one await that decides whether the request does any + // admitted work at all; nothing has been paid for yet when it runs, so a shed + // there discards nothing but the permits it is holding. + // - EVERYTHING after it stays on the FULL request budget. `pin_sources_for_oid` + // runs once per oid candidate and from the second candidate on runs after real + // probe and walk work; the marker pair runs only on a provenance miss, which is + // after the per-source loop may already have walked; the per-source trio and the + // legacy pager's fetches interleave with admitted walk work by construction. A + // short deadline anchored at admission would be long spent by the time those run + // in a legitimately slow scan, so putting any of them under it sheds a + // PROGRESSING request rather than an idle one. + let budget_shed = || { + AppError::Overloaded(format!( + "ipfs scan incomplete (budget) for CID {cid_str}; retry shortly" + )) + }; + let remaining = || request_deadline.saturating_duration_since(std::time::Instant::now()); + + // Resolve the content-addressed CID to the object's git oid(s). A real pin + // CID digests the raw object content (`Cid::from_git_object_bytes`), NOT the + // git oid (git frames content with a `" \0"` header first), so we + // map it back through `pinned_cids` rather than treating the digest as an oid + // (#173). The cid index is non-unique, so one CID can map to several oids (a + // tree and a blob whose raw bytes collide, or content pinned under two oids); + // we try each candidate below rather than pick one arbitrarily and false-404 + // when the chosen one is withheld or absent while another is readable (#173). + // An empty result is an opaque 404, uniform with a genuine not-found and a + // visibility denial. // - // F6/KTD-5: both initial metadata queries run while the scarce walk permits - // acquired above are ALREADY held (RAII, for the whole request) and BEFORE the - // per-repo loop's first budget gate. With no pool statement_timeout, a query - // blocked in Postgres would otherwise pin those walk slots for the entire stall - // — past the request budget — capacity-503'ing later requests. Clamp BOTH to the - // remaining request budget. On timeout return the same retryable budget 503 the - // later stages shed (the "budget" source); returning here drops the RAII permits, - // freeing the slot. list_visibility_rules_for_repos is the access-control query, - // so its timeout returns BEFORE the loop — the scan can NEVER run with an empty - // rule map and serve an unfiltered listing that exposes private repos (FAIL CLOSED). - let repos = match tokio::time::timeout( - request_deadline.saturating_duration_since(std::time::Instant::now()), - state.db.list_all_repos(), + // Clamped to the LESSER of the resolve budget and the request budget, so a resolve + // budget set larger than the request budget degrades to the request budget instead + // of extending it. + let resolve_remaining = resolve_deadline.saturating_duration_since(std::time::Instant::now()); + let oids = match tokio::time::timeout( + std::cmp::min(resolve_remaining, remaining()), + state.db.oids_for_cid(&canonical_cid), ) .await { - Ok(Ok(repos)) => repos, - // Bare conversion (not `AppError::Internal`) so connection-class sqlx - // failures downcast to `AppError::Db` → 503 `db_unavailable` (#251). + Ok(Ok(v)) => v, + // Bare conversion, never `AppError::Internal`: a connection-class sqlx failure + // downcasts to `AppError::Db` and answers 503 `db_unavailable` rather than a 500 + // (#251). Every clamped site in this handler uses this arm for the same reason, + // so a stalled pool and a closed pool stay distinguishable to the caller. Ok(Err(e)) => return Err(e.into()), Err(_elapsed) => { + // Name the clock that actually bound this await, in the log AND in the body: + // the two budgets are separately settable, so pointing an operator at the + // knob that did nothing here is the same defect as not naming one at all. + // Compared as DEADLINES, not as remainders read at two different instants: + // when the two clocks coincide the later read is always the smaller one, so + // a remainder comparison would attribute a tie to whichever was read second. + if resolve_deadline <= request_deadline { + tracing::warn!( + resolve_budget_secs = state.config.ipfs_resolve_budget_secs, + "/ipfs oids_for_cid exceeded the pre-walk resolve budget \ + (GITLAWB_IPFS_RESOLVE_BUDGET_SECS); shedding a retryable 503 and freeing the walk permit" + ); + return Err(AppError::Overloaded(format!( + "ipfs resolve incomplete (resolve budget) for CID {cid_str}; retry shortly" + ))); + } tracing::warn!( budget_secs = state.config.ipfs_request_budget_secs, - "/ipfs list_all_repos exceeded the request budget \ + "/ipfs oids_for_cid exceeded the request budget \ (GITLAWB_IPFS_REQUEST_BUDGET_SECS); shedding a retryable 503 and freeing the walk permit" ); - return Err(AppError::Overloaded(format!( - "ipfs scan incomplete (budget) for CID {cid_str}; retry shortly" - ))); + return Err(budget_shed()); } }; + if oids.is_empty() { + return Err(AppError::RepoNotFound(format!( + "no git object found for CID {cid_str}" + ))); + } + let caller = caller_owned.as_deref(); + + // Per-request walk budget + memos + throttle flag, shared by the provenance path + // and the legacy scan so both honor the same fan-out ceiling, per-repo memo, and + // IP brake. The caller is constant for one request, so `repo.id` alone keys the memo. + let mut walk = WalkState { + provenance_walks: 0, + scan_walks: 0, + probes: 0, + visits: 0, + truncated_by: Vec::new(), + deterministic_fault: false, + allowed_blob_memo: HashMap::new(), + allowed_tree_memo: HashMap::new(), + reachable_ct_memo: HashMap::new(), + }; + // Set when a walk-requiring candidate is skipped because the source IP's walk quota + // is spent (#173 review, F-C): the scan keeps going so a later walk-free copy still + // serves; only if nothing is servable is it turned into the 429. + let mut throttled = false; + let rctx = ResolveCtx { + caller, + caller_owned: &caller_owned, + headers: &headers, + peer, + cid_str: &cid_str, + canonical_cid: &canonical_cid, + request_deadline, + admission: &admission, + }; - // Fetch every repo's visibility rules in one query rather than one per row - // (the gate runs each row against its OWN rules — KTD2a). A row absent from - // the map has no rules. - let repo_ids: Vec = repos.iter().map(|r| r.id.clone()).collect(); - let rules_by_repo = match tokio::time::timeout( - request_deadline.saturating_duration_since(std::time::Instant::now()), - state.db.list_visibility_rules_for_repos(&repo_ids), - ) - .await - { - Ok(Ok(rules)) => rules, - // Same #251 downcast path as list_all_repos above. - Ok(Err(e)) => return Err(e.into()), - // FAIL CLOSED (security-critical): a timeout on the access-control query must - // DENY. Returning here — before the loop — means the scan can never fall - // through and apply an empty rule map, which would serve an unfiltered listing - // exposing private repos. It sheds the same retryable budget 503 as above. - Err(_elapsed) => { - tracing::warn!( - budget_secs = state.config.ipfs_request_budget_secs, - "/ipfs list_visibility_rules_for_repos exceeded the request budget \ - (GITLAWB_IPFS_REQUEST_BUDGET_SECS); denying (fail closed) and freeing the walk permit" - ); - return Err(AppError::Overloaded(format!( - "ipfs scan incomplete (budget) for CID {cid_str}; retry shortly" - ))); + // Legacy scan pager, advanced LAZILY only when a legacy NULL-provenance pin is hit + // — the provenance path must never trigger it (that fan-out is exactly what + // provenance removes, #173 round 2). Declared here, outside the oid loop, so its + // cursor and its fetched rows are accounted per REQUEST: a per-candidate pager + // would restore the very fan-out the paging removes. + let mut pager = LegacyScanPager::default(); + // Resume from the caller's sealed continuation, if they sent one that opens. The + // node holds NO scan state of its own: the position rides in the token, which is + // what keeps concurrent ladders from advancing or resetting each other. Every + // failure class (tampered, a prior boot's key, expired, malformed, minted for a + // different CID) lands on the same `None` and starts at the front, silently, + // so no probe distinguishes them (INV-13). + // + // The position names the CANDIDATE it resumes as well as the row, so opening it is + // two steps: locate that candidate in the freshly ordered list, then seed the row. + // A sealed hex that is no longer in the list (the object was unpinned under that oid + // between rungs) is treated exactly like an absent token, restarting at the front, + // rather than resumed against some other candidate or turned into a 404 built from a + // table this request never looked at. + let mut resumed_at: Option = None; + // Where this REQUEST started, kept for the strictly-ahead filter at the mint site. A + // front-started request leaves it `None`, which reads as "before everything", so every + // seal it proposes passes. + let mut scan_start: Option<(String, (String, String))> = None; + if let Some(token) = scan_query.scan.as_deref() { + if let Some(pos) = gitlawb_core::scan_token::open_scan_token( + &state.ipfs_scan_token_key, + &canonical_cid, + token, + chrono::Utc::now().timestamp(), + ) { + if let Some(at) = oids.iter().position(|oid| *oid == pos.sha256_hex) { + resumed_at = Some(at); + scan_start = Some(( + pos.sha256_hex.clone(), + (pos.created_at_key.clone(), pos.id.clone()), + )); + // The empty row pair is the front-of-table sentinel: "this candidate, no + // row cursor yet", which is what the advance to the next candidate seals. + // It cannot collide with a real row: `repos.created_at` is NOT NULL and + // written from a serialized timestamp, and `repos.id` is `{owner}/{name}` + // so it always contains a slash. + pager.cursor = (!pos.created_at_key.is_empty() || !pos.id.is_empty()) + .then_some((pos.created_at_key, pos.id)); + // Set even under the sentinel, where the row walk does start at the front: + // this request SKIPPED the candidates ordered before the resumed one, so + // absence is not proven within it and the tail must keep the retryable + // shed rather than fall through to the definitive 404. + pager.resumed = true; + } } - }; + } + // The one position the caller echoes back, written at most once per request: by the + // ceiling that truncated the request's proposer, or by that proposer's finish handing + // the ladder to the next oid candidate. Sealed at the tail rather than here so + // exactly one site mints a token and the wrap case can clear it in one place. + let mut scan_continuation: Option = None; + // True while every candidate ahead of the one being walked FINISHED this request. + // On a front-started request that is the proposer rule: the first candidate that did + // not finish owns the seal, and once it finishes the role passes to the next one. + let mut earlier_all_finished = true; + + for (cand_idx, sha256_hex) in oids.iter().enumerate() { + // Exactly ONE candidate per request may seal a position or advance the ladder, + // and which one depends on where the REQUEST started, not on which candidate is + // interesting. + // + // RESUMED: only the resumed candidate. The pager was seeded from the caller's + // cursor, so `pager.rows` holds the suffix `[start_row, end)`; a later candidate + // that walks "from index 0" walked that suffix and has never seen + // `[front, start_row)`. Letting it seal would record coverage it does not have + // and strand every row in front of the caller's cursor. + // + // FRONT-STARTED: the first candidate that has not finished. Here the suffix + // argument does not exist: every candidate's row loop covers the fetched table + // from the front, so a later candidate's ceiling stop is honest coverage. This + // arm is what mints rung 1 when the first candidate wraps under budget and a + // later one stops on a settled row; silencing it would shed a tainted tokenless + // 503 and end a ladder that works today. + let is_proposer = match resumed_at { + Some(at) => cand_idx == at, + None => earlier_all_finished, + }; + // A pinned object records EVERY repo it was pinned from (#173 round 8, F1). + // Resolve a PROVENANCED pin by trying each source repo (bounded to + // MAX_PIN_SOURCES) through the SAME gate; the first that authorizes serves — no + // scan fan-out. A shared object first pinned from a private/quarantined repo + // still serves from a later PUBLIC source. Deterministic (ORDER BY on the + // union), so no ordering can turn an authorized copy into a 404. + let sources = match tokio::time::timeout( + remaining(), + state.db.pin_sources_for_oid(sha256_hex), + ) + .await + { + Ok(Ok(v)) => v, + Ok(Err(e)) => return Err(e.into()), + Err(_elapsed) => { + tracing::warn!( + budget_secs = state.config.ipfs_request_budget_secs, + "/ipfs pin_sources_for_oid exceeded the request budget \ + (GITLAWB_IPFS_REQUEST_BUDGET_SECS); shedding a retryable 503 and freeing the walk permit" + ); + return Err(budget_shed()); + } + }; + // Provenance fast-path: try each recorded source repo through the SAME gate + // (bounded to first-pinner + MAX_PIN_SOURCES). Empty for a legacy NULL-provenance + // pin. The first source that authorizes serves — no scan fan-out on the common + // path. + for repo_id in &sources { + // These three per-source lookups run while the scarce walk permits are + // ALREADY held, exactly like the legacy scan's preload below, so they carry + // the same clamp (#174 F6/KTD-5). The pool sets no statement_timeout, so an + // unclamped query blocked in Postgres would pin a walk slot for the whole + // stall, past the request budget, and capacity-503 later requests. Returning + // here drops the permits. The quarantine bit and the visibility rules are + // both access control, so a timeout must DENY rather than fall through with + // an empty answer (FAIL CLOSED). + let repo = match tokio::time::timeout(remaining(), state.db.get_repo_by_id(repo_id)) + .await + { + Ok(Ok(Some(r))) => r, + // A source repo is gone: skip it; a later source or the scan fallback + // below may still resolve. + Ok(Ok(None)) => continue, + Ok(Err(e)) => return Err(e.into()), + Err(_elapsed) => { + tracing::warn!( + budget_secs = state.config.ipfs_request_budget_secs, + "/ipfs get_repo_by_id exceeded the request budget \ + (GITLAWB_IPFS_REQUEST_BUDGET_SECS); shedding a retryable 503 and freeing the walk permit" + ); + return Err(budget_shed()); + } + }; + let quarantined = match tokio::time::timeout( + remaining(), + state.db.is_repo_quarantined(repo_id), + ) + .await + { + Ok(Ok(q)) => q, + Ok(Err(e)) => return Err(e.into()), + Err(_elapsed) => { + tracing::warn!( + budget_secs = state.config.ipfs_request_budget_secs, + "/ipfs is_repo_quarantined exceeded the request budget \ + (GITLAWB_IPFS_REQUEST_BUDGET_SECS); denying (fail closed) and freeing the walk permit" + ); + return Err(budget_shed()); + } + }; + let rules_map = match tokio::time::timeout( + remaining(), + state + .db + .list_visibility_rules_for_repos(std::slice::from_ref(repo_id)), + ) + .await + { + Ok(Ok(rules)) => rules, + Ok(Err(e)) => return Err(e.into()), + Err(_elapsed) => { + tracing::warn!( + budget_secs = state.config.ipfs_request_budget_secs, + "/ipfs per-source list_visibility_rules_for_repos exceeded the request budget \ + (GITLAWB_IPFS_REQUEST_BUDGET_SECS); denying (fail closed) and freeing the walk permit" + ); + return Err(budget_shed()); + } + }; + let rules = rules_map.get(repo_id).map(Vec::as_slice).unwrap_or(&[]); + match gate_and_serve( + &state, + &repo, + rules, + quarantined, + sha256_hex, + &rctx, + &mut walk, + false, + ) + .await + { + GateOutcome::Served(resp) => return Ok(resp), + GateOutcome::Throttled => { + throttled = true; + continue; + } + // The provenance path targets a bounded source list rather than the + // table, so there is no scan position to resume from: taint and move on, + // exactly as before. Only the visit ceiling can reach here (the probe + // ceiling is `legacy_scan`-only). + GateOutcome::CeilingStop(reason) => { + record_scan_truncation( + &mut walk, + &mut scan_continuation, + reason, + None, + sha256_hex, + is_proposer, + ); + continue; + } + GateOutcome::Skip => continue, + } + } + + // Bounded legacy-scan fallback. Run it when the provenance set could not have + // served the caller AND may be INCOMPLETE: + // - empty -> a legacy NULL-provenance pin (recorded before provenance existed), or + // - at_cap -> `record_pin_source` stops inserting at MAX_PIN_SOURCES and drops + // later sources SILENTLY, so a full table may hide a servable source + // (e.g. a later PUBLIC pinner buried by 16 attacker sources — the + // pin-source griefing hole). The scan gates every repo through the + // real per-caller gate, so it finds that copy. + // - marked -> a `record_pin_source` for this object failed outright (U3, #173). + // `record_pin_source` is best effort at every pin call site, so a + // non-empty below-cap set is NOT self-evidently complete: an object + // first pinned from a PRIVATE repo and later pushed from a PUBLIC + // one whose record failed names only the private source. The + // durable `pin_sources_incomplete` marker is the node's own record + // that a source is missing, so the fallback stays available for + // exactly those objects instead of 404ing a servable public copy. + // Only a set with NONE of these three signals is treated as complete (every + // recorded source was just tried), so it skips the scan and lets the tail 404, and + // ordinary denials never fan out to O(repos) (INV-10 / F3). Both extra queries run + // only on a provenance MISS (we return above on Served) by a caller that still has + // work budget, so neither costs the serve path nor a shed caller, and the fallback + // is not an authorization bypass: the scan gates every repo + // through the SAME per-caller gate, so a caller who may not read the object is + // still denied. + // + // F3 (#173, INV-10/INV-15): peek the per-IP WORK-budget limiter WITHOUT + // consuming a token so an already-throttled source is shed BEFORE the + // O(repos) preload; the consuming per-probe charge inside gate_and_serve is + // left UNCHANGED (it is load-bearing for the across-request bound), so this + // adds no double-charge. This peeks `ipfs_work_rate_limiter`, the SAME bucket + // the per-probe charge below debits — NOT the route limiter (`ipfs_rate_limiter`, + // charged once per request by the middleware): peeking the route bucket here + // would re-shed a request the route already admitted (R6, U5). + // + // The peek runs BEFORE the two marker queries (#173 round 11, F5): shedding is + // the whole point of a peek, so a spent-budget caller should not pay two + // lookups per request first. It stays AFTER the provenance walk, so no caller + // who could have been served is shed. The one caller this moves: a spent-budget + // caller whose source set turns out COMPLETE now takes the 429 tail instead of + // the 404 tail. That is the honest answer (its search never ran), and it drops + // an oracle, since the old order let a throttled caller tell a complete source + // set from an incomplete one by 404 vs 429. + if let Some(key) = + crate::rate_limit::client_key(rctx.headers, rctx.peer, state.push_limiter_trust) + { + if state.ipfs_work_rate_limiter.is_throttled(&key).await { + throttled = true; + // Skipped for a spent bucket is NOT finished: no scan ran, so nothing was + // covered. Leaving it unfinished keeps the proposer role here, so the + // caller's existing token resumes this candidate once the bucket refills + // instead of the ladder advancing past work that never happened. + earlier_all_finished = false; + continue; + } + } + // Earlier candidates were finished by earlier rungs, so their scans are owed + // nothing. The skip sits HERE on purpose: above it the provenance phase can still + // serve outright from a recorded source, and below it the two marker queries would + // charge a spent-for-nothing lookup pair per skipped candidate. + if resumed_at.is_some_and(|at| cand_idx < at) { + continue; + } + let needs_scan = sources.is_empty() + || { + #[cfg(test)] + bump_marker_queries(); + let at_cap = match tokio::time::timeout(remaining(), async { + #[cfg(test)] + stall_marker_query(MarkerQuery::AtCap).await; + state.db.pin_sources_at_cap(sha256_hex).await + }) + .await + { + Ok(Ok(v)) => v, + Ok(Err(e)) => return Err(e.into()), + Err(_elapsed) => { + tracing::warn!( + budget_secs = state.config.ipfs_request_budget_secs, + "/ipfs pin_sources_at_cap exceeded the request budget \ + (GITLAWB_IPFS_REQUEST_BUDGET_SECS); shedding a retryable 503 and freeing the walk permit" + ); + return Err(budget_shed()); + } + }; + at_cap + || match tokio::time::timeout(remaining(), async { + #[cfg(test)] + stall_marker_query(MarkerQuery::Incomplete).await; + state.db.pin_sources_incomplete(sha256_hex).await + }) + .await + { + Ok(Ok(v)) => v, + Ok(Err(e)) => return Err(e.into()), + Err(_elapsed) => { + tracing::warn!( + budget_secs = state.config.ipfs_request_budget_secs, + "/ipfs pin_sources_incomplete exceeded the request budget \ + (GITLAWB_IPFS_REQUEST_BUDGET_SECS); shedding a retryable 503 and freeing the walk permit" + ); + return Err(budget_shed()); + } + } + }; + // Set when THIS candidate's row loop exits having walked every row the pager + // fetched. It is the witness that the candidate covered the table, and it must be + // per candidate: the shared `pager.exhausted` is a per-REQUEST flag set the moment + // any short page is fetched, so a candidate that a ceiling stopped mid-page would + // read as covered and the ladder would advance over the rows it refused. + let mut wrapped = false; + if needs_scan { + // Walk the candidate repos one bounded page at a time. Pages already + // fetched by an earlier oid candidate are re-read from `pager.rows` for + // free; only the tail of the scan costs another query. + let mut idx = 0usize; + loop { + if idx == pager.rows.len() { + if pager.exhausted { + wrapped = true; + break; + } + // Buying another page is only worth its query if a row on it could + // still reach a verdict, and a verdict needs the probe and the + // acquire these ceilings are refusing. Spent means stop reading — + // this is the check that keeps the DB-facing selection bounded, so + // a one-probe request cannot pull the node's whole inventory. + // + // A page of pure denials is still NOT a hard stop into a 404: a + // quarantined row or a visibility deny costs no probe, so paging + // must continue past them or a public object buried behind many + // private repos would falsely 404. What bounds that case is not a + // denial count but the DB-facing ceilings just below, and every + // truncation they cause carries a continuation so the buried object + // stays reachable across requests (#173 round 13, F2). + // + // Stopping at any of these leaves every unread repo unproven, so + // each TAINTS: the tail sheds a retryable 503 naming the ceiling, + // never a definitive 404 (#173, F2). + // + // All four breaks fire at `idx == pager.rows.len()`, so + // `pager.cursor` is the same well-defined resume boundary in every + // arm and every one of them mints a continuation. These two are not + // an afterthought to the two below: the probe and visit ceilings + // BIND FIRST on any inventory carrying root-readable repos, long + // before the far larger row ceiling, so a tokenless break here is + // the common case rather than the rare one, and a tokenless shed is + // byte-identical to the wrapped-scan answer that tells the caller + // their ladder is over. + if walk.probes >= state.ipfs_max_legacy_probes { + record_scan_truncation( + &mut walk, + &mut scan_continuation, + "probe-ceiling", + pager.cursor.clone(), + sha256_hex, + is_proposer, + ); + break; + } + if walk.visits >= state.config.ipfs_max_repo_visits { + record_scan_truncation( + &mut walk, + &mut scan_continuation, + "visit-ceiling", + pager.cursor.clone(), + sha256_hex, + is_proposer, + ); + break; + } + // Row ceiling (F2). The two checks above only bind once a probe or a + // visit has been spent, and the gate returns Skip on quarantine and + // on a root-scope deny BEFORE either counter moves, so an + // all-denying inventory paged the node's whole repo table at zero + // probes, anonymously, while holding a scarce walk permit. This is + // the check that actually stops that scan. + if pager.fetched_rows >= state.ipfs_max_legacy_scan_rows { + record_scan_truncation( + &mut walk, + &mut scan_continuation, + "row-ceiling", + pager.cursor.clone(), + sha256_hex, + is_proposer, + ); + break; + } + // Rule-bytes ceiling: the row ceiling bounds rows, not the rules each + // row drags in, and the pager retains every fetched page's rules for + // the whole request. The cut is made by the QUERY, so the oversized + // tail is never materialized; `fetch_next_page` drops the rows behind + // it and the request that asked for them is the one that truncates. + if pager.rule_bytes_exceeded { + record_scan_truncation( + &mut walk, + &mut scan_continuation, + "rules-ceiling", + pager.cursor.clone(), + sha256_hex, + is_proposer, + ); + break; + } + // Page toll (F2). Every page is work bought by an anonymous caller, + // so it is charged to the per-IP WORK bucket (the same bucket the + // per-probe charge debits) immediately before the query it pays + // for. Without it a denial-only inventory could be re-paged for free + // by re-requesting, which is the across-request half of the same + // amplification. Reuses the `source_key` already resolved at + // admission; no resolvable key (a test oneshot with no peer or + // trusted header) skips the charge, exactly as the walk and probe + // brakes do. + // + // A spent bucket sets `throttled` and breaks WITHOUT tainting and + // WITHOUT a token: the caller's own bucket stopped them, their + // previous token still resumes them after it refills, and the tail + // renders the 429 when nothing else tainted. + if let Some(key) = &source_key { + if !state.ipfs_work_rate_limiter.check(key).await { + throttled = true; + break; + } + } + pager + .fetch_next_page(&state, request_deadline, &cid_str) + .await?; + if idx == pager.rows.len() { + // The other walked-every-fetched-row exit, and on any inventory + // whose row count is a multiple of the page size it is the NORMAL + // end of the table: `fetch_next_page` only sets `exhausted` on a + // SHORT page, so a full last page leaves the flag clear and the + // empty page after it lands here. Instrumenting only the + // `exhausted` break above leaves `wrapped` false on that path and + // the ladder dies tokenless with later candidates unexamined. + wrapped = true; + break; + } + } + let row = &pager.rows[idx]; + idx += 1; + let rules = pager + .rules + .get(&row.repo.id) + .map(Vec::as_slice) + .unwrap_or(&[]); + match gate_and_serve( + &state, + &row.repo, + rules, + row.quarantined, + sha256_hex, + &rctx, + &mut walk, + true, + ) + .await + { + GateOutcome::Served(resp) => return Ok(resp), + // A throttled walk-requiring candidate is skipped, not fatal: + // keep scanning for a later walk-free copy (#173 review, F-C). + GateOutcome::Throttled => throttled = true, + // A ceiling refused THIS row, so the resume position is the row in + // front of it, not `pager.cursor`, which by now sits at the end of + // the fetched page and would skip every row the ceiling refused. The + // four arms at the top of the loop seal `pager.cursor` legitimately: + // they only fire once every fetched row has been walked. + // + // Stopping here rather than skipping on is also why the FINAL page is + // covered: the `pager.exhausted` break sits ahead of every mint arm, + // so a ceiling reached while walking the last page used to shed with + // no token at all. + GateOutcome::CeilingStop(reason) => { + // Nothing in front of it means nothing was settled, so there is + // no position to seal and this stop contributes none. Only a + // LATER candidate reaches this with nothing settled: it re-walks + // the already-fetched rows from index 0 without passing the + // ceiling arms above, so it can be refused on its very first row. + // The candidate that fetched those rows cannot, because those + // arms run before every fetch and a spent budget breaks there + // instead. Sealing `pager.cursor` here would push the resume + // position past rows this candidate never examined. + let resume = (idx >= 2).then(|| { + let prev = &pager.rows[idx - 2]; + (prev.created_at_key.clone(), prev.repo.id.clone()) + }); + record_scan_truncation( + &mut walk, + &mut scan_continuation, + reason, + resume, + sha256_hex, + is_proposer, + ); + break; + } + GateOutcome::Skip => {} + } + } + } + + // FINISHED: this candidate covered everything it was owed this request, either by + // walking every fetched row (`wrapped`) or by owing no scan at all. The + // `needs_scan` arm is not a nicety: a properly provenanced candidate runs no row + // loop, so without it the resumed candidate can never finish, the advance below + // never fires, and the ladder dies tokenless with later candidates unexamined. + // + // No "and sealed nothing" conjunct: every truncation arm in the row loop breaks + // out of it immediately, so one candidate cannot both seal and walk to the end in + // a single request. + let finished = wrapped || !needs_scan; + if !finished { + earlier_all_finished = false; + } + // The advance. On a RESUMED request the proposer's finish is what moves the ladder + // to the next candidate, sealed at the front-of-table sentinel because that + // candidate has to walk the whole table with a fresh budget. It goes through + // `record_scan_truncation` so the walk is TAINTED as well as sealed: the tail + // emits a continuation only on a tainted walk, so a bare seal here would be + // discarded and the request would fall through to a definitive 404. + // + // Not on a front-started request: there the proposer role simply passes to the + // next unfinished candidate within this same rung, and that candidate seals its + // own stop row. + // + // The final candidate's finish deliberately seals nothing. Absence of a token is + // the ladder's end-of-run signal, and the scan-wrapped clause below turns it into + // the retryable shed. + if is_proposer && finished && resumed_at.is_some() { + if let Some(next) = oids.get(cand_idx + 1) { + record_scan_truncation( + &mut walk, + &mut scan_continuation, + "candidate-advance", + Some((String::new(), String::new())), + next, + true, + ); + } + } + } + + // A RESUMED scan that reached the end of the table has proven absence only over + // `[token, end)`; the rows before the token were never looked at this request, so + // the definitive 404 is not available and the honest answer is the retryable 503. + // + // The condition is evaluated HERE, on `pager.exhausted`, and deliberately not at any + // particular break site. That is what covers the degenerate zero-row resume: a token + // at or past the last row (which the row ceiling emits whenever the row count is an + // exact multiple of the ceiling, and which repo deletion between ladder steps also + // reaches) fetches an EMPTY short page, sets `exhausted`, and breaks without + // gating anything. An implementation keying this on having fetched a page passes + // every other case and turns exactly that incomplete search into a false 404. + // + // A wrapped scan emits NO continuation: there is nothing left to resume, and the + // absence of the token is what tells the caller their ladder is over. With several + // oid candidates that is the FINAL candidate's wrap; an earlier one's hands the + // ladder on instead, and the seal it leaves in the slot is what keeps this clause off. + // + // Gated on nothing having been sealed, for two reasons now. A ceiling can stop a + // resumed scan PART WAY through the last page, which leaves `exhausted` set with rows + // still unwalked in front of the cursor; and on a multi-candidate CID the request may + // already carry the advance to the next candidate. Either way the walk is over for + // this rung but the search is not, and clearing the seal would strand exactly what + // the token was minted to reach. + if pager.resumed && pager.exhausted && scan_continuation.is_none() { + walk.taint("scan-wrapped"); + } - // Request-scoped memo of the per-repo allowed-blob set (KTD1, #126). The - // caller is constant for one request, so `repo.id` alone is a safe, - // sufficient key — never a coarse caller "class", which - // `visibility_check`'s exact full-DID reader match would make unsafe. + // Nothing served — four distinct tails, in precedence order: + // 1. A candidate repo is persistently broken (a corrupt repo, a bad `.git/config`), + // and that was the SOLE reason nothing served → terminal, non-retryable 500 + // (#174 F5/U4). A retry cannot fix it, and a 503 here would invite a conformant + // client to retry-storm a fresh `cat-file` per attempt against the broken repo. + // Gated on nothing else having tainted: when a transient skip co-occurs, the + // object may live in the repo that was skipped transiently, so a retry CAN + // surface it and the retryable 503 below is the honest answer. The body is + // opaque; the raw git detail was logged at the probe and never reaches the client. + // 2. The scan was cut short (a cap, the request budget, or a transient stage + // failure), so the object was NOT proven absent/unreadable everywhere → 503, + // retryable, and explicitly NOT a definitive not-found (#173 F2). This outranks + // the throttle: an incomplete search must not masquerade as a clean rate-limit + // outcome. The message names the truncation sources so an operator can map the + // shed to the right knob or backend, and carries no object/OID/metadata. When a + // ceiling was the cut, the shed also carries the sealed continuation the caller + // echoes as `?scan=` to resume. // - // We flipped from a deny-set (`withheld_blob_oids`) to an allowed-set - // (`allowed_blob_set_for_caller`) so dangling blobs — never enumerated by - // the reachable walk — fail closed instead of slipping through an empty - // deny entry (#126). - let mut allowed_memo: HashMap> = HashMap::new(); - - // Verdict-or-taint bookkeeping (F2): a candidate repo the loop cannot bring to a - // VERDICT (visibility deny / probe-says-absent / walk-gate deny / served) marks - // the scan truncated with its source. A truncated scan that finds nothing must - // NOT report 404 — the object may sit in a repo we skipped — so the terminal arm - // sheds a retryable 503 naming the sources, keyed so the operator can tell which - // knob (or backend) to look at. - let mut truncated_by: Vec<&'static str> = Vec::new(); - fn taint(truncated_by: &mut Vec<&'static str>, source: &'static str) { - if !truncated_by.contains(&source) { - truncated_by.push(source); - } - } - // A DETERMINISTIC probe fault (a corrupt repo / bad `.git/config`; #174 F5/U4) is - // separate from a transient taint: a retry cannot fix it, so a scan that found - // nothing must NOT shed the retryable 503 (which would invite a conformant client - // to retry-storm a fresh `git cat-file` per attempt against the broken repo). It - // sheds a terminal, non-retryable 500 instead — but only if nothing served, so one - // corrupt repo never masks a healthy repo that carries the object. - let mut deterministic_fault = false; - - // Budget gate shared by the four per-stage checks (F3): the remaining - // request budget, or — once exhausted — None, after logging the stage and - // knob and tainting "budget"; the call site only breaks (the scan STOPS, - // leaving this and every later candidate unproven, never a false 404). A - // stage is never started with zero remaining; the probe and read - // subprocesses additionally carry their own deadline (the lesser of - // git_service_timeout_secs and the remainder, reaped by process-group - // teardown), so this pre-start check is a gate, not their entire bound. The - // acquire and walk stages clamp their deadlines to the returned remainder. - fn budget_gate( - truncated_by: &mut Vec<&'static str>, - deadline: std::time::Instant, + // ONE deliberate exception to that precedence (#173 round 13, F2): the legacy + // scan's PAGE toll breaks the pager WITHOUT tainting, so a request stopped only + // by its own spent work bucket falls through to the 429 below rather than the + // 503 here. The reason is that a 503 says "the node's search was cut short, + // retry" and invites an immediate retry straight back into the same empty + // bucket; a 429 names what actually stopped the caller and carries the honest + // wait. Their previously issued token is still valid, so the retry after the + // refill resumes rather than restarts. A request that tainted for any OTHER + // reason and then also ran its bucket dry still lands here, per the ordering + // as written. + // 3. A walk-requiring candidate was skipped for a spent IP quota while the scan + // otherwise completed → 429 (the brake bit; a cheaper copy was sought first). + // 4. A full scan under the caps found nothing readable → opaque 404, uniform with + // a genuine not-found and a visibility denial. + if walk.deterministic_fault && walk.truncated_by.is_empty() { + return Err(AppError::Git( + "ipfs object probe could not complete: a candidate repository is corrupt".into(), + )); + } + if !walk.truncated_by.is_empty() { + // Seal the continuation HERE, the single mint site. The position is the last + // row the pager FETCHED, and on a scan that served nothing every fetched row is + // by construction private or quarantined, so its `created_at` and its `id` + // (which carries the owner's DID) are withheld fields and the token must be + // confidential, not merely tamper-evident (INV-13). A seal failure is not fatal + // to the shed: drop the continuation and answer the plain 503, which degrades to + // the pre-token behaviour rather than turning a truncation into a 500. + // A rung owes a token only when it reached somewhere the caller has not already + // been. `walk.visits` is charged by the provenance phase as well as the scan, so a + // resumed request whose sources spend the ceiling reaches the scan's top-of-loop + // visit arm with nothing fetched, and `pager.cursor` is still the caller's own + // incoming position: sealing it hands them back the token they just sent. `gl` + // echoes a token up to its resume cap, each rung re-running the whole provenance + // phase, so the ladder amplifies one anonymous request into nine while advancing + // nothing and the token makes it look like progress. + // + // Strictly ahead has two arms. A proposal naming the SAME candidate must carry a + // row past the start row. A proposal naming a DIFFERENT candidate is the advance, + // which only a finished candidate can produce, so it is ahead by construction even + // though the front sentinel it seals sorts below every real row. + // + // ONE site, the same argument the single mint site is already built on: a filter + // here cannot be bypassed by a future sealing arm. The ceiling arms stay uniform + // (all of them seal `pager.cursor`) rather than each carrying a copy of this rule. + // + // The row comparison is Rust's byte-wise `Ord` on `(created_at_key, id)`, while the + // pager's keyset predicate ordered the same TEXT columns under the DATABASE's + // collation, so on a non-`C` collation the two can disagree for ids differing in + // case or punctuation. It cannot skip rows: a dropped seal claims no coverage, it + // only ends the rung, and the caller's recovery is a fresh ladder from the front. + // The case this filter exists for is exact equality of a value with itself, which + // no collation moves. + let advancing = scan_continuation.filter(|sealed| { + let advanced = match &scan_start { + None => true, + Some((start_hex, start_row)) => { + sealed.sha256_hex != *start_hex || sealed.row > *start_row + } + }; + if !advanced { + // `record_scan_truncation` already logged `sealed_continuation = true` for + // this seal, and a 503 carrying no token next to that line is exactly the + // confusion that log exists to prevent. This is the correction, and like + // the line it corrects it is a boolean fact only: the position and the + // candidate it names are withheld data. + tracing::debug!( + seal_dropped_not_advancing = true, + "/ipfs dropped a scan continuation that reached no row past the \ + request's own start; shedding without one" + ); + } + advanced + }); + let continuation = advancing.and_then(|sealed| { + let SealedScanPos { + row: (created_at_key, id), + sha256_hex, + } = sealed; + match gitlawb_core::scan_token::seal_scan_token( + &state.ipfs_scan_token_key, + &canonical_cid, + &gitlawb_core::scan_token::ScanPosition { + created_at_key, + id, + sha256_hex, + }, + chrono::Utc::now().timestamp() + SCAN_TOKEN_TTL_SECS, + ) { + Ok(token) => Some(token), + Err(e) => { + tracing::warn!(error = %e, "/ipfs could not seal a scan continuation; \ + shedding the truncation 503 without one"); + None + } + } + }); + return Err(AppError::SearchIncomplete { + message: format!( + "CID {cid_str} search incomplete ({}); retry", + walk.truncated_by.join("+") + ), + continuation, + }); + } + if throttled { + return Err(AppError::TooManyRequests( + "ipfs retrieval rate limit exceeded — try again later".into(), + )); + } + Err(AppError::RepoNotFound(format!( + "no git object found for CID {cid_str}" + ))) +} + +/// Outcome of gating one repo for one candidate oid. +enum GateOutcome { + /// The object passed the gate; serve this response. + Served(Response), + /// This repo does not serve the object (absent, denied, quarantined, walk-capped, + /// or a walk error) — try the next candidate. + Skip, + /// A walk-requiring candidate hit the per-IP walk quota; skip it but let the caller + /// record the throttle so a later walk-free copy can still serve. + Throttled, + /// A per-request CEILING (probes, repo visits) refused this row before it could reach + /// a verdict, and will refuse every row after it too. Distinct from `Skip` because the + /// caller owes the ladder a resume position in front of this row, not just a taint: + /// the taint alone sheds a 503 whose missing token reads as "ladder over", stranding + /// this row and everything behind it on an inventory that never changes. The caller is + /// the only one that can name that position, which is why this returns rather than + /// tainting here. + CeilingStop(&'static str), +} + +/// A sealed resume position together with the candidate whose walk produced it. +/// +/// The candidate rides along because one CID can map to several git oids, so a bare row +/// pair names a row without naming whose walk it belongs to. It is carried by IDENTITY +/// (the oid hex), never by position in the candidate list, which is ordered by hex and +/// mutates between rungs. +struct SealedScanPos { + /// The keyset row pair to resume that candidate at. The empty pair is the + /// front-of-table sentinel: resume this candidate with no cursor. + row: (String, String), + /// The candidate oid this position resumes. + sha256_hex: String, +} + +/// Taint the walk with a truncation reason and seal the position the caller echoes back, +/// together, at one site. +/// +/// Keeping the two together is the point. Every earlier drip on this path was a CEILING +/// that tainted somewhere the mint could not see, so the shed carried no token. This is +/// not the only place the walk is tainted: the transient skips (`acquire`, `read`, +/// `budget`, the walk cap) taint directly and seal nothing, because they refuse one row +/// rather than stopping the scan, and the rows behind them are still walked. A ceiling is +/// what stops the scan, so a ceiling is what owes the caller a position. +/// +/// `may_seal` is the caller's proposer verdict, and it is the whole of the multi-candidate +/// rule. Exactly one candidate per request may seal: on a resumed request the resumed +/// candidate (every other one walked only the suffix `[start_row, end)` the pager holds, +/// so its stop is not coverage of the table), and on a front-started request the first +/// candidate that has not finished (there every candidate walks from the front, so a later +/// candidate's stop IS honest coverage). A non-proposer's truncation still TAINTS, since +/// the scan really was cut short, but it contributes no position. +/// +/// That scope, not an ordering comparison, is what keeps the ladder moving forward. Every +/// sealing arm breaks its row loop and only one candidate may seal, so the slot is written +/// at most once per request; the debug assertion below states that invariant where a future +/// change would trip it. The forward-only keep-the-maximum comparison this replaced was +/// the defect: a budget-starved later candidate contributing nothing could not lower a +/// maximum, so the token resumed past rows that candidate had never examined and the CID +/// became permanently unretrievable. +fn record_scan_truncation( + walk: &mut WalkState, + slot: &mut Option, + reason: &'static str, + pos: Option<(String, String)>, + sha256_hex: &str, + may_seal: bool, +) { + walk.taint(reason); + let pos = pos.filter(|_| may_seal); + // The one log that separates a rung from a dead end. A truncation that seals nothing + // sheds a tokenless 503, which the client reads as "your ladder is over", so an + // operator staring at a stranded caller needs to see WHICH ceiling stopped the scan + // and whether it handed back a way to continue. The position itself is withheld data + // (its `created_at` and its `id` carry a private repo's owner DID), so log only + // whether one exists, never its value. A `true` here is the seal being RECORDED, not + // the response carrying it: the mint site drops a seal that reached no row past the + // request's own start, and logs its own line saying so when it does. + tracing::debug!( + reason, + sealed_continuation = pos.is_some(), + "/ipfs legacy scan truncated" + ); + if let Some(pos) = pos { + debug_assert!( + slot.is_none(), + "one candidate per request may seal, and every sealing arm breaks its row \ + loop, so the slot is written at most once" + ); + *slot = Some(SealedScanPos { + row: pos, + sha256_hex: sha256_hex.to_string(), + }); + } +} + +/// Outcome of the bounded, off-worker object read for one gated candidate (F6, #173). +enum ServedRead { + /// Verified: the object's bytes hash to the requested CID; serve them. + Ok(Vec), + /// The bytes do not hash to the requested CID (a legacy provider-CID row); withhold. + Mismatch(String), + /// The object exceeds the served-object size cap; withhold rather than buffer it. + TooLarge(u64), + /// A git subprocess failed to run (spawn/IO error, not a "no such object"). Logged at + /// the handler layer and skipped — an infra failure must surface as an error, not a + /// silent 404 for an authorized caller (INV-25 spirit, #173). + ReadErr(String), +} + +/// Immutable per-request context threaded into the gate. +struct ResolveCtx<'a> { + caller: Option<&'a str>, + caller_owned: &'a Option, + headers: &'a HeaderMap, + peer: Option, + cid_str: &'a str, + /// Canonical base32 form of the requested CID (`cid.to_string()`), used by the + /// serve-side integrity check to confirm the served bytes actually hash to the + /// requested content address (F2, #173). Compared against the recomputed CID, NOT + /// `cid_str` — a client may send an equivalent non-canonical multibase spelling. + canonical_cid: &'a str, + /// One absolute clock for the whole admitted request (#174 F3). No stage starts + /// once it is exhausted, and the acquire wait plus the probe/walk/read child + /// deadlines clamp to the remainder, so an admitted request cannot hold its scarce + /// walk slot by drawing a fresh per-stage timeout on every candidate. + request_deadline: std::time::Instant, + /// The request's walk admission (#174 U1). A clone goes into every `spawn_blocking` + /// below, so the permits release only when the last holder drops — the handler's + /// clone, or an abandoned or panicking closure's, whichever outlives the other. + admission: &'a std::sync::Arc, +} + +/// Per-request walk budget + memos, shared across the provenance path and the legacy +/// scan so the fan-out ceiling and per-repo memoization span the whole request. +struct WalkState { + /// Walks spent by the PROVENANCE phase, checked against `walk_cap` on its own. + provenance_walks: u32, + /// Walks spent by the legacy-scan fallback, checked against the SAME `walk_cap` + /// but from its own zero. The two phases are budgeted separately because they are + /// not alternatives: the fallback exists precisely to reach a source the + /// provenance set dropped, and a shared counter let the provenance phase's denials + /// spend the budget the fallback needs to get there (#173 round 13, F3). + scan_walks: u32, + /// Count of legacy (NULL-provenance) repos actually probed this request, so the + /// scan can stop at `ipfs_max_legacy_probes` instead of fanning out to O(repos) + /// `acquire` + `cat-file` (#173, F1, INV-10). Only the legacy path bumps it. + probes: u32, + /// Count of repos this request has VISITED: every candidate that got past the + /// visibility gate and reached the acquire stage, on the provenance path as well + /// as the legacy scan (#174 F2). Every visit costs an acquire (worst case a full + /// Tigris archive download on a cache miss) plus a `cat-file` probe, so one + /// request can trigger at most `ipfs_max_repo_visits` object-store fetches. This + /// is the broader of the two ceilings: the probe ceiling above bounds only the + /// legacy scan's fan-out, and a provenance-only request never reaches it. + visits: usize, + /// Why the scan reached no verdict on one or more candidates: a cap cut it short + /// (the legacy probe ceiling, the walk ceiling, the request budget) or a stage + /// failed transiently (acquire, probe, walk, read). A truncated scan did NOT prove + /// the object absent/unreadable everywhere, so the tail returns a retryable 503 + /// rather than a definitive 404 (#173 F2), and the sources name the knob or backend + /// the operator should look at (#174 F2). Deduplicated, so one source appears once + /// however many candidates hit it. + truncated_by: Vec<&'static str>, + /// Set when a candidate repo is persistently broken (a corrupt repo, a bad + /// `.git/config`; #174 F5/U4). It yields no absence verdict either, but a retry + /// cannot fix it, so the tail sheds a terminal 500 instead of the retryable 503 — + /// and only when nothing else tainted, so one broken repo never converts a + /// retryable outcome into a terminal one. + deterministic_fault: bool, + allowed_blob_memo: HashMap>, + allowed_tree_memo: HashMap>, + reachable_ct_memo: HashMap>, +} + +impl WalkState { + /// Record that a candidate was skipped WITHOUT a verdict, naming the stage. + fn taint(&mut self, source: &'static str) { + if !self.truncated_by.contains(&source) { + self.truncated_by.push(source); + } + } + + /// Remaining request budget, or `None` once it is spent. A stage is never started + /// with zero remaining: the call site taints "budget" and stops, leaving this and + /// every later candidate unproven rather than reporting a false absence. + fn budget_left( + &mut self, + ctx: &ResolveCtx<'_>, budget_secs: u64, repo_name: &str, stage: &'static str, ) -> Option { - let left = deadline.saturating_duration_since(std::time::Instant::now()); + let left = ctx + .request_deadline + .saturating_duration_since(std::time::Instant::now()); if left.is_zero() { tracing::warn!( repo = %repo_name, stage, budget_secs, "/ipfs request budget exhausted before the stage \ - (GITLAWB_IPFS_REQUEST_BUDGET_SECS); stopping the scan without a verdict" + (GITLAWB_IPFS_REQUEST_BUDGET_SECS); stopping without a verdict" ); - taint(truncated_by, "budget"); + self.taint("budget"); return None; } Some(left) } +} - // Cap on EXPENSIVE walks only (F2): counts the repos that actually require the - // full-history `allowed_blob_set_for_caller_bounded` walk (a path-scoped blob), - // checked immediately before the spawn_blocking below. Cheap probe-only visits - // are bounded by `repos_visited` — counting them here starved later-ordered - // repos out of a plain 200 on nodes with more readable repos than the cap. - let mut repos_walked: usize = 0; - // Ceiling on VISITS (F2): every repo past the visibility gate costs an acquire - // (worst case a full Tigris archive download on a cache miss) plus a cat-file - // probe, so one request can trigger at most `ipfs_max_repo_visits` object-store - // fetches. On exhaustion the scan STOPS — there is no cheaper way to continue. - let mut repos_visited: usize = 0; - - for repo in &repos { - // Repo-level read gate against THIS row's own rules (KTD2a). Deny is a - // VERDICT: this repo would never serve the caller, so skipping it cannot - // hide content from them. - let rules: &[crate::db::VisibilityRule] = rules_by_repo - .get(&repo.id) - .map(Vec::as_slice) - .unwrap_or(&[]); - if visibility_check(rules, repo.is_public, &repo.owner_did, caller, "/") == Decision::Deny { - continue; +/// Gate ONE repo for ONE candidate oid and, if the caller may read it, serve it. The +/// SINGLE gate both the provenance path and the legacy scan call, so INV-11 (quarantine +/// hard-drops before visibility), INV-2 (the repo's own "/" gate), and the per-object +/// reachability walk hold identically on both paths (KTD5). Never re-resolves via +/// `authorize_repo_read`, whose fuzzy match could authorize a different physical row +/// than the one read (KTD2a). +// The per-repo gate genuinely needs the row, its rules, its quarantine bit, the oid, +// the request context, the shared walk budget, and whether this is the fan-out-bounded +// legacy scan; bundling them buys nothing over the existing threshold. +#[allow(clippy::too_many_arguments)] +async fn gate_and_serve( + state: &AppState, + repo: &crate::db::RepoRecord, + rules: &[crate::db::VisibilityRule], + quarantined: bool, + sha256_hex: &str, + ctx: &ResolveCtx<'_>, + walk: &mut WalkState, + // True only for the legacy NULL-provenance scan, which iterates every repo. The + // provenance path targets one repo (no fan-out) and passes false, so it does not + // consume the per-request probe budget below. + legacy_scan: bool, +) -> GateOutcome { + // Quarantine gate (INV-11): a quarantined mirror is hidden from every reader, owner + // included, BEFORE any visibility check — so an owner whom visibility would Allow + // still 404s. + if quarantined { + return GateOutcome::Skip; + } + // Repo-level "/" read gate against THIS row's own rules (INV-2, KTD2a). + if visibility_check(rules, repo.is_public, &repo.owner_did, ctx.caller, "/") == Decision::Deny { + return GateOutcome::Skip; + } + // Legacy-scan fan-out control (#173, F1/F3, INV-10). The legacy path probes every + // root-visible repo, and the probe below (`acquire` — a possible cold-cache + // Tigris fetch — plus a `git cat-file -t` subprocess) is the expensive part. + // Cap it per request BEFORE that work runs, so an anonymous caller wielding a + // CID from the public pins index cannot amplify one request into O(repos) + // subprocesses. A legacy scan is inherently fan-out (unlike a targeted + // provenance fetch), so EVERY legacy probe is charged to the source IP from the + // first one, not just the ones past a free budget. A per-request-only budget + // reset each request, leaving a NULL-provenance CID open to unbounded ACROSS- + // request amplification: N requests spending N x budget cold `acquire` calls + // against Tigris with zero limiter contact (#173, F3, jatmn). Charging the first + // probe makes those requests accumulate against the per-IP `ipfs_work_rate_limiter` + // (the resolver's WORK bucket, separate from the once-per-request route brake + // `ipfs_rate_limiter` — R6, U5), closing that path. The per-request cap below stays + // as the second bound (a single request's ceiling). A spent quota is the same non-fatal Throttled as the + // walk brake: keep scanning for a walk-free copy, and only a wholly-unservable + // request becomes the 429. No resolvable key (a test oneshot with no peer/header) + // skips the brake, as the walk brake does. The provenance path targets one repo + // (no fan-out) and is exempt (`legacy_scan == false`). + if legacy_scan { + if walk.probes >= state.ipfs_max_legacy_probes { + // Budget spent: stop probing and mark the scan truncated so the tail + // reports an incomplete search (503), not a false 404 (#173, F2). The + // CALLER records it, because the resume position belongs to the row this + // refused and only the caller knows it. + return GateOutcome::CeilingStop("probe-ceiling"); } - - // Budget gate for the acquire stage (F3), checked ahead of the visit - // bookkeeping so an unstarted acquire is not counted as a visit. - let Some(budget_left) = budget_gate( - &mut truncated_by, - request_deadline, - state.config.ipfs_request_budget_secs, - &repo.name, - "repo acquire", - ) else { - break; - }; - - // Visit ceiling (F2): bound the acquire+probe cost class. Stopping here - // leaves the remaining candidates unproven, so the scan is truncated. - if repos_visited >= state.config.ipfs_max_repo_visits { - tracing::warn!( - ceiling = state.config.ipfs_max_repo_visits, - "/ipfs request hit the per-request repo-visit ceiling \ - (GITLAWB_IPFS_MAX_REPO_VISITS); stopping the scan without a verdict" - ); - taint(&mut truncated_by, "visit-ceiling"); - break; - } - repos_visited += 1; - - // Bound the per-repo acquire under `git_acquire_timeout_secs`: this loop shares - // the P1-2 stall vector (a hung Tigris HEAD/GET on one repo would otherwise - // block the whole /ipfs request). On expiry keep the fail-closed skip — never - // serve an un-acquired repo; a public copy (if any) still gets its turn — but - // the repo got no verdict, so the skip taints the scan. Clamped to the - // remaining request budget (F3) so per-repo acquires cannot each draw a - // fresh full timeout past it. - let acquire_deadline = std::cmp::min( - std::time::Duration::from_secs(state.config.git_acquire_timeout_secs), - budget_left, - ); - let repo_path = match tokio::time::timeout( - acquire_deadline, - state.repo_store.acquire(&repo.owner_did, &repo.name), - ) - .await + if let Some(key) = + crate::rate_limit::client_key(ctx.headers, ctx.peer, state.push_limiter_trust) { - Ok(Ok(p)) => p, - Ok(Err(e)) => { - tracing::warn!(repo = %repo.name, err = %e, "repo acquire failed during /ipfs scan; skipping repo without a verdict"); - taint(&mut truncated_by, "acquire"); - continue; - } - Err(_elapsed) => { - tracing::warn!(repo = %repo.name, "repo acquire timed out during /ipfs scan; skipping repo without a verdict"); - taint(&mut truncated_by, "acquire"); - continue; + if !state.ipfs_work_rate_limiter.check(&key).await { + return GateOutcome::Throttled; } - }; - - // Budget gate for the probe stage (F3): a probe is never STARTED with zero - // remaining, and a started probe now runs its git child under a deadline - // clamped to the remainder (below), so it can never complete past the budget. - let Some(probe_budget) = budget_gate( - &mut truncated_by, - request_deadline, - state.config.ipfs_request_budget_secs, - &repo.name, - "object-type probe", - ) else { - break; - }; + } + walk.probes += 1; + } + // Visit ceiling (#174 F2), checked before the acquire it bounds. On exhaustion the + // scan STOPS on this candidate without a verdict: there is no cheaper way to reach + // one, since a verdict needs the acquire and probe this ceiling is refusing. + if walk.visits >= state.config.ipfs_max_repo_visits { + tracing::warn!( + ceiling = state.config.ipfs_max_repo_visits, + repo = %repo.name, + "/ipfs request hit the per-request repo-visit ceiling \ + (GITLAWB_IPFS_MAX_REPO_VISITS); skipping repo without a verdict" + ); + return GateOutcome::CeilingStop("visit-ceiling"); + } + walk.visits += 1; + + // Budget gate for the acquire stage (#174 F3). + let Some(acquire_budget) = walk.budget_left( + ctx, + state.config.ipfs_request_budget_secs, + &repo.name, + "repo acquire", + ) else { + return GateOutcome::Skip; + }; + // Bound the per-repo acquire under `git_acquire_timeout_secs`: this gate runs while + // the /ipfs walk permit is held (F5), so a hung or cold-Tigris acquire would otherwise + // pin the global walk slot for the whole request. On expiry skip the repo (a public + // copy may still serve) and mark the search truncated so a wholly-unserved request + // tails to a retryable 503, never a false 404 (reopened the #174 P1-2 stall vector on + // this path otherwise). Clamped to the remaining request budget so per-repo acquires + // cannot each draw a fresh full timeout past it (#174 F3). + let acquire_deadline = std::cmp::min( + std::time::Duration::from_secs(state.config.git_acquire_timeout_secs), + acquire_budget, + ); + let repo_path = match tokio::time::timeout( + acquire_deadline, + state.repo_store.acquire(&repo.owner_did, &repo.name), + ) + .await + { + Ok(Ok(p)) => p, + // An acquire FAILURE is not an absence verdict either: the repo may well hold the + // object, we just could not open it. + Ok(Err(e)) => { + tracing::warn!(repo = %repo.name, err = %e, "repo acquire failed during /ipfs gate; skipping repo without a verdict"); + walk.taint("acquire"); + return GateOutcome::Skip; + } + Err(_elapsed) => { + tracing::warn!(repo = %repo.name, "repo acquire timed out during /ipfs gate; skipping repo without a verdict"); + walk.taint("acquire"); + return GateOutcome::Skip; + } + }; - // Check whether the object exists in this repo before any expensive - // reachability walk. This prevents random-CID spray from triggering - // full-history git walks on repos that don't carry the object. Absent - // (`Ok(None)`) is a VERDICT; a probe that could not run is not. The - // `git cat-file -t` shells out, so run it OFF the async worker under the - // reaped bounded runner (#174 F3) — a hung/corrupt object store cannot pin a - // runtime worker or the held IPFS permits past the deadline. + // Existence probe before any walk (random-CID spray must not trigger a walk on a + // repo that lacks the object). Off the async runtime — it shells out to + // `git cat-file -t`. Fail closed (skip) on a task panic. + let Some(probe_budget) = walk.budget_left( + ctx, + state.config.ipfs_request_budget_secs, + &repo.name, + "object-type probe", + ) else { + return GateOutcome::Skip; + }; + let obj_type = { + let rp = repo_path.clone(); + let sha = sha256_hex.to_string(); + // The probe shells to the REAL `git`, as the unbounded `object_type` always + // did, independent of `state.git_bin`. That knob is the WALK binary: tests + // point it at a fake that answers `rev-list` and friends, and routing the + // existence probe through it would ask that fake to impersonate + // `cat-file --batch-check` as well (#174). + let git_bin = "git".to_string(); + // Bound the probe CHILD itself (process-group teardown via + // `object_type_bounded` -> `run_bounded_git`), not just an outer tokio timeout + // racing an uncancellable `spawn_blocking`: this probe runs while the /ipfs walk + // permit is held, so a wedged cat-file (corrupt pack, NFS stall) must be REAPED + // at the deadline rather than left to linger and delay admission release + // (#173 round-10, KTD2). No outer timeout, mirroring the bounded walk below. The + // child's own deadline is the lesser of `git_service_timeout_secs` and the + // remaining request budget (#174 F3), so a started probe cannot finish past it. let probe_deadline = std::time::Instant::now() + std::cmp::min( std::time::Duration::from_secs(state.config.git_service_timeout_secs), probe_budget, ); - // The probe shells to the real `git` (as `object_type` historically did), - // independent of `state.git_bin` (which tests point at a fake walk git). - let probe_path = repo_path.clone(); - let probe_sha = sha256_hex.clone(); - let probe_admission = std::sync::Arc::clone(&admission); - let obj_type = match tokio::task::spawn_blocking(move || { - // Admission clone (#174 U1): the slot stays taken until this blocking - // work returns, even if the handler future was dropped or this closure panics. + let probe_admission = std::sync::Arc::clone(ctx.admission); + match tokio::task::spawn_blocking(move || { + // Admission clone (#174 U1): the slot stays taken until this blocking work + // returns, even if the handler future was dropped or this closure panics. let _admission = probe_admission; - store::object_type_bounded("git", &probe_path, &probe_sha, probe_deadline) + store::object_type_bounded(&git_bin, &rp, &sha, probe_deadline) }) .await { Ok(Ok(Some(t))) => t, - Ok(Ok(None)) => continue, - // Transient probe fault (unreadable/mid-repack store): unproven, retryable. + // Absence is the one verdict the probe can reach on its own. + Ok(Ok(None)) => return GateOutcome::Skip, + // Transient fault (an unreadable or mid-repack store, or the reaped + // deadline): unproven and retryable, so taint rather than 404. Ok(Err(store::ProbeError::Transient(e))) => { - tracing::warn!(repo = %repo.name, err = %e, "object-type probe hit a transient store fault during /ipfs scan; skipping repo without a verdict"); - taint(&mut truncated_by, "probe"); - continue; + tracing::warn!(repo = %repo.name, err = %e, "object-type probe hit a transient store fault under the /ipfs walk permit; skipping repo without a verdict"); + walk.taint("probe"); + return GateOutcome::Skip; } - // Deterministic probe fault (corrupt repo / bad config): a retry cannot fix - // it, so it does NOT taint (which would shed a retryable 503). It records a - // terminal condition that the terminal arm renders as a non-retryable 500 - // only if nothing served. The raw git detail stays in the log; the client - // body is opaque. + // Deterministic fault (a corrupt repo, a bad `.git/config`): a retry cannot + // fix it, so it must NOT taint — a retryable 503 would invite a conformant + // client to retry-storm a fresh `cat-file` per attempt against the broken + // repo. The tail renders it as a terminal 500, and only if nothing served. Ok(Err(store::ProbeError::Deterministic(e))) => { - tracing::warn!(repo = %repo.name, err = %e, "object-type probe hit a deterministic fault (corrupt repo/config) during /ipfs scan; skipping repo without a verdict"); - deterministic_fault = true; - continue; + tracing::warn!(repo = %repo.name, err = %e, "object-type probe hit a deterministic fault (corrupt repo/config); skipping repo without a verdict"); + walk.deterministic_fault = true; + return GateOutcome::Skip; } - Err(join_err) => { - tracing::warn!(repo = %repo.name, err = %join_err, "object-type probe task panicked during /ipfs scan; skipping repo without a verdict"); - taint(&mut truncated_by, "probe"); - continue; + Err(e) => { + tracing::warn!(repo = %repo.name, err = %e, "object-type probe task panicked; skipping repo without a verdict"); + walk.taint("probe"); + return GateOutcome::Skip; } - }; + } + }; - // Per-blob gating only applies when a path-scoped rule exists (KTD4). - // Without any path-scoped rule, the "/" gate above is the whole story. - // Trees/commits are always served under path-scoped rules (KTD3). - let path_scoped = has_path_scoped_rule(rules); - if path_scoped && obj_type == "blob" { - if !allowed_memo.contains_key(&repo.id) { - // Budget gate for the walk stage (F3): a walk is never STARTED - // with zero remaining (probed-present is not a serve), and a - // started walk runs its git children under a deadline clamped - // to the remainder (the min below), so a walk can never - // complete past the budget. - let Some(_budget_left) = budget_gate( - &mut truncated_by, - request_deadline, + // Per-object gating applies only under a path-scoped rule (KTD4); otherwise the "/" + // gate above is the whole story. A blob is gated on the caller's allowed-blob set, a + // tree on the allowed-tree set (#135), a commit/tag on the repo's reachable + // commit/tag set (#173) — each a full-history walk sharing the per-request cap and + // per-walk IP quota. + let path_scoped = has_path_scoped_rule(rules); + let gated = path_scoped && matches!(obj_type.as_str(), "blob" | "tree" | "commit" | "tag"); + if gated { + let already = match obj_type.as_str() { + "blob" => walk.allowed_blob_memo.contains_key(&repo.id), + "tree" => walk.allowed_tree_memo.contains_key(&repo.id), + "commit" | "tag" => walk.reachable_ct_memo.contains_key(&repo.id), + other => unreachable!("gated admits only blob/tree/commit/tag, got {other}"), + }; + if !already { + // Budget gate for the walk stage (#174 F3): probed-present is not a serve, so + // a walk is never STARTED with no budget left. + if walk + .budget_left( + ctx, state.config.ipfs_request_budget_secs, &repo.name, "visibility walk", - ) else { - break; - }; - // Walk cap (F2), checked at the one site that actually spends a walk: - // on exhaustion skip THIS repo without a verdict and KEEP scanning — - // later candidates may still reach a cheap probe-only verdict (a plain - // public copy serves its 200 with no walk at all). - if repos_walked >= state.config.ipfs_max_repos_walked { - tracing::warn!( - cap = state.config.ipfs_max_repos_walked, - repo = %repo.name, - "/ipfs request hit the per-request walk cap \ - (GITLAWB_IPFS_MAX_REPOS_WALKED); skipping repo without a verdict" - ); - taint(&mut truncated_by, "walk-cap"); - continue; + ) + .is_none() + { + return GateOutcome::Skip; + } + // Per-request fan-out ceiling (INV-10): once this many walks have run, skip + // THIS walk-requiring candidate and keep scanning (a later walk-free copy + // must still serve). Only this block bumps a counter, so walk-free + // candidates never consume budget. + // Both parents bound this loop, under different knobs: #173's + // `ipfs_max_history_walks` (an AppState field, seeded from config) and + // #174's `GITLAWB_IPFS_MAX_REPOS_WALKED`. Honor the tighter of the two, so + // neither knob silently stops working after the merge. + // + // The cap is charged PER PHASE (#173 round 13, F3): the provenance path and + // the legacy-scan fallback each get their own `walk_cap`, so the total walk + // work one request can buy is `2 * walk_cap` and no more. A single shared + // counter made a public object permanently unservable: every provenance + // source that is root-readable but path-scoped needs a walk to reach its + // deny, so a full source set of them spends the whole ceiling, and the + // fallback armed to find the source `record_pin_source` dropped then has + // nothing left to walk with: it skips that source here, taints, and every + // retry reproduces the same 503. + // + // Raising a single shared ceiling instead was rejected. The adversary + // controls how many provenance slots exist (they are grindable repo ids + // filling `pin_repo_sources`), so for any constant N a set of + // `walk_cap + N` path-scoped denials re-creates the exhaustion. Only a + // budget the provenance phase cannot draw from bounds the fallback's reach + // independently of what the source set contains. The taint name stays + // "walk-cap": to an operator the meaning is unchanged (a walk ceiling cut + // the search), and the knobs still mean what they say, now per phase. + // + // `AppState::ipfs_work_budget` in `crates/gitlawb-node/src/state.rs` + // duplicates this same `min()` as the walk term of the work-bucket floor, + // because a floor that does not reserve what this cap can spend 429s the + // legacy fallback short of its configured reach (#173 round 15, F2). The two + // `min()`s must move together, so an edit starting on this side finds the + // floor rather than only the other way round. + let walk_cap = std::cmp::min( + state.ipfs_max_history_walks as usize, + state.config.ipfs_max_repos_walked, + ); + let spent = if legacy_scan { + walk.scan_walks + } else { + walk.provenance_walks + }; + if spent as usize >= walk_cap { + // The walk ceiling truncated the search: a later repo (possibly one that + // authorizes this caller) is left unwalked, so absence is unproven — + // record it so the tail returns 503, not a false 404 (#173, F2). + tracing::warn!( + cap = walk_cap, + repo = %repo.name, + "/ipfs request hit the per-request walk cap; skipping repo without a verdict" + ); + walk.taint("walk-cap"); + return GateOutcome::Skip; + } + // Brake each spawned walk on the source IP (#173, F3, INV-15), BEFORE + // spending walk budget: a throttled candidate neither walks nor consumes + // budget and must not end the request — skip it and keep scanning + // (#173 review, F-C). No key (a test oneshot with no peer/header) skips the + // brake, as the other IP brakes do. On the LEGACY path the probe brake + // above already charged THIS candidate to the source (#173, F3, jatmn), so + // the walk brake must not double-charge it: only the provenance path + // (`legacy_scan == false`, no probe toll) charges here. + if !legacy_scan { + if let Some(key) = + crate::rate_limit::client_key(ctx.headers, ctx.peer, state.push_limiter_trust) + { + if !state.ipfs_work_rate_limiter.check(&key).await { + return GateOutcome::Throttled; + } } - repos_walked += 1; - let rp = repo_path.clone(); - let r = rules.to_vec(); - let is_public = repo.is_public; - let owner = repo.owner_did.clone(); - let caller_for_walk = caller_owned.clone(); - let git_bin = state.git_bin.clone(); - let git_service_timeout = - std::time::Duration::from_secs(state.config.git_service_timeout_secs); - let walk_deadline = request_deadline; - // Full-history walk shells out to git — keep it off the async runtime, - // bounded and reaped like the served-git ops (#174). - let walk_admission = std::sync::Arc::clone(&admission); - let walk = tokio::task::spawn_blocking(move || { - // Admission clone (#174 U1): the slot stays taken until this blocking - // work returns, even if the handler future was dropped or this closure panics. - let _admission = walk_admission; - // Derive the walk's budget from the request deadline HERE, inside the - // closure, not on the async side before the task is queued. The walk - // starts its own clock when it runs, so a budget computed at queue time - // would hand it the full remainder measured from whenever the blocking - // pool got to it — the queue delay would go uncharged and the walk could - // finish past the request budget. Computing it at task start charges the - // delay against the deadline; a queue delay that eats the whole remainder - // saturates this to zero and the walk fails closed (no verdict, taint), - // which is the safe direction. Same fix as the upload-pack walk in - // `api/repos.rs`, and this route is anonymously reachable. - // TESTING GAP: the queue-delay path is reasoned, not executed. Observing - // it needs a runtime with the blocking pool pinned and parked, and the - // `#[sqlx::test]` harness gives no seam for that (the sibling fix, - // 28a6ca4, shipped with the same gap for the same reason). - let walk_timeout = std::cmp::min( - git_service_timeout, - walk_deadline.saturating_duration_since(std::time::Instant::now()), - ); - allowed_blob_set_for_caller_bounded( + } + if legacy_scan { + walk.scan_walks += 1; + } else { + walk.provenance_walks += 1; + } + + let rp = repo_path.clone(); + let r = rules.to_vec(); + let is_public = repo.is_public; + let owner = repo.owner_did.clone(); + let caller_for_walk = ctx.caller_owned.clone(); + let kind = obj_type.clone(); + // Every walk is the DURATION-BOUNDED twin (`run_bounded_git` teardown under + // `git_service_timeout_secs`): the handler holds its /ipfs walk permit + // across this spawn_blocking, and a held permit is only safe if no walk + // child can outlive the deadline (#174 F5). + let git_bin = state.git_bin.clone(); + let git_service_timeout = + std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let walk_deadline = ctx.request_deadline; + let walk_admission = std::sync::Arc::clone(ctx.admission); + let result = tokio::task::spawn_blocking(move || { + // Admission clone (#174 U1): the slot stays taken until this blocking + // work returns, even if the handler future was dropped or this closure + // panics. + let _admission = walk_admission; + // Derive the walk's budget from the request deadline HERE, inside the + // closure, not on the async side before the task is queued. The walk + // starts its own clock when it runs, so a budget computed at queue time + // would hand it the full remainder measured from whenever the blocking + // pool got to it — the queue delay would go uncharged and the walk could + // finish past the request budget. Computing it at task start charges the + // delay against the deadline; a queue delay that eats the whole remainder + // saturates this to zero and the walk fails closed (no verdict, taint), + // which is the safe direction. Same fix as the upload-pack walk in + // `api/repos.rs`, and this route is anonymously reachable. + // TESTING GAP: the queue-delay path is reasoned, not executed. Observing + // it needs a runtime with the blocking pool pinned and parked, and the + // `#[sqlx::test]` harness gives no seam for that. + let walk_timeout = std::cmp::min( + git_service_timeout, + walk_deadline.saturating_duration_since(std::time::Instant::now()), + ); + match kind.as_str() { + "blob" => allowed_blob_set_for_caller_bounded( &rp, &git_bin, walk_timeout, @@ -529,153 +1952,180 @@ pub async fn get_by_cid( is_public, &owner, caller_for_walk.as_deref(), - ) - }) - .await; - // Fail closed on EITHER a task panic (JoinError) or a walk error: - // we cannot prove the caller may read here, so skip this repo and - // let a public copy (if any) serve. Never serve on an unproven gate - // — and never report absent on one either (no verdict, taint). - let set = match walk { - Ok(Ok(set)) => set, - Ok(Err(e)) => { - tracing::warn!(repo = %repo.name, err = %e, "allowed-blob walk failed during /ipfs scan; skipping repo without a verdict"); - taint(&mut truncated_by, "walk-failure"); - continue; - } - Err(e) => { - tracing::warn!(repo = %repo.name, err = %e, "allowed-blob walk task panicked during /ipfs scan; skipping repo without a verdict"); - taint(&mut truncated_by, "walk-failure"); - continue; + ), + "tree" => allowed_tree_set_for_caller_bounded( + &rp, + &git_bin, + walk_timeout, + &r, + is_public, + &owner, + caller_for_walk.as_deref(), + ), + "commit" | "tag" => { + reachable_commit_tag_oids_bounded(&rp, &git_bin, walk_timeout) } - }; - allowed_memo.insert(repo.id.clone(), set); - } - // Not in the caller's reachable allowed-set: a VERDICT (deny), the walk - // proved this repo would never serve the blob to this caller. - let in_allowed = allowed_memo - .get(&repo.id) - .is_some_and(|set| set.contains(&sha256_hex)); - if !in_allowed { - continue; - } + other => unreachable!("gated admits only blob/tree/commit/tag, got {other}"), + } + }) + .await; + // Fail closed on a walk error or task panic: we cannot prove readability, so + // skip rather than serve on an unproven gate — and never report absent on one + // either, so the skip taints the scan. + let set = match result { + Ok(Ok(set)) => set, + Ok(Err(e)) => { + tracing::warn!(repo = %repo.name, err = %e, "allowed-set walk failed; skipping repo without a verdict"); + walk.taint("walk-failure"); + return GateOutcome::Skip; + } + Err(e) => { + tracing::warn!(repo = %repo.name, err = %e, "allowed-set walk task panicked; skipping repo without a verdict"); + walk.taint("walk-failure"); + return GateOutcome::Skip; + } + }; + match obj_type.as_str() { + "blob" => walk.allowed_blob_memo.insert(repo.id.clone(), set), + "tree" => walk.allowed_tree_memo.insert(repo.id.clone(), set), + _ => walk.reachable_ct_memo.insert(repo.id.clone(), set), + }; + } + let in_set = match obj_type.as_str() { + "blob" => walk.allowed_blob_memo.get(&repo.id), + "tree" => walk.allowed_tree_memo.get(&repo.id), + _ => walk.reachable_ct_memo.get(&repo.id), + } + .is_some_and(|set| set.contains(sha256_hex)); + if !in_set { + return GateOutcome::Skip; } - - // Budget gate for the content-read stage (F3): the read subprocess is - // unclamped, so it never starts past the budget. Tainting instead of - // serving keeps the terminal arm honest and the stop unconditional; the - // retryable 503 tells the caller to come back rather than letting an - // over-budget request keep spending. - let Some(read_budget) = budget_gate( - &mut truncated_by, - request_deadline, - state.config.ipfs_request_budget_secs, - &repo.name, - "content read", - ) else { - break; - }; - - // Now that we've passed the gate, read the content. A failed read after a - // passed gate is not an absence verdict — the probe just said the object - // exists here — so the skip taints the scan. Like the probe, the read shells - // out to `git cat-file `, so run it OFF the async worker under the reaped - // bounded runner clamped to the remaining budget (#174 F3). - let read_deadline = std::time::Instant::now() - + std::cmp::min( - std::time::Duration::from_secs(state.config.git_service_timeout_secs), - read_budget, - ); - // Real `git`, as the read historically used, independent of `state.git_bin`. - let read_path = repo_path.clone(); - let read_sha = sha256_hex.clone(); - let read_type = obj_type.clone(); - let read_admission = std::sync::Arc::clone(&admission); - let content = match tokio::task::spawn_blocking(move || { - // Admission clone (#174 U1): the slot stays taken until this blocking - // work returns, even if the handler future was dropped or this closure panics. - let _admission = read_admission; - store::read_object_content_bounded( - "git", - &read_path, - &read_sha, - &read_type, - read_deadline, - ) - }) - .await - { - Ok(Ok(c)) => c, - Ok(Err(e)) => { - tracing::warn!(repo = %repo.name, err = %e, "object content read failed during /ipfs scan; skipping repo without a verdict"); - taint(&mut truncated_by, "read"); - continue; - } - Err(join_err) => { - tracing::warn!(repo = %repo.name, err = %join_err, "object content read task panicked during /ipfs scan; skipping repo without a verdict"); - taint(&mut truncated_by, "read"); - continue; - } - }; - - // 3. Return the content with IPFS-style headers - let mut headers = HeaderMap::new(); - headers.insert( - HeaderName::from_static("content-type"), - HeaderValue::from_static("application/octet-stream"), - ); - headers.insert( - HeaderName::from_static("x-content-cid"), - HeaderValue::from_str(&cid_str).unwrap_or_else(|_| HeaderValue::from_static("invalid")), - ); - headers.insert( - HeaderName::from_static("x-git-hash"), - HeaderValue::from_str(&sha256_hex) - .unwrap_or_else(|_| HeaderValue::from_static("invalid")), - ); - - return Ok((StatusCode::OK, headers, content).into_response()); } - // Deterministic fault (F5/U4): a candidate repo is persistently broken (corrupt - // repo / bad `.git/config`), so the object is not proven absent AND a retry cannot - // change that. Shed a TERMINAL, non-retryable 500 rather than the retryable 503 - // below — a 503 would let a conformant client retry-storm a fresh `git cat-file` - // per attempt against the broken repo. The body is opaque (a generic message via - // `AppError::Git` -> 500, no Retry-After): the raw git stderr — which leaks - // filesystem paths / config — was logged at the probe, and never reaches the client. + // Passed the gate — bound the object, read it OFF the async worker, and verify the + // content address, all before any byte egresses. F6 (#173): read_object_content runs a + // blocking `git cat-file` and buffers the whole object; called directly on the Axum + // worker (the type-probe and walk are already off-worker) it blocks a runtime thread, + // and unbounded it can exhaust memory for a large public blob (enumerable from the pins + // index). Precheck the SIZE and run size + read + verify inside spawn_blocking. A + // content-addressed serve cannot verify a STREAMED body (the digest is known only after + // the last byte, by which point the prefix has already egressed), so we never stream: + // buffer-verify-then-serve up to the cap and withhold anything larger. F2's integrity + // check moves in here too, so no unverified bytes are ever assembled into a response. // - // Gate on `truncated_by.is_empty()`: the terminal 500 fires only when a - // deterministic fault is the SOLE reason nothing served. When a TRANSIENT taint - // co-occurs (a DIFFERENT repo was skipped by budget / acquire / walk-cap / probe / - // visit-ceiling), the requested object may live in that transiently-skipped repo, - // so fall through to the retryable 503 below — a retry can re-probe it and serve - // the content. Reporting a terminal 500 there would wrongly tell a conformant - // client not to retry, leaving reachable content unreachable until the unrelated - // broken repo is repaired. - if deterministic_fault && truncated_by.is_empty() { - return Err(AppError::Git( - "ipfs object probe could not complete: a candidate repository is corrupt".into(), - )); - } - - // Truncated scan (F2): at least one candidate repo yielded no verdict, so the - // object is not proven absent. A 404 here would misreport existing content, so - // shed retryable instead — Overloaded is the single 503 + Retry-After site in - // error.rs, and the message names the truncation sources so the operator can - // map the shed to the right knob or backend. - if !truncated_by.is_empty() { - return Err(AppError::Overloaded(format!( - "ipfs scan incomplete ({}) for CID {cid_str}; retry shortly", - truncated_by.join("+") - ))); - } - - // Complete scan: every candidate reached a verdict and none served, so the - // object is definitively absent (or denied) for this caller. - Err(AppError::RepoNotFound(format!( - "no git object found for CID {cid_str}" - ))) + // Budget gate for the read stage (#174 F3): the read never starts past the request + // budget, and its shared deadline clamps to the remainder. + let Some(read_budget) = walk.budget_left( + ctx, + state.config.ipfs_request_budget_secs, + &repo.name, + "content read", + ) else { + return GateOutcome::Skip; + }; + let max_bytes = state.ipfs_max_served_object_bytes; + let read_repo = repo_path.clone(); + let read_sha = sha256_hex.to_string(); + let read_type = obj_type.clone(); + let want_cid = ctx.canonical_cid.to_string(); + // Real `git` for the size and content reads, as the probe above and for the same + // reason: `state.git_bin` is the walk binary. + let git_bin = "git".to_string(); + // Bound the size+read CHILDREN themselves (process-group teardown at + // `git_service_timeout_secs` via the `*_bounded` twins), not an outer tokio timeout + // over an uncancellable `spawn_blocking`: a hung cat-file must be REAPED at the + // deadline rather than left to pin the held /ipfs walk permit (#173 round-10, KTD2). + // No outer timeout, mirroring the bounded walk; a `GitServiceTimeout` from either + // twin surfaces as `ServedRead::ReadErr` -> truncated (retryable 503). + // ONE deadline spans the size and content reads, so a single served candidate + // holds the /ipfs walk permit for at most `git_service_timeout_secs` total, not + // one full timeout per stage (mirrors `build_filtered_pack`'s shared deadline). + let read_deadline = std::time::Instant::now() + + std::cmp::min( + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + read_budget, + ); + let read_admission = std::sync::Arc::clone(ctx.admission); + let read = tokio::task::spawn_blocking(move || -> ServedRead { + // Admission clone (#174 U1): the slot stays taken until this blocking work + // returns, even if the handler future was dropped or this closure panics. + let _admission = read_admission; + #[cfg(test)] + break_size_probe_if_armed(&read_repo, &read_sha); + match store::object_size_bounded(&git_bin, &read_repo, &read_sha, read_deadline) { + Ok(size) if size > max_bytes => return ServedRead::TooLarge(size), + Ok(_) => {} + // Every failure of this stage is a fault, never a not-found: the type probe + // above already returned Present, and the probe no longer has an absence + // value to collapse a corrupt object or a failed spawn into (#173 round 12). + Err(e) => return ServedRead::ReadErr(e.to_string()), + } + let content = match store::read_object_content_bounded( + &git_bin, + &read_repo, + &read_sha, + &read_type, + read_deadline, + ) { + Ok(c) => c, + Err(e) => return ServedRead::ReadErr(e.to_string()), + }; + let served = gitlawb_core::cid::Cid::from_git_object_bytes(&content).to_string(); + if served != want_cid { + return ServedRead::Mismatch(served); + } + ServedRead::Ok(content) + }) + .await; + let served_read = match read { + Ok(sr) => sr, + Err(e) => { + tracing::warn!(repo = %repo.name, err = %e, "object read task panicked"); + walk.taint("read"); + return GateOutcome::Skip; + } + }; + let content = match served_read { + ServedRead::Ok(c) => c, + ServedRead::TooLarge(size) => { + tracing::warn!( + repo = %repo.name, size, max = max_bytes, + "withholding object: exceeds the served-object size cap (F6)" + ); + #[cfg(test)] + note_oversize_reject(); + return GateOutcome::Skip; + } + ServedRead::Mismatch(served) => { + tracing::warn!( + repo = %repo.name, requested = %ctx.canonical_cid, served = %served, + "withholding object: served bytes do not hash to the requested CID (legacy provider-CID row?)" + ); + return GateOutcome::Skip; + } + ServedRead::ReadErr(e) => { + // Infra failure (git spawn/IO), NOT a not-found: mark the search truncated so + // a wholly-unserved request tails to a retryable 503, never a definitive 404 + // for an authorized caller (INV-25 spirit — logging alone is not surfacing). + tracing::warn!(repo = %repo.name, err = %e, "error reading git object content"); + walk.taint("read"); + return GateOutcome::Skip; + } + }; + let mut resp_headers = HeaderMap::new(); + resp_headers.insert( + HeaderName::from_static("content-type"), + HeaderValue::from_static("application/octet-stream"), + ); + resp_headers.insert( + HeaderName::from_static("x-content-cid"), + HeaderValue::from_str(ctx.cid_str).unwrap_or_else(|_| HeaderValue::from_static("invalid")), + ); + resp_headers.insert( + HeaderName::from_static("x-git-hash"), + HeaderValue::from_str(sha256_hex).unwrap_or_else(|_| HeaderValue::from_static("invalid")), + ); + GateOutcome::Served((StatusCode::OK, resp_headers, content).into_response()) } /// GET /api/v1/ipfs/pins @@ -694,6 +2144,266 @@ pub async fn list_pins(State(state): State) -> Result>, +> = std::sync::OnceLock::new(); + +#[cfg(test)] +fn size_probe_seam_key(repo_path: &std::path::Path, sha256_hex: &str) -> String { + format!("{}::{sha256_hex}", repo_path.display()) +} + +/// Arm the seam: the next serve read of `sha256_hex` FROM `repo_path` loses its loose +/// object between the type probe and the size probe, so the size probe fails on an object +/// git just confirmed present. +#[cfg(test)] +pub(crate) fn break_size_probe_for(repo_path: &std::path::Path, sha256_hex: &str) { + SIZE_PROBE_BREAKERS + .get_or_init(Default::default) + .lock() + .expect("size-probe seam mutex") + .insert(size_probe_seam_key(repo_path, sha256_hex)); +} + +#[cfg(test)] +fn break_size_probe_if_armed(repo_path: &std::path::Path, sha256_hex: &str) { + let armed = SIZE_PROBE_BREAKERS.get().is_some_and(|s| { + s.lock() + .expect("size-probe seam mutex") + .remove(&size_probe_seam_key(repo_path, sha256_hex)) + }); + if armed { + let _ = std::fs::remove_file( + repo_path + .join("objects") + .join(&sha256_hex[0..2]) + .join(&sha256_hex[2..]), + ); + } +} + +thread_local! { + static PRELOAD_QUERIES: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +#[cfg(test)] +pub(crate) fn reset_preload_queries() { + PRELOAD_QUERIES.with(|c| c.set(0)); +} + +#[cfg(test)] +pub(crate) fn preload_queries() -> usize { + PRELOAD_QUERIES.with(|c| c.get()) +} + +#[cfg(test)] +fn bump_preload_queries() { + PRELOAD_QUERIES.with(|c| c.set(c.get() + 1)); +} + +// Test-only INV-10 cost counter (#173, jatmn): how many repo ROWS the legacy scan's +// database-facing selection actually materialized this request. The query counter above +// cannot see the failure it guards — one unbounded `SELECT ... FROM repos` is a single +// query that pulls the node's entire inventory, so it reads 1 either way. Counting rows +// is what goes red if the paging is reverted. +#[cfg(test)] +thread_local! { + static SCAN_ROWS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +#[cfg(test)] +pub(crate) fn reset_scan_rows() { + SCAN_ROWS.with(|c| c.set(0)); +} + +#[cfg(test)] +pub(crate) fn scan_rows() -> usize { + SCAN_ROWS.with(|c| c.get()) +} + +#[cfg(test)] +fn note_scan_rows(n: usize) { + SCAN_ROWS.with(|c| c.set(c.get() + n)); +} + +// Test-only counter for the LIMIT the legacy scan actually sends to SQL, summed over +// the request's fetches. The row counter above measures what came BACK, so it cannot +// tell a query that asked for the remaining budget from one that asked for a full page +// and then dropped the tail: both return the same rows. The limit is the DB-facing ask, +// which is the quantity the operator ceiling is supposed to bound. +#[cfg(test)] +thread_local! { + static SCAN_LIMIT: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +#[cfg(test)] +pub(crate) fn reset_scan_limit() { + SCAN_LIMIT.with(|c| c.set(0)); +} + +#[cfg(test)] +pub(crate) fn scan_limit() -> usize { + SCAN_LIMIT.with(|c| c.get()) +} + +#[cfg(test)] +fn note_scan_limit(n: usize) { + SCAN_LIMIT.with(|c| c.set(c.get() + n)); +} + +// Test-only INV-10 cost counter: how many visibility-rule ROWS the legacy scan actually +// pulled out of the database this request. The byte ceiling is the guard, but a byte +// count computed from the rows AFTER they arrive cannot tell a bounded query from an +// unbounded one -- both report the same total. Counting the rows the query returned is +// what goes red when the bound moves back out of the query and into a post-fetch sum. +#[cfg(test)] +thread_local! { + static SCAN_RULE_ROWS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +#[cfg(test)] +pub(crate) fn reset_scan_rule_rows() { + SCAN_RULE_ROWS.with(|c| c.set(0)); +} + +#[cfg(test)] +pub(crate) fn scan_rule_rows() -> usize { + SCAN_RULE_ROWS.with(|c| c.get()) +} + +#[cfg(test)] +fn note_scan_rule_rows(n: usize) { + SCAN_RULE_ROWS.with(|c| c.set(c.get() + n)); +} + +// Test-only cost counter (F5, #173 round 11): how many times the fallback gate ran the +// `pin_sources_at_cap` / `pin_sources_incomplete` pair. The work-budget peek sits ahead +// of them, so an already-throttled caller leaves this at 0; putting the peek back after +// the pair turns that assertion red. Same thread_local discipline as the preload counter. +#[cfg(test)] +thread_local! { + static MARKER_QUERIES: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +#[cfg(test)] +pub(crate) fn reset_marker_queries() { + MARKER_QUERIES.with(|c| c.set(0)); +} + +#[cfg(test)] +pub(crate) fn marker_queries() -> usize { + MARKER_QUERIES.with(|c| c.get()) +} + +#[cfg(test)] +fn bump_marker_queries() { + MARKER_QUERIES.with(|c| c.set(c.get() + 1)); +} + +/// Which of the two `needs_scan` marker queries a test wants to stall. +#[cfg(test)] +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum MarkerQuery { + AtCap, + Incomplete, +} + +// Test-only fault-injection seam for the `needs_scan` marker pair +// (`pin_sources_at_cap`, `pin_sources_incomplete`), same idea as +// `RepoStore::tigris_stall`: hold one specific await open so the clamp around it is +// the one observed to fire. +// +// A `LOCK TABLE` fixture cannot isolate these two. `pin_sources_at_cap` reads +// `pin_repo_sources` and `pin_sources_incomplete` reads `pinned_cids`, and BOTH tables +// are already read by `oids_for_cid` and `pin_sources_for_oid` earlier in the same +// admission-held region, so a lock taken before the request stalls one of those instead +// and the RED is attributed to the wrong clamp. Taking the lock mid-request does not +// help either: the window between `pin_sources_for_oid` returning and this pair running +// is a single `get_repo_by_id` round trip (measured at ~0.15ms against this Postgres), +// so timing a lock into it is a race that flakes under load. +// +// Armed per target so the second query can be reached with the first left untouched +// (`at_cap` must return `false` for the `||` to evaluate `pin_sources_incomplete`). +// `thread_local` for the same reason as the counters above: `#[sqlx::test]` runs each +// case on its own current-thread runtime, so arming here is invisible to cases running +// in parallel. +#[cfg(test)] +thread_local! { + static MARKER_QUERY_STALL: std::cell::Cell> = + const { std::cell::Cell::new(None) }; +} + +#[cfg(test)] +pub(crate) fn arm_marker_query_stall(which: MarkerQuery, stall: std::time::Duration) { + MARKER_QUERY_STALL.with(|c| c.set(Some((which, stall)))); +} + +#[cfg(test)] +pub(crate) fn disarm_marker_query_stall() { + MARKER_QUERY_STALL.with(|c| c.set(None)); +} + +/// Awaited INSIDE each marker query's `tokio::time::timeout`, never before it: a stall +/// placed outside the clamp would elapse with the clamp never firing and prove nothing. +#[cfg(test)] +async fn stall_marker_query(which: MarkerQuery) { + let armed = MARKER_QUERY_STALL.with(|c| c.get()); + if let Some((target, stall)) = armed { + if target == which { + tokio::time::sleep(stall).await; + } + } +} + +// Test-only INV-10 cost counter (F6, U6/U7): how many times the serve path withheld an +// object because it exceeded `ipfs_max_served_object_bytes`. The bounded read must reject +// an oversized object rather than buffer it on the worker; the counter is the both-ways +// guard (a removed size precheck stops incrementing it and serves the oversized object). +// Set from the match arm after `spawn_blocking` resolves, i.e. on the test's runtime +// thread, so the thread-local is read on the same thread it is written. +#[cfg(test)] +thread_local! { + static OVERSIZE_REJECTS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +#[cfg(test)] +pub(crate) fn reset_oversize_rejects() { + OVERSIZE_REJECTS.with(|c| c.set(0)); +} + +#[cfg(test)] +pub(crate) fn oversize_rejects() -> usize { + OVERSIZE_REJECTS.with(|c| c.get()) +} + +#[cfg(test)] +fn note_oversize_reject() { + OVERSIZE_REJECTS.with(|c| c.set(c.get() + 1)); +} + #[cfg(test)] mod closed_pool_tests { use super::*; @@ -740,8 +2450,8 @@ mod closed_pool_tests { ); } - /// #251 / CodeRabbit nit: cover `get_by_cid`'s `list_all_repos` conversion - /// path — a valid CID must still yield 503 on a closed pool. + /// #251 / CodeRabbit nit: cover `get_by_cid`'s DB-error conversion path — a + /// valid CID must still yield 503 on a closed pool. #[sqlx::test] async fn get_by_cid_closed_pool_returns_503_db_unavailable(pool: PgPool) { let state = crate::test_support::test_state(pool.clone()).await; @@ -788,8 +2498,11 @@ mod tests { //! walk, plus a per-IP route rate limit. These are handler-layer proofs: mount the //! real handler/router, drive one request, assert the exact 503 shed, then name the //! mutation that turns each RED. The per-source key resolves an IP only (`Some(ip)` - //! vs `None`), never a DID — both arms are driven so neither is vacuous. + //! vs `None`), never a DID — both arms are driven so neither is vacuous. The + //! CID-resolution / visibility-gate behavior of the handler itself is covered by the + //! `#[sqlx::test]` suite in `test_support.rs`. + use super::{arm_marker_query_stall, disarm_marker_query_stall, MarkerQuery}; use axum::body::Body; use axum::extract::ConnectInfo; use axum::http::{Method, Request, StatusCode}; @@ -898,9 +2611,219 @@ mod tests { .expect("git rev-parse runs"); assert!(out.status.success(), "rev-parse failed"); let oid = String::from_utf8_lossy(&out.stdout).trim().to_string(); + // Register the CID index entry the resolver needs to map a requested CID back + // to this oid, keyed on the object's raw CONTENT (what the serve path + // recomputes and verifies against) and with NULL provenance, which is what + // routes a request to the bounded legacy scan (#173). + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(content) + .as_str() + .to_string(); + state + .db + .record_pinned_cid(&oid, &cid, None) + .await + .expect("register the seeded blob in the CID index"); (rec.id, oid) } + /// A 64-hex object id that exists in no repo, for the scan-verdict tests whose + /// whole point is that nothing serves. + fn absent_oid() -> String { + "f2".repeat(32) + } + + /// Status plus decoded JSON body, for the F2 row-ceiling tests that assert on the + /// error code and the `continuation` field together. + async fn status_and_body(resp: axum::response::Response) -> (StatusCode, serde_json::Value) { + let status = resp.status(); + let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .expect("read body"); + let json = serde_json::from_slice(&bytes).unwrap_or_else( + |_| serde_json::json!({ "raw": String::from_utf8_lossy(&bytes).to_string() }), + ); + (status, json) + } + + /// The `continuation` token from a `search_incomplete` body, or `None`. + fn continuation_of(body: &serde_json::Value) -> Option { + body.get("continuation") + .and_then(|v| v.as_str()) + .map(str::to_string) + } + + /// Deterministic ascending `created_at` for the seeded-inventory fixtures. The + /// paged scan orders on the STORED `created_at` TEXT then `id`, so whole-second + /// stamps from one base keep text order and time order identical (a `to_rfc3339` + /// with sub-second digits on some rows and not others would not). + fn scan_order_stamp(i: usize) -> chrono::DateTime { + use chrono::TimeZone; + chrono::Utc + .with_ymd_and_hms(2020, 1, 1, 0, 0, 0) + .unwrap() + .checked_add_signed(chrono::Duration::seconds(i as i64)) + .expect("in-range stamp") + } + + /// Stamp an already-seeded repo row's `created_at` with [`scan_order_stamp`], for a + /// fixture whose RED depends on WHICH row the scan reaches last. + /// + /// `upsert_mirror_repo` (under `seed_repo_with_blob`) stamps `Utc::now()`, so a + /// mirror row's scan position is its seeding instant rendered by `to_rfc3339`, whose + /// fractional-second field is variable-width. The scan compares the stored TEXT, so + /// two rows seeded milliseconds apart can order by digit count rather than by time. + /// Restamping with the whole-second values keeps text order and seed order identical. + async fn stamp_scan_order(pool: &sqlx::PgPool, repo_id: &str, i: usize) { + let at = scan_order_stamp(i).to_rfc3339(); + let done = sqlx::query("UPDATE repos SET created_at = $1 WHERE id = $2") + .bind(&at) + .bind(repo_id) + .execute(pool) + .await + .expect("restamp a seeded repo's scan position"); + assert_eq!( + done.rows_affected(), + 1, + "restamping {repo_id} must hit exactly the row the fixture seeded" + ); + } + + /// Seed `n` PRIVATE repos owned by a foreign DID, in scan order, with `rules_each` + /// path-scoped rules apiece. An anonymous caller is denied at the root gate on every + /// one, and a root deny costs neither a probe nor a visit, which is exactly the + /// hole the row ceiling closes. Their `disk_path`s do not exist on purpose: if a + /// deny ever stopped short-circuiting, the missing-dir probe would taint the scan + /// with a different source and the tests' taint assertions would catch it. + async fn seed_root_denying_repos( + state: &crate::state::AppState, + prefix: &str, + n: usize, + rules_each: usize, + ) { + let owner = "did:key:z6MkF2RowCeilingOwnerAAAAAAAAAAAAAAAAAAA"; + for i in 0..n { + let at = scan_order_stamp(i); + let id = format!("{prefix}-{i:04}"); + state + .db + .create_repo(&crate::db::RepoRecord { + id: id.clone(), + name: format!("{prefix}-{i:04}"), + owner_did: owner.to_string(), + description: None, + is_public: false, + default_branch: "main".to_string(), + created_at: at, + updated_at: at, + disk_path: format!("/nonexistent/{prefix}-{i:04}"), + forked_from: None, + machine_id: None, + }) + .await + .expect("seed a root-denying repo"); + for r in 0..rules_each { + state + .db + .set_visibility_rule( + &id, + &format!("withheld-{r}/**"), + crate::db::VisibilityMode::B, + &["did:key:z6MkU3NotTheCallerBBBBBBBBBBBBBBBBBBBBBB".to_string()], + owner, + ) + .await + .expect("seed a visibility rule"); + } + } + } + + /// Seed `n` QUARANTINED mirror rows in scan order. Quarantine is the other denial + /// class that returns from the gate before a probe or a visit is spent, so it drives + /// the same unbounded pager the private-repo fixture does. + async fn seed_quarantined_repos(state: &crate::state::AppState, prefix: &str, n: usize) { + for i in 0..n { + state + .db + .upsert_mirror_repo( + "z6quarantine", + &format!("{prefix}-{i:04}"), + &format!("/nonexistent/{prefix}-{i:04}"), + None, + true, + ) + .await + .expect("seed a quarantined mirror row"); + } + } + + /// A GET carrying an optional `?scan=` continuation token. + fn get_cid_scan(cid: &str, peer: Option, scan: Option<&str>) -> Request { + let uri = match scan { + Some(t) => format!("/ipfs/{cid}?scan={}", urlencode(t)), + None => format!("/ipfs/{cid}"), + }; + let mut req = Request::builder() + .method(Method::GET) + .uri(uri) + .body(Body::empty()) + .unwrap(); + if let Some(p) = peer { + req.extensions_mut().insert(ConnectInfo(p)); + } + req + } + + /// Percent-encode the few characters base64url tokens cannot contain but a hostile + /// or tampered token can. Keeps the invalid-token probes honest: a raw `+` in a + /// query string decodes to a space, which would make a tamper test pass for the + /// wrong reason. + fn urlencode(s: &str) -> String { + s.bytes() + .map(|b| match b { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { + (b as char).to_string() + } + _ => format!("%{b:02X}"), + }) + .collect() + } + + /// Register a LEGACY (NULL-provenance) `pinned_cids` row and return its CID. + /// + /// The scan-verdict tests below predate the CID index (#173): they drove a bare + /// CID and relied on the handler treating the CID's own digest as the git oid. + /// The index-backed resolver does not do that (a pin CID digests raw object + /// content, not the framed git object), so without a row `oids_for_cid` comes back + /// empty and the handler 404s before any repo is visited. NULL provenance is what + /// routes the request to the bounded legacy scan, which is the loop these tests + /// are about. + async fn seed_legacy_pin(state: &crate::state::AppState, oid: &str) -> String { + let cid = cid_for_oid(oid); + state + .db + .record_pinned_cid(oid, &cid, None) + .await + .expect("seed a legacy NULL-provenance pin row"); + cid + } + + /// The CID to request for an object a test seeded through `seed_repo_with_blob`. + /// + /// That helper registers the index entry keyed on the object's raw CONTENT, which + /// is what the serve path recomputes and compares the requested CID against + /// (#173 F2). Deriving the key from the oid instead yields a CID the gate passes + /// and the integrity check then rejects as a legacy provider-CID row, so the + /// candidate is withheld and a serving test sees a skip rather than its 200. + async fn seed_legacy_pin_for_oid(state: &crate::state::AppState, oid: &str) -> String { + if let Some(cid) = state.db.cid_for_oid(oid).await.expect("read the CID index") { + return cid; + } + // A test that built its object by hand (to corrupt it, say) has no entry yet. + // Its object never serves, so the content check never runs and any stable key + // will do; derive one from the oid. + seed_legacy_pin(state, oid).await + } + /// CIDv1(raw, sha2-256) for a sha256 object id, as the handler resolves it. fn cid_for_oid(oid: &str) -> String { let oid_bytes = gitlawb_core::cid::sha256_hex_to_bytes(oid).unwrap(); @@ -909,6 +2832,43 @@ mod tests { .to_string() } + /// Walk-counting shim that runs the REAL walk (`state.git_bin`): each `rev-list` + /// appends one line to `log`, then every invocation execs the real `git`, so the + /// allowed-set a walk produces is the repo's genuine one. + /// + /// `walk_logging_fake_git` below answers every subcommand with nothing, so under it + /// EVERY walked repo yields an empty allowed set and no repo can ever authorize. The + /// per-phase budget tests need one candidate to deny after a real walk and a later + /// one to allow after another, so they need the real sets and the tally both. + #[cfg(unix)] + fn walk_logging_real_git(dir: &std::path::Path, log: &std::path::Path) -> String { + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + rev-list) echo walk >> \"{}\" ;;\n\ + esac\n\ + exec git \"$@\"\n", + log.display() + ); + let git_path = dir.join("walkgit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + git_path.to_str().unwrap().to_string() + } + + /// How many expensive walks the shim above has recorded so far. + #[cfg(unix)] + fn walks_logged(log: &std::path::Path) -> usize { + std::fs::read_to_string(log) + .map(|s| s.lines().count()) + .unwrap_or(0) + } + /// Fake git for the WALK only (`state.git_bin`): empty refs, `rev-parse` /// resolves, and each `rev-list` appends one line to `log` and prints nothing — /// every walked repo yields an EMPTY allowed-set (path-gate deny verdict) and @@ -938,12 +2898,176 @@ mod tests { git_path.to_str().unwrap().to_string() } - /// F2 buried-row repro: with more readable repos than `ipfs_max_repos_walked`, - /// existing PUBLIC content past the cap must still serve. The cap counts + /// #173 (jatmn, INV-10): the legacy scan's DATABASE-facing selection is bounded + /// too, not just its probes. The probe ceiling only starts counting once a probe + /// runs, so before this fix an anonymous GET for a CID enumerable from the public + /// pins index loaded every repo row, every matching visibility rule, and the whole + /// node's quarantine set — work proportional to the node's inventory, bought at a + /// probe budget of 1, with the scarce walk permits held throughout. + /// + /// Page size 1 and probe budget 1 against THREE candidate repos: the scan may read + /// exactly one page, spend its one probe, and stop. It must then report the + /// truncation (503), never a false 404 — the two later repos were never looked at. + /// + /// The ROW count is the load-bearing assertion. A query counter cannot see this + /// regression: reverting to one unbounded `SELECT ... FROM repos` is a single query + /// that pulls the entire inventory, so the query count reads 1 either way. + /// MUTATION (RED): drop the pre-fetch probe-budget check and the pager walks every + /// page anyway — 3 rows materialized and 4 queries instead of 1 and 1. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_legacy_scan_stops_paging_when_the_probe_budget_is_spent( + pool: sqlx::PgPool, + ) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // One row per page and one probe per request: the smallest configuration in + // which "stopped early" and "read everything" are distinguishable. + state.ipfs_legacy_scan_page_rows = 1; + state.ipfs_max_legacy_probes = 1; + + for name in ["one", "two", "three"] { + seed_repo_with_blob( + &state, + tmp.path(), + "z6pager", + name, + format!("pager row {name}\n").as_bytes(), + ) + .await; + } + + // An oid no repo carries, so every probe reaches a clean absent verdict and the + // only thing that can cut the scan short is the budget under test. + let cid = seed_legacy_pin(&state, &absent_oid()).await; + crate::api::ipfs::reset_scan_rows(); + crate::api::ipfs::reset_preload_queries(); + let peer: SocketAddr = "203.0.113.90:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&cid, Some(peer))) + .await + .unwrap(); + + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "stopping early must taint the scan: a truncated search is a retryable 503, \ + never a definitive 404" + ); + assert_eq!( + crate::api::ipfs::scan_rows(), + 1, + "the selection must materialize only the page it can afford to gate, never \ + the node's whole repo inventory (INV-10)" + ); + assert_eq!( + crate::api::ipfs::preload_queries(), + 1, + "and it must stop asking for pages once the probe budget is spent" + ); + } + + /// #173 (jatmn, INV-10): the pager is per REQUEST, not per oid candidate. + /// + /// The `pinned_cids` index is unique on the git oid but NOT on the cid, so one CID + /// can resolve to several oids and `get_by_cid` tries each. If the pager were + /// re-created inside that loop, every extra candidate would re-page the whole + /// inventory and the fan-out this fix removes would come straight back — a CID with + /// k source-less candidates would cost k full scans of the node. + /// + /// Two DISTINCT absent oids seeded under ONE cid, both source-less so both reach + /// `needs_scan`. Three candidate repos at one row per page, with the probe and visit + /// budgets left at their generous defaults so nothing truncates: the scan runs to + /// exhaustion and 404s honestly. The whole request must cost ONE pass — 4 page + /// queries (3 full pages plus the short page that proves exhaustion) and 3 rows — + /// because the second candidate re-reads rows the first already paid for. + /// + /// MUTATION (RED): shadow `pager` with a fresh `LegacyScanPager::default()` inside + /// the `for sha256_hex in &oids` loop and the counters double to 8 and 6. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_second_oid_candidate_reuses_pages_from_the_first(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // One row per page so each page is individually visible in the counters. The + // probe and visit budgets stay at their defaults: this is about page REUSE, so + // nothing may truncate. + state.ipfs_legacy_scan_page_rows = 1; + + for name in ["one", "two", "three"] { + seed_repo_with_blob( + &state, + tmp.path(), + "z6reuse", + name, + format!("reuse row {name}\n").as_bytes(), + ) + .await; + } + + // Two oids no repo carries, sharing one cid: every probe reaches a clean absent + // verdict, so the scan completes for both candidates and nothing taints. + let first_oid = absent_oid(); + let second_oid = "f3".repeat(32); + let cid = seed_legacy_pin(&state, &first_oid).await; + state + .db + .record_pinned_cid(&second_oid, &cid, None) + .await + .expect("co-locate a second source-less oid under the same cid"); + assert_eq!( + state.db.oids_for_cid(&cid).await.unwrap().len(), + 2, + "precondition: the CID must resolve to two candidates, or the reuse this \ + test is about never happens" + ); + + crate::api::ipfs::reset_scan_rows(); + crate::api::ipfs::reset_preload_queries(); + let peer: SocketAddr = "203.0.113.92:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&cid, Some(peer))) + .await + .unwrap(); + + assert_eq!( + resp.status(), + StatusCode::NOT_FOUND, + "every candidate reached a verdict under generous budgets, so the honest \ + answer is the definitive 404 — a 503 here would mean something truncated \ + and the counters below would be measuring the wrong thing" + ); + assert_eq!( + crate::api::ipfs::preload_queries(), + 4, + "the pager is per REQUEST: a second oid candidate re-reads the pages the \ + first already paid for and must never re-query. Expected one pass over 3 \ + repos at 1 row per page = 4 page queries (3 full + the short page that \ + proves exhaustion); a per-candidate pager reads 8" + ); + assert_eq!( + crate::api::ipfs::scan_rows(), + 3, + "and one pass materializes each repo row exactly once (3), not once per \ + oid candidate (6)" + ); + } + + /// F2 buried-row repro: with more readable repos than `ipfs_max_repos_walked`, + /// existing PUBLIC content past the cap must still serve. The cap counts /// EXPENSIVE walks only — this request has no path-scoped rules anywhere, so it /// runs ZERO walks (the fake-git walk log stays empty) and the cap can never cut - /// the scan: the blob buried in the OLDER-updated repo (iterated last under - /// `list_all_repos`' updated_at DESC) serves its 200. Before F2 the cap counted + /// the scan: the blob buried in the LAST-iterated repo serves its 200. Iteration + /// is `(created_at, id)` ASC since the scan was paged (#173, jatmn), so the + /// blob-carrying repo is seeded LAST to keep it buried. Before F2 the cap counted /// visibility-passing VISITS and broke the loop into the opaque 404 — existing /// content misreported absent because of unrelated repos. MUTATION (RED): count /// visits against the cap again (re-add the check+increment at the visibility @@ -964,28 +3088,30 @@ mod tests { cfg.ipfs_max_repos_walked = 1; state.config = Arc::new(cfg); - // Seed the blob-carrying repo FIRST so its updated_at is OLDER: the empty - // repo is iterated first and the blob row sits past the old visit budget. - let (_, oid) = seed_repo_with_blob( + // Seed the blob-carrying repo LAST so its created_at is NEWEST: under the + // paged `(created_at, id)` ASC order the empty repo is iterated first and the + // blob row sits past the old visit budget. + seed_repo_with_blob( &state, tmp.path(), "z6f2buried", - "buried", - b"buried row proof\n", + "fresh", + b"unrelated content\n", ) .await; - seed_repo_with_blob( + let (_, oid) = seed_repo_with_blob( &state, tmp.path(), "z6f2buried", - "fresh", - b"unrelated content\n", + "buried", + b"buried row proof\n", ) .await; let peer: SocketAddr = "203.0.113.60:5000".parse().unwrap(); + let cid = seed_legacy_pin_for_oid(&state, &oid).await; let resp = ipfs_router(state) - .oneshot(get_cid(&cid_for_oid(&oid), Some(peer))) + .oneshot(get_cid(&cid, Some(peer))) .await .unwrap(); assert_eq!( @@ -1009,7 +3135,7 @@ mod tests { /// F2 walk-cap skip-and-continue: exhausting `ipfs_max_repos_walked` skips the /// walk-NEEDING repo without a verdict but keeps the scan alive. Three public - /// repos carry the same blob, newest first: the first (path-scoped) consumes the + /// repos carry the same blob, in iteration order: the first (path-scoped) consumes the /// cap-of-1 walk and denies (empty allowed-set — a verdict); the second /// (path-scoped) needs a walk the cap forbids and is skipped WITHOUT one (taint); /// the third is plain public and serves the 200 from a cheap probe — found beats @@ -1031,15 +3157,16 @@ mod tests { cfg.ipfs_max_repos_walked = 1; state.config = Arc::new(cfg); - // Insert order = oldest first, so iteration (updated_at DESC) is reversed: - // gatedwalk, then gatedskip, then pubcopy. Identical content -> one CID. + // Iteration is `(created_at, id)` ASC since the scan was paged (#173, jatmn), + // so insert order IS iteration order: gatedwalk, then gatedskip, then pubcopy. + // Identical content -> one CID. let content = b"skip and continue proof\n"; - let (_, oid) = - seed_repo_with_blob(&state, tmp.path(), "z6f2skip", "pubcopy", content).await; - let (skip_id, _) = - seed_repo_with_blob(&state, tmp.path(), "z6f2skip", "gatedskip", content).await; let (walk_id, _) = seed_repo_with_blob(&state, tmp.path(), "z6f2skip", "gatedwalk", content).await; + let (skip_id, _) = + seed_repo_with_blob(&state, tmp.path(), "z6f2skip", "gatedskip", content).await; + let (_, oid) = + seed_repo_with_blob(&state, tmp.path(), "z6f2skip", "pubcopy", content).await; for id in [&walk_id, &skip_id] { state .db @@ -1055,8 +3182,9 @@ mod tests { } let peer: SocketAddr = "203.0.113.61:5000".parse().unwrap(); + let cid = seed_legacy_pin_for_oid(&state, &oid).await; let resp = ipfs_router(state) - .oneshot(get_cid(&cid_for_oid(&oid), Some(peer))) + .oneshot(get_cid(&cid, Some(peer))) .await .unwrap(); assert_eq!( @@ -1077,149 +3205,602 @@ mod tests { ); } - /// F2 visit ceiling: `ipfs_max_repo_visits` bounds the acquire+probe cost class - /// (each visit can be a full Tigris archive fetch on a cache miss). Unlike the - /// walk cap there is no cheap way to keep scanning, so exhaustion STOPS the scan - /// — and the stop is a truncation, not an absence: with ceiling 1 the newer - /// empty repo consumes the only visit and the blob-carrying older repo is never - /// probed, so the request sheds a retryable 503 + Retry-After, never a false - /// 404. MUTATION (RED): drop the ceiling check and the blob serves (200); drop - /// only the taint on the break and the 503 decays to a 404. + /// A reader DID that is on no rule in these fixtures, so every path-scoped rule + /// naming it denies the anonymous caller at the rule's path. + #[cfg(unix)] + const OTHER_READER: &str = "did:key:z6MkU3IpfsReaderCCCCCCCCCCCCCCCCCCCCCCCC"; + + /// Seed a repo holding `content` at `/src/secret.txt` and give it a path-scoped + /// rule over `/src/**` naming a reader that is not the caller. The repo stays + /// readable at "/" (the rule does not match "/", so the mirror row's public flag + /// decides), which is what makes the object cost a real allowed-set walk before it + /// is denied: a root deny would short-circuit ahead of the walk and spend nothing. + #[cfg(unix)] + async fn seed_path_denying_repo( + state: &crate::state::AppState, + tmp: &std::path::Path, + owner: &str, + name: &str, + content: &[u8], + ) -> (String, String) { + let (id, oid) = seed_repo_with_blob(state, tmp, owner, name, content).await; + state + .db + .set_visibility_rule( + &id, + "/src/**", + crate::db::VisibilityMode::B, + &[OTHER_READER.to_string()], + owner, + ) + .await + .expect("seed the path-scoped deny rule"); + (id, oid) + } + + /// F3 per-phase walk budgets: a PUBLIC source that only the legacy-scan fallback + /// can reach must still serve after path-scoped provenance denials spent the whole + /// walk cap. + /// + /// One shared `walks` counter made that impossible. Every provenance source that is + /// root-readable but path-scoped needs its own allowed-set walk to reach its deny, + /// so `MAX_PIN_SOURCES + 1` such sources consume the entire cap; the fallback the + /// at-cap/incomplete markers then arm has nothing left to spend, skips its first + /// walk-needing candidate at `walk-cap`, and the request tails to a retryable 503 + /// that every retry reproduces. A public object, permanently unservable. + /// + /// The existing buried-public test cannot see this: its extra repos do not exist on + /// disk, so they never reach the `!already` block and consume no walk. This fixture + /// uses REAL repos with REAL denying rules, and the walk log is what proves each one + /// genuinely spent a walk rather than being skipped for free. + /// + /// Both caps are set to 2, so `walk_cap` is 2 per phase. The first request runs + /// WITHOUT the fallback armed and pins the provenance phase's own bound (exactly 2 + /// walks, never more, for a complete source set). The second arms the fallback and + /// is the RED: pre-fix the public repo is skipped at `walk-cap` and the request 503s. + /// The third pins that the fresh scan budget is capacity, not a gate change: an + /// object held ONLY by a path-denying repo is still not served to the anonymous + /// caller, with the fallback armed for it too. + #[cfg(unix)] #[sqlx::test] - async fn get_by_cid_visit_ceiling_stops_scan_with_503(pool: sqlx::PgPool) { + async fn get_by_cid_fallback_reaches_public_source_past_provenance_walk_spend( + pool: sqlx::PgPool, + ) { let tmp = tempfile::TempDir::new().unwrap(); let mut state = crate::test_support::test_state(pool.clone()).await; let repos_dir = tmp.path().join("repos"); std::fs::create_dir_all(&repos_dir).unwrap(); state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + let walk_log = tmp.path().join("walks.log"); + state.git_bin = walk_logging_real_git(tmp.path(), &walk_log); + // `walk_cap` is the min of the two knobs, so both go to 2. + state.ipfs_max_history_walks = 2; let mut cfg = (*state.config).clone(); - cfg.ipfs_max_repo_visits = 1; + cfg.ipfs_max_repos_walked = 2; state.config = Arc::new(cfg); - // Blob repo first (older, iterated second); empty repo second (newer, - // consumes the single visit). - let (_, oid) = seed_repo_with_blob( + // Identical content in every holder, so one CID resolves to one oid that all of + // them carry. Iteration is `(created_at, id)` ASC, so insert order is scan order. + let content = b"per-phase walk budget proof\n"; + let (prov_one, oid) = + seed_path_denying_repo(&state, tmp.path(), "z6f3phase", "provdeny-one", content).await; + let (prov_two, _) = + seed_path_denying_repo(&state, tmp.path(), "z6f3phase", "provdeny-two", content).await; + // The fallback's holder. Its rule IS path-scoped (so the object still costs a + // walk) but covers a path this object is not at, so the walk's allowed-set + // decides on the mirror row's public flag and ALLOWS. A path-scoped rule can + // never name an anonymous reader, so this is the only shape in which a walked + // repo authorizes anon. + let (public_id, _) = + seed_repo_with_blob(&state, tmp.path(), "z6f3phase", "pubreach", content).await; + state + .db + .set_visibility_rule( + &public_id, + "/decoy/**", + crate::db::VisibilityMode::B, + &[OTHER_READER.to_string()], + "z6f3phase", + ) + .await + .unwrap(); + // A second object held ONLY by a path-denying repo, for the denial-class check. + let denied_content = b"held only where anon is denied\n"; + let (denied_id, denied_oid) = seed_path_denying_repo( &state, tmp.path(), - "z6f2visit", - "buried", - b"visit ceiling proof\n", + "z6f3phase", + "deniedsolo", + denied_content, ) .await; - seed_repo_with_blob(&state, tmp.path(), "z6f2visit", "fresh", b"unrelated\n").await; - let peer: SocketAddr = "203.0.113.62:5000".parse().unwrap(); - let resp = ipfs_router(state) - .oneshot(get_cid(&cid_for_oid(&oid), Some(peer))) + // Provenance: the two denying repos are the recorded sources of `oid`; the + // public holder is NOT, which is exactly the dropped-source case. + state.db.record_pin_source(&oid, &prov_one).await.unwrap(); + state.db.record_pin_source(&oid, &prov_two).await.unwrap(); + state + .db + .record_pin_source(&denied_oid, &denied_id) + .await + .unwrap(); + state + .db + .mark_pin_sources_incomplete(&denied_oid, "") .await .unwrap(); + + let cid = seed_legacy_pin_for_oid(&state, &oid).await; + let denied_cid = seed_legacy_pin_for_oid(&state, &denied_oid).await; + let router = ipfs_router(state.clone()); + + // 1. Provenance only: the source set carries no incompleteness signal, so no + // fallback runs. Both sources walk and deny, and the phase spends its cap + // exactly, never more, whatever the fallback later gets. + let (status, body) = + status_and_body(router.clone().oneshot(get_cid(&cid, None)).await.unwrap()).await; assert_eq!( - resp.status(), - StatusCode::SERVICE_UNAVAILABLE, - "a visit-ceiling truncation must shed a retryable 503, not report absent" + status, + StatusCode::NOT_FOUND, + "a complete source set that denies everywhere is a definitive miss: {body}" ); assert_eq!( - resp.headers() - .get("retry-after") - .and_then(|h| h.to_str().ok()), - Some("1"), - "the truncation 503 must carry Retry-After" + walks_logged(&walk_log), + 2, + "the provenance phase must spend exactly its own walk_cap of 2: two REAL \ + path-denying sources, each walked to reach its deny" ); - } - - /// F2 negative arm: a COMPLETE scan that finds nothing keeps its definitive 404 - /// — the truncation 503 must never fire when every candidate reached a verdict. - /// Two public repos both probe clean (the requested CID is nowhere), no rules, - /// no cap or ceiling hit: 404 with no Retry-After. MUTATION (RED): taint the - /// scan unconditionally and this decays into a 503. - #[sqlx::test] - async fn get_by_cid_complete_scan_keeps_definitive_404(pool: sqlx::PgPool) { - let tmp = tempfile::TempDir::new().unwrap(); - let mut state = crate::test_support::test_state(pool.clone()).await; - let repos_dir = tmp.path().join("repos"); - std::fs::create_dir_all(&repos_dir).unwrap(); - state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); - state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; - - seed_repo_with_blob(&state, tmp.path(), "z6f2clean", "one", b"content one\n").await; - seed_repo_with_blob(&state, tmp.path(), "z6f2clean", "two", b"content two\n").await; - // valid_cid() is the "hello" blob — present in neither repo. - let peer: SocketAddr = "203.0.113.63:5000".parse().unwrap(); - let resp = ipfs_router(state) - .oneshot(get_cid(&valid_cid(), Some(peer))) + // 2. Arm the fallback (the node's own record that a source is missing) and the + // buried public holder must serve, on the scan phase's own budget. + state + .db + .mark_pin_sources_incomplete(&oid, "") + .await + .unwrap(); + std::fs::remove_file(&walk_log).unwrap(); + let resp = router.clone().oneshot(get_cid(&cid, None)).await.unwrap(); + let status = resp.status(); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) .await .unwrap(); assert_eq!( - resp.status(), - StatusCode::NOT_FOUND, - "a complete clean scan is a definitive absence — 404, never the 503 shed" + status, + StatusCode::OK, + "a public source reachable only through the fallback must serve even after \ + path-scoped provenance denials spent the whole walk cap: {}", + String::from_utf8_lossy(&body) ); - assert!( - resp.headers().get("retry-after").is_none(), - "a definitive 404 must not advertise a retry" + assert_eq!( + &body[..], + content.as_slice(), + "the served bytes must be the public holder's object" + ); + assert_eq!( + walks_logged(&walk_log), + 3, + "two provenance-phase walks plus ONE scan-phase walk: the phases hold \ + separate budgets and neither exceeds the cap of 2" + ); + + // 3. The fresh scan budget is capacity, not a gate change: an object held only + // where anon is denied stays denied, fallback armed and all. + std::fs::remove_file(&walk_log).unwrap(); + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid(&denied_cid, None)) + .await + .unwrap(), + ) + .await; + assert_ne!( + status, + StatusCode::OK, + "a path-scoped deny must still deny under the per-phase budgets: {body}" + ); + assert_eq!( + status, + StatusCode::NOT_FOUND, + "every holder of that object reached a real deny verdict, so the miss is \ + definitive rather than truncated: {body}" ); } - /// F2 acquire taint: a repo row with NO local copy over a Tigris backend that - /// stalls (a silent local endpoint — accepted, never answered) hits the 1s - /// acquire timeout at the read-acquire site. The skip carries no verdict, so the - /// scan is truncated: retryable 503 + Retry-After, never the old silent-skip 404. - /// MUTATION (RED): drop the taint on the acquire-timeout arm and this decays to - /// a 404. + /// F3 must-not: the per-phase split raises the total walk work to `2 * walk_cap` + /// and no further. Two provenance sources spend the provenance budget; three more + /// path-denying repos, none of them recorded sources, offer the fallback more + /// walk-needing candidates than its own budget. The scan takes two and skips the + /// rest at `walk-cap`, so the request tails to the tainted 503 rather than walking + /// on. + /// + /// The walk count is asserted BEFORE the status so an unbounded scan fails HERE, + /// on the bound, and not on some downstream difference. + #[cfg(unix)] #[sqlx::test] - async fn get_by_cid_acquire_timeout_taints_scan_to_503(pool: sqlx::PgPool) { + async fn get_by_cid_per_phase_walk_budgets_stay_bounded_at_twice_the_cap(pool: sqlx::PgPool) { let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; let repos_dir = tmp.path().join("repos"); std::fs::create_dir_all(&repos_dir).unwrap(); - let mut state = crate::test_support::test_state(pool.clone()).await; - // Endpoint-pinned test client (no AWS_* env reads — env is racy under a - // parallel test run); the silent local endpoint stalls the HEAD - // deterministically. - let endpoint = crate::test_support::silent_http_endpoint().await; - let tigris = - crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint) - .await; - state.repo_store = crate::git::repo_store::RepoStore::new(repos_dir, Some(tigris), pool); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + let walk_log = tmp.path().join("walks.log"); + state.git_bin = walk_logging_real_git(tmp.path(), &walk_log); + state.ipfs_max_history_walks = 2; let mut cfg = (*state.config).clone(); - cfg.git_acquire_timeout_secs = 1; + cfg.ipfs_max_repos_walked = 2; state.config = Arc::new(cfg); - // Row exists in the DB but has no local copy, so the read acquire must - // consult Tigris (local-miss path) and stall until the timeout. + let content = b"bounded total walk work\n"; + let (prov_one, oid) = + seed_path_denying_repo(&state, tmp.path(), "z6f3total", "provdeny-one", content).await; + let (prov_two, _) = + seed_path_denying_repo(&state, tmp.path(), "z6f3total", "provdeny-two", content).await; + for name in ["fallback-one", "fallback-two", "fallback-three"] { + seed_path_denying_repo(&state, tmp.path(), "z6f3total", name, content).await; + } + state.db.record_pin_source(&oid, &prov_one).await.unwrap(); + state.db.record_pin_source(&oid, &prov_two).await.unwrap(); state .db - .upsert_mirror_repo("z6f2acq", "ghost", "/unused-ghost", None, false) + .mark_pin_sources_incomplete(&oid, "") .await .unwrap(); - let peer: SocketAddr = "203.0.113.64:5000".parse().unwrap(); - let resp = ipfs_router(state) - .oneshot(get_cid(&valid_cid(), Some(peer))) - .await - .unwrap(); - assert_eq!( - resp.status(), - StatusCode::SERVICE_UNAVAILABLE, - "an acquire timeout leaves the repo unproven — the scan must shed 503, not 404" + let cid = seed_legacy_pin_for_oid(&state, &oid).await; + let (status, body) = status_and_body( + ipfs_router(state) + .oneshot(get_cid(&cid, None)) + .await + .unwrap(), + ) + .await; + let walks = walks_logged(&walk_log); + assert!( + walks <= 4, + "total walk work must stay within 2 * walk_cap = 4 however many walk-needing \ + candidates the fallback is offered, got {walks}" ); assert_eq!( - resp.headers() - .get("retry-after") - .and_then(|h| h.to_str().ok()), - Some("1"), - "the truncation 503 must carry Retry-After" + status, + StatusCode::SERVICE_UNAVAILABLE, + "the fallback's own budget runs out on the surplus candidates, so the tail \ + is the truncated-search 503: {body}" ); + assert_eq!(body["error"], "search_incomplete", "{body}"); } - /// F2 found-beats-taint on the acquire arm: an acquire timeout taints the - /// scan but must NOT stop it — the loop `continue`s, and a later repo that - /// genuinely carries the object still serves. The NEWER row (visited first - /// under `list_all_repos`' updated_at DESC) is a Tigris-backed ghost whose - /// acquire stalls against the silent endpoint and times out at 1s; the - /// OLDER row is a plain public repo carrying the blob, reached next and + /// F2 (#173 round 15): the derived work floor must fit ONE COMPLETE COMBINED + /// resolution, provenance walks included, not just the legacy search. + /// + /// `AppState::ipfs_work_budget` floors the per-IP work bucket at + /// `ipfs_max_legacy_probes + pages`, but the SAME bucket is debited once per + /// provenance visibility walk, before the fallback the markers arm has run at all. + /// So with `GITLAWB_IPFS_RATE_LIMIT` below the floor (the only configuration where + /// the floor is what sizes the bucket), the provenance phase eats into the budget + /// the floor exists to reserve for the search, and the "one complete legacy search + /// per window" guarantee stops holding: the search 429s short of its configured + /// reach, and the retry re-pays the same provenance charges. + /// + /// The seams, stated the way the sibling fixtures do, and the ledger they produce: + /// + /// * `ipfs_rate_limit = 1`, below the floor, so the floor is what binds. + /// * `ipfs_max_legacy_probes = 4`, above the three probes the scan spends, so the + /// probe ceiling is NOT what stops the holder (it is a second brake that can + /// strand it independently of the work bucket, which is why the GREEN is + /// asserted as a SERVED 200 rather than as merely not-429). + /// * `ipfs_max_legacy_scan_rows = 128`, one page at the production page size, so + /// the scan buys exactly one page toll. + /// * `ipfs_max_repos_walked = 2`, so `walk_cap` is `min(17, 2) = 2` and exactly + /// fits the two path-scoped provenance deniers per phase. + /// * `ipfs_max_repo_visits` stays at its 1024 default against the 5 visits here, + /// so no other ceiling binds. + /// + /// Debits, in order: 2 provenance walks (the `!legacy_scan` charge, one per denier, + /// with no probe toll on that phase), 1 page toll, then one probe per legacy + /// candidate. The two deniers are re-visited by the scan for free as far as WALKS + /// go (the allowed-set memo persists across phases) but each still pays its probe, + /// so the holder's own probe is the SIXTH debit. + /// + /// Old floor `4 + 1 = 5`: that sixth debit finds the bucket empty, + /// `gate_and_serve` returns `Throttled` WITHOUT tainting, and the tail renders the + /// work-path 429. New floor `4 + 1 + min(17, 2) = 7`: the holder is reached, + /// walked on the scan phase's own budget, and served, with one token to spare. + /// + /// The route limiter is deliberately left at `test_support`'s default rather than + /// sized from this cfg. `ipfs_router` layers no `rate_limit_by_ip` at all, so that + /// saves nothing today, but a route bucket sized from `ipfs_rate_limit = 1` would + /// shed the request at the door and the RED would be a 429-vs-429 collision with no + /// discriminant. For the same reason the RED assertion pins the "ipfs retrieval" + /// prefix: the route brake's body is "rate limit exceeded", a substring of the + /// work path's "ipfs retrieval rate limit exceeded", so a bare status check or the + /// shorter string cannot tell the two brakes apart. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_work_floor_fits_provenance_walks_plus_one_full_legacy_search( + pool: sqlx::PgPool, + ) { + use crate::state::AppState; + use clap::Parser; + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool.clone()); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let cfg = crate::config::Config::parse_from([ + "gitlawb-node", + "--ipfs-rate-limit", + "1", + "--ipfs-max-legacy-probes", + "4", + "--ipfs-max-legacy-scan-rows", + "128", + "--ipfs-max-repos-walked", + "2", + ]); + // The knobs live in TWO places: `build_state` seeds the probe and scan-row + // ceilings the resolver enforces as AppState fields from constants, independent + // of Config, while `walk_cap` reads `state.config.ipfs_max_repos_walked`. A cfg + // installed without the seams would size the bucket from one set of values and + // run the scan under another. + state.ipfs_max_legacy_probes = AppState::ipfs_legacy_probe_budget(&cfg); + state.ipfs_max_legacy_scan_rows = AppState::ipfs_legacy_scan_row_budget(&cfg); + assert_eq!( + state.ipfs_legacy_scan_page_rows, + crate::api::ipfs::LEGACY_SCAN_PAGE_ROWS, + "fixture precondition: the page seam stays at the production page size, so \ + the row ceiling above is exactly one page and the scan buys one page toll" + ); + assert_eq!( + state.ipfs_max_history_walks, + crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, + "fixture precondition: the history-walk seam stays at the constant, so \ + walk_cap is min(17, 2) = 2 and the repos-walked knob is what binds" + ); + state.config = Arc::new(cfg.clone()); + // The bucket is sized from the seam under test, never by hand: that is what + // makes the floor change, and nothing else, the difference between RED and GREEN. + let floor = AppState::ipfs_work_budget(&cfg); + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(floor, std::time::Duration::from_secs(3600)); + + // Identical content everywhere, so one CID resolves to one oid all three repos + // carry. Scan order is `(created_at, id)` ASC and the holder must be paged AFTER + // both deniers, or its probe is not the debit that finds the bucket empty. + let content = b"one complete combined resolution\n"; + let (prov_one, oid) = + seed_path_denying_repo(&state, tmp.path(), "z6f2floor", "provdeny-one", content).await; + let (prov_two, _) = + seed_path_denying_repo(&state, tmp.path(), "z6f2floor", "provdeny-two", content).await; + // The holder's rule IS path-scoped, so reaching its verdict still costs a walk, + // but it covers a path this object is not at, so the walk's allowed set decides + // on the mirror row's public flag and ALLOWS an anonymous reader. + let (holder_id, _) = + seed_repo_with_blob(&state, tmp.path(), "z6f2floor", "holder", content).await; + state + .db + .set_visibility_rule( + &holder_id, + "/decoy/**", + crate::db::VisibilityMode::B, + &[OTHER_READER.to_string()], + "z6f2floor", + ) + .await + .unwrap(); + stamp_scan_order(&pool, &prov_one, 0).await; + stamp_scan_order(&pool, &prov_two, 1).await; + stamp_scan_order(&pool, &holder_id, 2).await; + + // The two deniers are the recorded sources; the holder is not, which is the + // dropped-source case. Two sources sits well under MAX_PIN_SOURCES (16), so + // `pin_sources_at_cap` cannot arm the fallback: the durable incomplete marker is + // what arms it. + state.db.record_pin_source(&oid, &prov_one).await.unwrap(); + state.db.record_pin_source(&oid, &prov_two).await.unwrap(); + state + .db + .mark_pin_sources_incomplete(&oid, "") + .await + .unwrap(); + let cid = seed_legacy_pin_for_oid(&state, &oid).await; + + let work_bucket = state.ipfs_work_rate_limiter.clone(); + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.173:5000".parse().unwrap(); + let resp = router.oneshot(get_cid(&cid, Some(peer))).await.unwrap(); + let status = resp.status(); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let rendered = String::from_utf8_lossy(&body).to_string(); + // Drain what the request left, so the failure messages carry the measured + // ledger rather than only its consequence. + let mut spare = 0usize; + while work_bucket.check("203.0.113.173").await { + spare += 1; + } + + assert!( + !rendered.contains("ipfs retrieval"), + "the work floor must reserve a full legacy search AFTER the provenance \ + phase has taken its walks off the same bucket. The holder's own probe \ + found the bucket empty and the tail rendered the work-path 429 (floor \ + {floor}, {spare} of it unspent): {rendered}" + ); + assert_eq!( + status, + StatusCode::OK, + "the buried public holder must be SERVED within one window, not merely \ + spared the 429: the probe ceiling is a second brake that can strand it on \ + its own (floor {floor}, {spare} unspent): {rendered}" + ); + assert_eq!( + &body[..], + content.as_slice(), + "the served bytes must be the holder's object" + ); + assert_eq!( + spare, 1, + "the measured ledger is 2 provenance walks + 1 page toll + 3 legacy probes \ + = 6 debits against a floor of {floor}, so exactly one token is left. A \ + different remainder means the debit order moved and the RED above is no \ + longer pinned on the holder's probe" + ); + } + + /// F2 visit ceiling: `ipfs_max_repo_visits` bounds the acquire+probe cost class + /// (each visit can be a full Tigris archive fetch on a cache miss). Unlike the + /// walk cap there is no cheap way to keep scanning, so exhaustion STOPS the scan + /// — and the stop is a truncation, not an absence: with ceiling 1 the + /// first-iterated empty repo consumes the only visit and the blob-carrying repo + /// behind it is never probed, so the request sheds a retryable 503 + Retry-After, never a false + /// 404. MUTATION (RED): drop the ceiling check and the blob serves (200); drop + /// only the taint on the break and the 503 decays to a 404. + #[sqlx::test] + async fn get_by_cid_visit_ceiling_stops_scan_with_503(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + let mut cfg = (*state.config).clone(); + cfg.ipfs_max_repo_visits = 1; + state.config = Arc::new(cfg); + + // Empty repo seeded first, so under the paged `(created_at, id)` ASC order it + // is iterated first and consumes the single visit; the blob repo behind it is + // never probed. + seed_repo_with_blob(&state, tmp.path(), "z6f2visit", "fresh", b"unrelated\n").await; + let (_, oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6f2visit", + "buried", + b"visit ceiling proof\n", + ) + .await; + + let peer: SocketAddr = "203.0.113.62:5000".parse().unwrap(); + let cid = seed_legacy_pin_for_oid(&state, &oid).await; + let resp = ipfs_router(state) + .oneshot(get_cid(&cid, Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a visit-ceiling truncation must shed a retryable 503, not report absent" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the truncation 503 must carry Retry-After" + ); + } + + /// F2 negative arm: a COMPLETE scan that finds nothing keeps its definitive 404 + /// — the truncation 503 must never fire when every candidate reached a verdict. + /// Two public repos both probe clean (the requested CID is nowhere), no rules, + /// no cap or ceiling hit: 404 with no Retry-After. MUTATION (RED): taint the + /// scan unconditionally and this decays into a 503. + #[sqlx::test] + async fn get_by_cid_complete_scan_keeps_definitive_404(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + seed_repo_with_blob(&state, tmp.path(), "z6f2clean", "one", b"content one\n").await; + seed_repo_with_blob(&state, tmp.path(), "z6f2clean", "two", b"content two\n").await; + + // valid_cid() is the "hello" blob — present in neither repo. + let peer: SocketAddr = "203.0.113.63:5000".parse().unwrap(); + let cid = seed_legacy_pin(&state, &absent_oid()).await; + let resp = ipfs_router(state) + .oneshot(get_cid(&cid, Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::NOT_FOUND, + "a complete clean scan is a definitive absence — 404, never the 503 shed" + ); + assert!( + resp.headers().get("retry-after").is_none(), + "a definitive 404 must not advertise a retry" + ); + } + + /// F2 acquire taint: a repo row with NO local copy over a Tigris backend that + /// stalls (a silent local endpoint — accepted, never answered) hits the 1s + /// acquire timeout at the read-acquire site. The skip carries no verdict, so the + /// scan is truncated: retryable 503 + Retry-After, never the old silent-skip 404. + /// MUTATION (RED): drop the taint on the acquire-timeout arm and this decays to + /// a 404. + #[sqlx::test] + async fn get_by_cid_acquire_timeout_taints_scan_to_503(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + // Endpoint-pinned test client (no AWS_* env reads — env is racy under a + // parallel test run); the silent local endpoint stalls the HEAD + // deterministically. + let endpoint = crate::test_support::silent_http_endpoint().await; + let tigris = + crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint) + .await; + state.repo_store = crate::git::repo_store::RepoStore::new(repos_dir, Some(tigris), pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + let mut cfg = (*state.config).clone(); + cfg.git_acquire_timeout_secs = 1; + state.config = Arc::new(cfg); + + // Row exists in the DB but has no local copy, so the read acquire must + // consult Tigris (local-miss path) and stall until the timeout. + state + .db + .upsert_mirror_repo("z6f2acq", "ghost", "/unused-ghost", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.64:5000".parse().unwrap(); + let cid = seed_legacy_pin(&state, &absent_oid()).await; + let resp = ipfs_router(state) + .oneshot(get_cid(&cid, Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an acquire timeout leaves the repo unproven — the scan must shed 503, not 404" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the truncation 503 must carry Retry-After" + ); + } + + /// F2 found-beats-taint on the acquire arm: an acquire timeout taints the + /// scan but must NOT stop it — the loop `continue`s, and a later repo that + /// genuinely carries the object still serves. The FIRST-iterated row (the paged + /// scan orders on `(created_at, id)` ASC since #173/jatmn, so it is the row + /// created first) is a Tigris-backed ghost whose acquire stalls against the + /// silent endpoint and times out at 1s; the row behind it is a plain public + /// repo carrying the blob, reached next and /// served from a cheap probe — found beats taint: 200 with the blob bytes, /// never the truncation 503. MUTATION (RED): turn the acquire-timeout arm's /// `continue` into a `break` and the public copy never serves (503). @@ -1230,42 +3811,44 @@ mod tests { std::fs::create_dir_all(&repos_dir).unwrap(); let mut state = crate::test_support::test_state(pool.clone()).await; state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; - // Seed the blob repo through a LOCAL-ONLY store first, so seeding never - // consults the (deliberately unreachable) Tigris endpoint. + // Seed through a LOCAL-ONLY store first, so seeding never consults the + // (deliberately unreachable) Tigris endpoint. The ghost row goes in FIRST: + // it is a bare DB insert, and under the paged `(created_at, id)` ASC order + // the row created first is the row iterated first. state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir.clone(), pool.clone()); + state + .db + .upsert_mirror_repo("z6f2acqcont", "ghost", "/unused-ghost", None, false) + .await + .unwrap(); let content = b"acquire taint continue proof\n"; let (_, oid) = seed_repo_with_blob(&state, tmp.path(), "z6f2acqcont", "pubcopy", content).await; // Swap in a Tigris-backed store over the SAME repos_dir (the seeded bare - // repo stays a fast local hit) and add a NEWER ghost row with no local - // copy: its acquire consults the silent local endpoint and stalls to the - // 1s timeout (endpoint-pinned test client, no AWS_* env reads). + // repo stays a fast local hit). The ghost has no local copy, so its acquire + // consults the silent local endpoint and stalls to the 1s timeout + // (endpoint-pinned test client, no AWS_* env reads). let endpoint = crate::test_support::silent_http_endpoint().await; let tigris = crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint) .await; state.repo_store = crate::git::repo_store::RepoStore::new(repos_dir, Some(tigris), pool); - state - .db - .upsert_mirror_repo("z6f2acqcont", "ghost", "/unused-ghost", None, false) - .await - .unwrap(); let mut cfg = (*state.config).clone(); cfg.git_acquire_timeout_secs = 1; state.config = Arc::new(cfg); - // Ordering precondition: the ghost must be iterated FIRST (updated_at - // DESC — it was upserted after the blob repo), otherwise the pubcopy - // would serve before the taint ever fires and the continue-vs-break - // distinction would go untested. + // Ordering precondition: the ghost must be iterated FIRST, otherwise the + // pubcopy would serve before the taint ever fires and the continue-vs-break + // distinction would go untested. Read through the same paged selection the + // scan uses, so the precondition cannot drift from the real order. let order: Vec = state .db - .list_all_repos() + .list_repos_page_for_scan(None, 100) .await .unwrap() .into_iter() - .map(|r| r.name) + .map(|r| r.repo.name) .collect(); let ghost_pos = order.iter().position(|n| n == "ghost").unwrap(); let pub_pos = order.iter().position(|n| n == "pubcopy").unwrap(); @@ -1276,8 +3859,9 @@ mod tests { let peer: SocketAddr = "203.0.113.73:5000".parse().unwrap(); let started = std::time::Instant::now(); + let cid = seed_legacy_pin_for_oid(&state, &oid).await; let resp = ipfs_router(state) - .oneshot(get_cid(&cid_for_oid(&oid), Some(peer))) + .oneshot(get_cid(&cid, Some(peer))) .await .unwrap(); // The taint arm demonstrably FIRED on this run: the response can only @@ -1292,7 +3876,7 @@ mod tests { assert_eq!( resp.status(), StatusCode::OK, - "an acquire taint must not stop the scan — the later public copy serves" + "an acquire taint must not stop the scan: the later public copy serves" ); let body = axum::body::to_bytes(resp.into_body(), 1 << 20) .await @@ -1333,8 +3917,9 @@ mod tests { .unwrap(); let peer: SocketAddr = "203.0.113.65:5000".parse().unwrap(); + let cid = seed_legacy_pin(&state, &absent_oid()).await; let resp = ipfs_router(state) - .oneshot(get_cid(&valid_cid(), Some(peer))) + .oneshot(get_cid(&cid, Some(peer))) .await .unwrap(); assert_eq!( @@ -1394,8 +3979,9 @@ mod tests { std::fs::write(bare.join("HEAD"), b"junk\n").unwrap(); let peer: SocketAddr = "203.0.113.68:5000".parse().unwrap(); + let cid = seed_legacy_pin(&state, &absent_oid()).await; let resp = ipfs_router(state) - .oneshot(get_cid(&valid_cid(), Some(peer))) + .oneshot(get_cid(&cid, Some(peer))) .await .unwrap(); assert_eq!( @@ -1475,8 +4061,9 @@ mod tests { } let peer: SocketAddr = "203.0.113.69:5000".parse().unwrap(); + let cid = seed_legacy_pin(&state, &absent_oid()).await; let resp = ipfs_router(state) - .oneshot(get_cid(&valid_cid(), Some(peer))) + .oneshot(get_cid(&cid, Some(peer))) .await .unwrap(); assert_eq!( @@ -1582,8 +4169,9 @@ mod tests { std::fs::write(bare2.join("HEAD"), b"junk\n").unwrap(); let peer: SocketAddr = "203.0.113.71:5000".parse().unwrap(); + let cid = seed_legacy_pin(&state, &absent_oid()).await; let resp = ipfs_router(state) - .oneshot(get_cid(&valid_cid(), Some(peer))) + .oneshot(get_cid(&cid, Some(peer))) .await .unwrap(); assert_eq!( @@ -1668,8 +4256,9 @@ mod tests { ); let peer: SocketAddr = "203.0.113.66:5000".parse().unwrap(); + let cid = seed_legacy_pin_for_oid(&state, oid).await; let resp = ipfs_router(state) - .oneshot(get_cid(&cid_for_oid(oid), Some(peer))) + .oneshot(get_cid(&cid, Some(peer))) .await .unwrap(); assert_eq!( @@ -1717,8 +4306,9 @@ mod tests { } let peer: SocketAddr = "203.0.113.67:5000".parse().unwrap(); + let cid = seed_legacy_pin(&state, &absent_oid()).await; let resp = ipfs_router(state) - .oneshot(get_cid(&valid_cid(), Some(peer))) + .oneshot(get_cid(&cid, Some(peer))) .await .unwrap(); assert_eq!( @@ -1729,72 +4319,3666 @@ mod tests { ); } - /// F3 budget expiry mid-loop: one absolute request budget - /// (`ipfs_request_budget_secs`) bounds the whole admitted scan; per-repo - /// stages may not each draw a fresh timeout past it. Budget 1s, per-iteration - /// acquire timeout 2s; the NEWER row is a Tigris-backed ghost (no local copy, - /// silent local endpoint) whose acquire stalls, the OLDER row is a plain - /// public repo carrying the blob. The ghost's acquire runs clamped to the ~1s - /// remainder and times out; at the next repo the budget gate sees zero - /// remaining, taints "budget", and STOPS the scan, so the blob repo is never - /// visited (a visit would probe the healthy public copy and serve 200, which - /// the 503 assertion rules out) and the shed names the budget. Without the - /// budget the acquire would time out at its own 2s, the scan would continue, - /// and the buried blob would serve 200 (the recorded RED). MUTATION (RED): - /// remove the `request_deadline` capture (or make the remaining budget - /// infinite) and this serves 200 again. + // ---------------------------------------------------------------------------- + // #173 round 13, F2: the legacy scan's ROW ceiling, its caller-carried + // continuation token, and the per-page toll. + // + // The hole: the pager bought another page unless `walk.probes` or `walk.visits` + // was exhausted, but the gate returns Skip on quarantine and on a root-scope + // visibility deny BEFORE either counter increments. An all-quarantined or + // all-root-denying inventory therefore paged the node's entire repo table at zero + // probes, anonymously, retaining every row and rule set, while holding one of the + // scarce global walk permits for up to the whole request budget. + // ---------------------------------------------------------------------------- + + /// Scenario 0, the case every other scan test skips: a ceiling that EXCEEDS the + /// table, on a scan that was never resumed. This is the one path that still owes + /// the caller a definitive 404. + /// + /// The three fixtures above and beside it all park the holder PAST a ceiling, so + /// each one proves the truncating half: nothing unproven may answer 404. Nobody + /// was covering the converse, and it is the more dangerous direction to lose, + /// because a scan that quietly stops short and STILL answers 404 reports existing + /// content as absent. Here five rows sit under a ceiling of 64, the requested CID + /// genuinely resolves to nothing, and the scan runs off the end of the table with + /// its ceilings untouched. + /// + /// The counters are what make "ran to exhaustion" an observation rather than an + /// inference. `scan_rows` reaching all five says the walk covered the table, and + /// `scan_limit` at 8 says both asks went out at the full page size, so no budget + /// ever shortened one. A 404 with either counter short would be the false-absent + /// answer wearing the right status. + /// + /// MUTATION (RED): drop the `pager.resumed &&` guard on the wrapped-scan taint and + /// this exhausted scan taints as `scan-wrapped`, so the tail becomes a 503 and the + /// definitive answer is never reached. #[sqlx::test] - async fn get_by_cid_request_budget_expiry_stops_scan_with_503(pool: sqlx::PgPool) { - let tmp = tempfile::TempDir::new().unwrap(); - let repos_dir = tmp.path().join("repos"); - std::fs::create_dir_all(&repos_dir).unwrap(); - let mut state = crate::test_support::test_state(pool.clone()).await; + async fn get_by_cid_unresumed_scan_under_the_ceiling_is_a_definitive_404(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool).await; state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; - // Seed the blob repo through a LOCAL-ONLY store first, so seeding never - // consults the (deliberately unreachable) Tigris endpoint. - state.repo_store = - crate::git::repo_store::RepoStore::for_testing(repos_dir.clone(), pool.clone()); - let (_, oid) = seed_repo_with_blob( - &state, - tmp.path(), - "z6f3budget", - "buried", - b"budget expiry proof\n", + state.ipfs_legacy_scan_page_rows = 4; + // Well clear of the five rows below: the point is a ceiling that never binds. + state.ipfs_max_legacy_scan_rows = 64; + seed_root_denying_repos(&state, "underceiling", 5, 0).await; + let cid = seed_legacy_pin(&state, &absent_oid()).await; + + let peer: SocketAddr = "203.0.113.160:5000".parse().unwrap(); + crate::api::ipfs::reset_scan_rows(); + crate::api::ipfs::reset_scan_limit(); + let (status, body) = status_and_body( + ipfs_router(state) + .oneshot(get_cid_scan(&cid, Some(peer), None)) + .await + .unwrap(), ) .await; - // Swap in a Tigris-backed store over the SAME repos_dir (the seeded bare - // repo stays a fast local hit) and add a NEWER ghost row with no local - // copy: its acquire consults the silent local endpoint and stalls past - // the budget (endpoint-pinned test client, no AWS_* env reads). - let endpoint = crate::test_support::silent_http_endpoint().await; - let tigris = - crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint) - .await; - state.repo_store = crate::git::repo_store::RepoStore::new(repos_dir, Some(tigris), pool); - state - .db - .upsert_mirror_repo("z6f3budget", "ghost", "/unused-ghost", None, false) - .await - .unwrap(); - let mut cfg = (*state.config).clone(); - cfg.ipfs_request_budget_secs = 1; - cfg.git_acquire_timeout_secs = 2; - state.config = Arc::new(cfg); - let peer: SocketAddr = "203.0.113.70:5000".parse().unwrap(); - let resp = ipfs_router(state) - .oneshot(get_cid(&cid_for_oid(&oid), Some(peer))) - .await - .unwrap(); + let rows = crate::api::ipfs::scan_rows(); + let limit = crate::api::ipfs::scan_limit(); assert_eq!( - resp.status(), - StatusCode::SERVICE_UNAVAILABLE, - "an exhausted request budget must stop the scan with a retryable 503; \ - scanning on into the later public blob repo would have served 200" + rows, 5, + "the scan must reach every seeded row before it may call the object absent. \ + Read {rows} of 5" ); assert_eq!( - resp.headers() - .get("retry-after") + limit, 8, + "two asks at the full page size (4 + 4): with the ceiling far above the \ + table, nothing may shorten the query, and a shortened one would mean the \ + 404 below rested on a bounded walk. Asked for {limit}" + ); + assert_eq!( + status, + StatusCode::NOT_FOUND, + "a scan that ran off the end of the table with no ceiling touched and no \ + resume token has proven absence over the whole inventory, so the answer is \ + the definitive 404, not a truncation 503: {body}" + ); + assert!( + continuation_of(&body).is_none(), + "there is nothing left to resume, so a definitive 404 must carry no \ + continuation: {body}" + ); + assert_ne!( + body["error"], "search_incomplete", + "the answer is an absence, not a truncation: {body}" + ); + } + + /// Scenario 1: an all-root-denied inventory stops at the row ceiling. + /// + /// Every seeded repo is private and the caller is anonymous, so each row is a root + /// deny: no probe, no visit, and pre-fix nothing that could stop the pager. The + /// scan must stop at the ceiling, taint (so the tail is the retryable 503, never a + /// false 404), free the walk permit, and hand back a continuation token. + /// + /// MUTATION A (RED): delete the row-ceiling check and `scan_rows()` reads the whole + /// seeded inventory instead of one ceiling's worth. + #[sqlx::test] + async fn get_by_cid_denial_only_scan_stops_at_row_ceiling(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 4; + seed_root_denying_repos(&state, "deny", 12, 0).await; + let cid = seed_legacy_pin(&state, &absent_oid()).await; + + let walk_pool = state.git_ipfs_walk_semaphore.clone(); + let free_before = walk_pool.available_permits(); + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.140:5000".parse().unwrap(); + + crate::api::ipfs::reset_scan_rows(); + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), None)) + .await + .unwrap(), + ) + .await; + + // The row COUNT first: it is the cost this ceiling exists to bound, and a + // status-first ordering would attribute a missing ceiling to the tail instead. + let rows = crate::api::ipfs::scan_rows(); + assert_eq!( + rows, 4, + "the ceiling (4) bounds the DB-facing selection exactly; a denial-only \ + inventory must not page the whole table. Read {rows} of 12 seeded rows" + ); + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "a scan cut short at the row ceiling left rows unproven, so the honest tail \ + is the retryable 503, never a definitive 404; got body {body}" + ); + assert_eq!( + body["error"], "search_incomplete", + "the shed must name the incomplete search: {body}" + ); + assert!( + continuation_of(&body).is_some(), + "a ceiling truncation must hand back a continuation so a holder past the \ + ceiling is still reachable: {body}" + ); + assert_eq!( + walk_pool.available_permits(), + free_before, + "the shed must free the scarce walk admission, not hold it for the request budget" + ); + + // The follow-up is ADMITTED: the shed released the walk permit rather than + // parking it, so the next caller is not capacity-503'd behind it. + let (status, _) = status_and_body( + router + .oneshot(get_cid_scan(&cid, Some(peer), None)) + .await + .unwrap(), + ) + .await; + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "the follow-up must be admitted and reach the same truncation verdict, not \ + shed at capacity behind a held permit" + ); + } + + /// Scenario 2: an all-QUARANTINED inventory, same contract. Quarantine is the other + /// denial class that returns from the gate before a probe or a visit is spent, so a + /// ceiling keyed on either counter would miss it entirely. + /// + /// MUTATION A (RED): as scenario 1. + #[sqlx::test] + async fn get_by_cid_quarantined_only_scan_stops_at_row_ceiling(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 4; + seed_quarantined_repos(&state, "quar", 12).await; + let cid = seed_legacy_pin(&state, &absent_oid()).await; + + let peer: SocketAddr = "203.0.113.141:5000".parse().unwrap(); + crate::api::ipfs::reset_scan_rows(); + let (status, body) = status_and_body( + ipfs_router(state) + .oneshot(get_cid_scan(&cid, Some(peer), None)) + .await + .unwrap(), + ) + .await; + + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "a quarantined-only inventory truncates at the ceiling like a denied one: {body}" + ); + assert_eq!(body["error"], "search_incomplete", "{body}"); + let rows = crate::api::ipfs::scan_rows(); + assert_eq!( + rows, 4, + "quarantine costs neither a probe nor a visit, so only the ROW ceiling can \ + stop this pager, and it stops it exactly. Read {rows} of 12 seeded rows" + ); + assert!( + continuation_of(&body).is_some(), + "the quarantined-inventory truncation carries a continuation too: {body}" + ); + } + + /// Scenario 3 (must-not): a buried PUBLIC row inside the ceiling still serves. The + /// ceiling bounds the search; it must never convert reachable content into a shed. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_public_row_inside_row_ceiling_still_serves(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + // Ceiling comfortably ABOVE the inventory: nothing may truncate here. + state.ipfs_max_legacy_scan_rows = 64; + + seed_root_denying_repos(&state, "buried", 5, 0).await; + // Seeded last, and `upsert_mirror_repo` stamps `now`, so this row sorts after + // every 2020-stamped denial row and is genuinely reached last. + let (_, oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6inside", + "holder", + b"inside ceiling\n", + ) + .await; + let cid = seed_legacy_pin_for_oid(&state, &oid).await; + + let peer: SocketAddr = "203.0.113.142:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid_scan(&cid, Some(peer), None)) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::OK, + "a public holder inside the ceiling must serve; a ceiling that sheds \ + reachable content is worse than the unbounded scan it replaced" + ); + } + + /// Scenario 4 (must-not): no ceiling ever produces a 404. + /// + /// Three legs against one genuinely-absent object over 5 denial rows at ceiling 2: + /// * a front-started truncated scan is 503 `search_incomplete` WITH a token; + /// * a token-resumed scan that reaches the table end taints `scan-wrapped` and + /// emits NO token (absence was proven only over `[start, end)`); + /// * only a front-started scan that exhausts under every ceiling reaches the 404. + /// + /// MUTATION B (RED): replace taint-and-break with a bare `break` and the first leg + /// becomes the 404 tail. + #[sqlx::test] + async fn get_by_cid_row_ceiling_never_returns_404(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 2; + seed_root_denying_repos(&state, "no404", 5, 0).await; + let cid = seed_legacy_pin(&state, &absent_oid()).await; + + let router = ipfs_router(state.clone()); + let peer: SocketAddr = "203.0.113.143:5000".parse().unwrap(); + + // Leg 1: front-started truncation. + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), None)) + .await + .unwrap(), + ) + .await; + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "a truncated scan is never a 404: {body}" + ); + assert_eq!(body["error"], "search_incomplete", "{body}"); + let mut token = continuation_of(&body).expect("leg 1 must emit a continuation"); + + // Leg 2: ladder to the end. 5 rows at ceiling 2 truncates twice, then the third + // resume reads the short final page and WRAPS. + let mut wrapped = None; + for step in 0..6 { + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), Some(&token))) + .await + .unwrap(), + ) + .await; + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "every rung of the ladder over an absent object is a retryable 503, \ + never a 404 (step {step}): {body}" + ); + match continuation_of(&body) { + Some(next) => token = next, + None => { + wrapped = Some(body); + break; + } + } + } + let wrapped = wrapped.expect("the ladder must reach the table end within its bound"); + assert!( + wrapped["message"] + .as_str() + .unwrap_or_default() + .contains("scan-wrapped"), + "a resumed scan that reaches the end must taint scan-wrapped, so absence \ + proven only over [start, end) is never reported as a definitive 404: {wrapped}" + ); + assert!( + continuation_of(&wrapped).is_none(), + "a wrapped scan emits NO token, since there is nothing left to resume: {wrapped}" + ); + + // Leg 3: the 404 tail stays reachable for a front-started scan that exhausts + // under every ceiling. + let mut wide = state.clone(); + wide.ipfs_max_legacy_scan_rows = 1000; + let (status, body) = status_and_body( + ipfs_router(wide) + .oneshot(get_cid_scan(&cid, Some(peer), None)) + .await + .unwrap(), + ) + .await; + assert_eq!( + status, + StatusCode::NOT_FOUND, + "a front-started scan that exhausts under every ceiling still gets the \ + definitive 404; a ceiling that swallowed it would make every miss retryable \ + forever: {body}" + ); + } + + /// Scenario 5: a holder buried PAST the ceiling becomes servable within the stated + /// bound by echoing tokens, under the PRODUCTION toll at its raised derived floor. + /// + /// Ceiling 4 over 10 denial rows with the public holder behind them: the bound is + /// `ceil(10 / 4) + 1 = 4` requests. Every intermediate response is the retryable + /// 503 with a token, and no 429 interrupts the ladder, which is what the floor fix + /// pins. The work bucket is sized to the DERIVED floor of a config whose page term + /// dominates (probe knob 1, row knob 896 = 7 pages, walk knob 1, so floor = 9); + /// under the old floor (`max(route, probes)` = 1) the very first page would 429. + /// + /// The walk knob is pinned at 1 rather than left at its default of 64. This ladder + /// is a pure legacy scan with no provenance phase, so the floor's walk term + /// (`min(17, ipfs_max_repos_walked)`, #173 round 15) buys nothing the fixture + /// spends; at the default it would hand the bucket 17 tokens of slack and the page + /// toll, which is the thing this test exists to hold the floor against, would stop + /// being what binds. + /// + /// The floor is 9 rather than the honest ladder's exact cost (6 pages + 1 probe = 7) + /// on purpose. A ladder that never resumes re-pages from the front every request and + /// costs 8, so at a bucket of 7 mutation C would trip the 429 guard one step before + /// the reach guard and its RED would be attributed to the toll rather than to the + /// missing continuation. A token of headroom keeps each guard reporting its own + /// property. + /// + /// MUTATION C (RED): emit the token but never open it on the way in, and the ladder + /// restarts at the front every time so the 200 never arrives. + /// + /// This test has NO pre-fix RED, and that is by design rather than an omission. + /// Mutation A (delete the row ceiling) must leave it GREEN, which means its + /// assertions have to tolerate the holder being served on the very first request, + /// exactly what an unbounded scan does. So the pre-fix head passes it. Its + /// load-bearing proof is mutation C, its designated mutant: C keeps the ceiling and + /// keeps minting tokens but never honours one, which is the only shape that makes + /// the holder permanently unservable. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_holder_past_scan_ceiling_serves_via_token_ladder(pool: sqlx::PgPool) { + use crate::state::AppState; + use clap::Parser; + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 4; + + // The production toll, sized exactly at its derived floor. The row knob here + // sizes the FLOOR (it reads the production 128-row page size); the ceiling the + // scan actually enforces is the AppState seam above, as with page rows. + let cfg = crate::config::Config::parse_from([ + "gitlawb-node", + "--ipfs-rate-limit", + "1", + "--ipfs-max-legacy-probes", + "1", + "--ipfs-max-legacy-scan-rows", + "896", + "--ipfs-max-repos-walked", + "1", + ]); + let floor = AppState::ipfs_work_budget(&cfg); + assert_eq!( + floor, 9, + "fixture precondition: 1 probe + 896/128 = 7 pages + min(17, 1) = 1 walk" + ); + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(floor, std::time::Duration::from_secs(3600)); + + seed_root_denying_repos(&state, "ladder", 10, 0).await; + let (_, oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6ladder", + "holder", + b"past the ceiling\n", + ) + .await; + let cid = seed_legacy_pin_for_oid(&state, &oid).await; + + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.144:5000".parse().unwrap(); + let bound = 10usize.div_ceil(4) + 1; + + let mut token: Option = None; + let mut served_at = None; + for step in 1..=bound { + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), token.as_deref())) + .await + .unwrap(), + ) + .await; + assert_ne!( + status, + StatusCode::TOO_MANY_REQUESTS, + "no 429 may interrupt an honest caller's ladder at step {step}: the work \ + floor must fit a full deep scan's page toll, or the reach bound is a \ + promise the toll breaks: {body}" + ); + if status == StatusCode::OK { + served_at = Some(step); + break; + } + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "an intermediate rung is the retryable 503 (step {step}): {body}" + ); + assert_eq!(body["error"], "search_incomplete", "{body}"); + token = Some( + continuation_of(&body) + .unwrap_or_else(|| panic!("rung {step} must carry a continuation: {body}")), + ); + } + assert!( + served_at.is_some(), + "a holder past the ceiling must be served within ceil(10/4)+1 = {bound} \ + token-echoing requests, or the ceiling has made it permanently unservable" + ); + } + + /// Scenario 7 (#173 round 14, F4): an operator ceiling BELOW the page size must + /// bound the QUERY, not just the loop that reads its result. + /// + /// Page size 4, ceiling 2. The scan may prove two rows, so two rows is what it may + /// select and rule-load inside the admission-held, budget-clamped region. A fetch + /// that always asks for a full page buys twice the ceiling and the row arm only + /// notices afterwards, which makes the page size an implicit floor under the knob. + /// `scan_limit()` is what separates the fix from a post-fetch trim: it records the + /// DB-facing ask, so a trim that hands back the same two rows still reads 4 here. + /// + /// The fixture also carries the fail-open boundary case. The LAST row is a real bare + /// repo, public at "/", holding a real blob at /src/secret.txt behind a path-scoped + /// rule naming a reader the anonymous caller is not, and its pin is LEGACY (NULL + /// provenance) so the gate reads the page's own `pager.rules` rather than re-querying + /// per repo. Every fetch here is budget-shortened, so that repo arrives as the last + /// row of a shortened page: the boundary where a page carrying a rule set loaded for + /// a different row set would fail OPEN and serve a withheld object. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_scan_ceiling_below_page_size_bounds_the_query(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 4; + state.ipfs_max_legacy_scan_rows = 2; + // The probe, visit, and walk budgets stay at their defaults on purpose: the + // boundary repo below spends a probe, a visit, and a history walk that the + // denial-only rows do not, and a fixture that starved them would withhold it for + // a reason that has nothing to do with its rules. + + seed_root_denying_repos(&state, "capbelow", 6, 0).await; + // Both repos below are mirror rows, so `upsert_mirror_repo` stamps `now` and they + // sort after every 2020-stamped denial row. The boundary repo is seeded second, + // so `(created_at, id)` puts it eighth and last. + let (_, holder_oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6capbelow", + "holder", + b"past a ceiling below the page size\n", + ) + .await; + let holder_cid = seed_legacy_pin_for_oid(&state, &holder_oid).await; + let (_, withheld_oid) = seed_path_denying_repo( + &state, + tmp.path(), + "z6capbelow", + "boundary", + b"withheld at the boundary row\n", + ) + .await; + let withheld_cid = seed_legacy_pin_for_oid(&state, &withheld_oid).await; + + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.150:5000".parse().unwrap(); + + crate::api::ipfs::reset_scan_rows(); + crate::api::ipfs::reset_scan_limit(); + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&holder_cid, Some(peer), None)) + .await + .unwrap(), + ) + .await; + // Both counters sum over the whole request and are cleared only by their resets, + // so they are captured HERE, before the ladder below adds its own fetches. + let rows = crate::api::ipfs::scan_rows(); + let limit = crate::api::ipfs::scan_limit(); + assert_eq!( + limit, 2, + "one fetch, and it must ask the database for the ceiling (2), not the page \ + size (4). Asked for {limit}" + ); + assert_eq!( + rows, 2, + "a ceiling below the page size still bounds the selection exactly. Read \ + {rows} of 8 seeded rows" + ); + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "the holder is row 7, past the ceiling, so this request must not serve it \ + and must tail to the retryable 503: {body}" + ); + assert_eq!( + body["error"], "search_incomplete", + "the shed must name the incomplete search: {body}" + ); + + // The ladder still reaches the holder: rungs covering rows 3-4, 5-6, then 7-8. + let bound = 8usize.div_ceil(2) + 1; + let mut token = Some( + continuation_of(&body) + .unwrap_or_else(|| panic!("rung 1 must carry a continuation: {body}")), + ); + let mut served_at = None; + for step in 2..=bound { + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&holder_cid, Some(peer), token.as_deref())) + .await + .unwrap(), + ) + .await; + if status == StatusCode::OK { + served_at = Some(step); + break; + } + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "an intermediate rung is the retryable 503 (step {step}): {body}" + ); + token = Some( + continuation_of(&body) + .unwrap_or_else(|| panic!("rung {step} must carry a continuation: {body}")), + ); + } + assert!( + served_at.is_some(), + "a holder past the ceiling must still be served within ceil(8/2)+1 = {bound} \ + token-echoing requests; a ceiling that shortens the query must not shorten \ + the reach" + ); + + // The withheld blob gets its OWN full ladder, and is denied on every rung. + // + // A not-served assertion alone is satisfied by a ladder that never reached the + // boundary row: the per-IP work limiter ends one with a shed that carries no + // continuation, and so does an early taint, and a bare `None => break` cannot + // tell either from an honest exhaustion. So this half also witnesses HOW the + // ladder ended: every intermediate rung is specifically the retryable 503, and + // the last one is the `scan-wrapped` taint, which only a resumed scan that + // reached the END of the table emits. That is what makes "the rules withheld + // it" the reading. (Not a 404: a resumed scan has proven absence only over + // `[token, end)`, so the node deliberately withholds the definitive 404 and + // answers 503 with no continuation instead.) + let mut token: Option = None; + let mut exhausted_at = None; + for step in 1..=bound { + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&withheld_cid, Some(peer), token.as_deref())) + .await + .unwrap(), + ) + .await; + assert_ne!( + status, + StatusCode::OK, + "the /src/** rule names a reader the anonymous caller is not, so the \ + boundary repo's blob must never be served, including on the rung that \ + reaches it as the last row of a shortened page (step {step}): {body}" + ); + match continuation_of(&body) { + Some(t) => { + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "an intermediate rung of the withheld ladder is the retryable \ + 503 (step {step}): {body}" + ); + token = Some(t); + } + None => { + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "the withheld ladder's last rung is the retryable 503 (step \ + {step}): {body}" + ); + assert!( + body["message"] + .as_str() + .unwrap_or_default() + .contains("scan-wrapped"), + "the withheld ladder must end by EXHAUSTING the inventory, which \ + only the scan-wrapped taint witnesses, not by a per-IP brake \ + that stopped it short of the boundary row (step {step}): {body}" + ); + exhausted_at = Some(step); + break; + } + } + } + assert!( + exhausted_at.is_some(), + "the withheld ladder must reach the end of the inventory within {bound} \ + rungs, otherwise no rung ever evaluated the rule that withholds the blob" + ); + } + + /// The positive control for the fixture above: the identical inventory with the + /// `/src/**` rule ABSENT serves the same blob through the same ladder. + /// + /// Its job is attribution. Without it, a not-served assertion is satisfied by any + /// fixture that never reaches the repo at all (a spent probe, a skipped walk, a + /// budget cut), so a RED there could not be read as "the rules decided". This test + /// is GREEN before and after the ceiling fix; only the pairing carries meaning. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_boundary_repo_serves_the_same_blob_without_the_path_rule( + pool: sqlx::PgPool, + ) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 4; + state.ipfs_max_legacy_scan_rows = 2; + + seed_root_denying_repos(&state, "capbelowctl", 6, 0).await; + let (_, holder_oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6capbelowctl", + "holder", + b"past a ceiling below the page size\n", + ) + .await; + let _ = seed_legacy_pin_for_oid(&state, &holder_oid).await; + // Same recipe as the fixture above, minus the path-scoped rule. + let (_, allowed_oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6capbelowctl", + "boundary", + b"withheld at the boundary row\n", + ) + .await; + let allowed_cid = seed_legacy_pin_for_oid(&state, &allowed_oid).await; + + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.151:5000".parse().unwrap(); + let bound = 8usize.div_ceil(2) + 1; + + let mut token: Option = None; + let mut served_at = None; + for step in 1..=bound { + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&allowed_cid, Some(peer), token.as_deref())) + .await + .unwrap(), + ) + .await; + if status == StatusCode::OK { + served_at = Some(step); + break; + } + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "an intermediate rung is the retryable 503 (step {step}): {body}" + ); + token = Some( + continuation_of(&body) + .unwrap_or_else(|| panic!("rung {step} must carry a continuation: {body}")), + ); + } + assert!( + served_at.is_some(), + "the boundary repo is public at \"/\" and the rule is what withholds its \ + blob, so with the rule absent the same blob must be served within {bound} \ + rungs" + ); + } + + /// Scenario 8 (#173 round 14, F4): a ceiling that is not a multiple of the page size + /// must shorten the LAST fetch to what is left of the budget. + /// + /// Page size 2, ceiling 3. The first fetch may ask for a full page; the second may + /// ask for one row only. A pager that asks for a page either way overshoots the + /// operator's ceiling by a page on every scan whose ceiling is not an exact multiple, + /// which is the general case. `scan_limit()` reads 2 + 1 = 3 for the fix and 2 + 2 = + /// 4 for a pager that trims after the query. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_scan_ceiling_not_a_page_multiple_shortens_the_last_fetch( + pool: sqlx::PgPool, + ) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 3; + + seed_root_denying_repos(&state, "capodd", 6, 0).await; + // Seventh and last: `upsert_mirror_repo` stamps `now`, past every 2020 stamp. + let (_, holder_oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6capodd", + "holder", + b"past a ceiling that is not a page multiple\n", + ) + .await; + let holder_cid = seed_legacy_pin_for_oid(&state, &holder_oid).await; + + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.152:5000".parse().unwrap(); + + crate::api::ipfs::reset_scan_rows(); + crate::api::ipfs::reset_scan_limit(); + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&holder_cid, Some(peer), None)) + .await + .unwrap(), + ) + .await; + // Captured before the ladder: both counters sum across the whole request. + let rows = crate::api::ipfs::scan_rows(); + let limit = crate::api::ipfs::scan_limit(); + assert_eq!( + limit, 3, + "two fetches, of 2 then 1: the second may ask only for the remaining budget. \ + Asked for {limit} rows in total" + ); + assert_eq!( + rows, 3, + "the ceiling (3) bounds the selection exactly, page size (2) or not. Read \ + {rows} of 7 seeded rows" + ); + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "the holder is row 7, past the ceiling, so this request must not serve it \ + and must tail to the retryable 503: {body}" + ); + assert_eq!( + body["error"], "search_incomplete", + "the shed must name the incomplete search: {body}" + ); + + let bound = 7usize.div_ceil(3) + 1; + let mut token = Some( + continuation_of(&body) + .unwrap_or_else(|| panic!("rung 1 must carry a continuation: {body}")), + ); + let mut served_at = None; + for step in 2..=bound { + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&holder_cid, Some(peer), token.as_deref())) + .await + .unwrap(), + ) + .await; + if status == StatusCode::OK { + served_at = Some(step); + break; + } + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "an intermediate rung is the retryable 503 (step {step}): {body}" + ); + token = Some( + continuation_of(&body) + .unwrap_or_else(|| panic!("rung {step} must carry a continuation: {body}")), + ); + } + assert!( + served_at.is_some(), + "a holder past the ceiling must still be served within ceil(7/3)+1 = {bound} \ + token-echoing requests" + ); + } + + /// Seed `n` PUBLIC (root-READABLE) mirror rows in scan order, with disk paths that + /// do not exist. + /// + /// The distinction from `seed_root_denying_repos` is the whole point: a private row + /// is denied at the root gate before `walk.probes` moves, so a denial-only fixture + /// can never reach the probe or visit ceilings. These rows pass the root gate, so + /// each one is CHARGED a probe, which is what drives the pager to the probe ceiling. + async fn seed_root_readable_repos(state: &crate::state::AppState, prefix: &str, n: usize) { + for i in 0..n { + state + .db + .upsert_mirror_repo( + &format!("z6readable{prefix}"), + &format!("{prefix}-{i:04}"), + &format!("/nonexistent/{prefix}-{i:04}"), + None, + false, + ) + .await + .expect("seed a root-readable mirror row"); + } + } + + /// The continuation must survive a repo id at the WRITE PATH's maximum. + /// + /// `repos.id` is `{owner}/{name}`, and the node's own slug validators admit 255 + /// bytes of owner and 100 of name, so a 356-byte id is reachable and repo names are + /// peer-controllable. When such a row lands on a truncation boundary the seal is the + /// only thing standing between it and a tokenless 503, and a tokenless 503 is + /// byte-identical to the wrapped-scan answer whose contract is "your ladder is + /// over". The boundary row is deterministic for a stable inventory, so every retry + /// reproduces it and every row past it is permanently unreachable. + /// + /// MUTATION (RED): narrow the token's id width back to 128 and the shed loses its + /// continuation. + #[sqlx::test] + async fn get_by_cid_row_ceiling_continuation_survives_a_max_length_repo_id(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 1; + state.ipfs_max_legacy_scan_rows = 1; + + let owner = format!("did:key:{}", "z".repeat(247)); + assert_eq!( + owner.len(), + 255, + "the largest owner the slug validator admits" + ); + let name = "n".repeat(100); + let at = scan_order_stamp(0); + state + .db + .create_repo(&crate::db::RepoRecord { + id: format!("{owner}/{name}"), + name: name.clone(), + owner_did: owner.clone(), + description: None, + is_public: false, + default_branch: "main".to_string(), + created_at: at, + updated_at: at, + disk_path: "/nonexistent/max-length-id".to_string(), + forked_from: None, + machine_id: None, + }) + .await + .expect("seed the boundary row"); + + let cid = seed_legacy_pin(&state, &absent_oid()).await; + let peer: SocketAddr = "203.0.113.150:5000".parse().unwrap(); + let (status, body) = status_and_body( + ipfs_router(state) + .oneshot(get_cid_scan(&cid, Some(peer), None)) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "{body}"); + assert_eq!(body["error"], "search_incomplete", "{body}"); + assert!( + continuation_of(&body).is_some(), + "a truncation on a row whose id the write path admits must still carry a \ + continuation; without one the ladder ends here forever: {body}" + ); + } + + /// The PROBE ceiling must advance the ladder, not end it. + /// + /// `ipfs_max_legacy_probes` binds first on any inventory containing root-readable + /// repos, long before the row ceiling that does mint a token. A probe-ceiling break + /// with no continuation makes the shed tokenless, which reads to the caller as "the + /// ladder is over", so a holder past the probe ceiling is unreachable on every + /// retry. + /// + /// The fixture seeds ROOT-READABLE rows on purpose: every other scan test in this + /// file uses `seed_root_denying_repos`, and a root deny returns before a probe is + /// charged, which is exactly why the shipped suite could not see this. + /// + /// MUTATION (RED): drop the continuation from the probe-ceiling arm and the ladder + /// never reaches the holder. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_probe_ceiling_ladders_to_a_holder_past_it(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + // The row ceiling is deliberately far out of reach: the probe ceiling is what + // must stop this scan, and it is what must carry the ladder forward. + state.ipfs_max_legacy_scan_rows = 1024; + state.ipfs_max_legacy_probes = 2; + // Generous so no 429 interrupts an honest caller's ladder; the toll is covered + // by its own test. + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1024, std::time::Duration::from_secs(3600)); + + seed_root_readable_repos(&state, "probe", 6).await; + let (_, oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6probe", + "holder", + b"past the probes\n", + ) + .await; + let cid = seed_legacy_pin_for_oid(&state, &oid).await; + + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.151:5000".parse().unwrap(); + let bound = 6usize.div_ceil(2) + 1; + + let mut token: Option = None; + let mut served_at = None; + for step in 1..=bound { + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), token.as_deref())) + .await + .unwrap(), + ) + .await; + if status == StatusCode::OK { + served_at = Some(step); + break; + } + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "an intermediate rung is the retryable 503 (step {step}): {body}" + ); + assert_eq!(body["error"], "search_incomplete", "{body}"); + token = Some(continuation_of(&body).unwrap_or_else(|| { + panic!( + "the probe-ceiling shed at step {step} must carry a continuation; \ + a tokenless shed is indistinguishable from a finished ladder: {body}" + ) + })); + } + assert!( + served_at.is_some(), + "a holder past the probe ceiling must be reached within {bound} \ + token-echoing requests, not stranded forever" + ); + } + + /// The probe ceiling must ladder past the row it STOPPED ON, not past the page. + /// + /// The sibling test above sets `ipfs_legacy_scan_page_rows == ipfs_max_legacy_probes`, + /// so the budget runs out exactly at a page boundary and the page-boundary cursor + /// happens to be the right resume point. Misalign the two and it is not: the ceiling + /// taints INSIDE `gate_and_serve`, the loop keeps consuming the rest of the page as + /// `Skip`, and the mint arms at the top of the loop seal `pager.cursor`, which by + /// then sits PAST every row the ceiling refused to probe. Those rows are skipped on + /// the resume as well, and the inventory is stable, so every ladder step reproduces + /// the same gap. + /// + /// Two rows, one probe: the filler spends the budget, the holder is the row the + /// ceiling stops on. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_probe_ceiling_ladders_past_the_row_it_stopped_on(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool.clone()); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 1024; + // Deliberately NOT equal to the page size: one probe, two rows per page. + state.ipfs_max_legacy_probes = 1; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1024, std::time::Duration::from_secs(3600)); + + seed_root_readable_repos(&state, "midpage", 1).await; + stamp_scan_order(&pool, "z6readablemidpage/midpage-0000", 0).await; + let (holder_id, oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6midpage", + "holder", + b"stopped on this row\n", + ) + .await; + stamp_scan_order(&pool, &holder_id, 1).await; + let cid = seed_legacy_pin_for_oid(&state, &oid).await; + + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.161:5000".parse().unwrap(); + + let mut token: Option = None; + let mut served_at = None; + for step in 1..=4 { + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), token.as_deref())) + .await + .unwrap(), + ) + .await; + if status == StatusCode::OK { + served_at = Some(step); + break; + } + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "an intermediate rung is the retryable 503 (step {step}): {body}" + ); + token = Some(continuation_of(&body).unwrap_or_else(|| { + panic!("the probe-ceiling shed at step {step} must carry a continuation: {body}") + })); + } + assert!( + served_at.is_some(), + "the row the probe ceiling stopped on must be reachable on the ladder; \ + a cursor sealed past it strands it on every retry" + ); + } + + /// A ceiling reached on the FINAL page must still mint a continuation. + /// + /// `pager.exhausted` breaks at the top of the loop AHEAD of every mint arm, so a + /// probe or visit ceiling that taints inside `gate_and_serve` while the last page is + /// being walked sheds `search_incomplete` with no token at all. `gl ipfs get` reads a + /// tokenless shed as "the ladder is over" (that is the wrapped-scan contract), so a + /// holder on that page is unreachable, permanently, on an inventory that never + /// changes. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_probe_ceiling_on_the_final_page_still_mints_a_continuation( + pool: sqlx::PgPool, + ) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool.clone()); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // One page holds the whole inventory, so the scan is exhausted the moment it + // starts and the break at `pager.exhausted` is the one that fires. + state.ipfs_legacy_scan_page_rows = 8; + state.ipfs_max_legacy_scan_rows = 1024; + state.ipfs_max_legacy_probes = 1; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1024, std::time::Duration::from_secs(3600)); + + seed_root_readable_repos(&state, "finalpage", 1).await; + stamp_scan_order(&pool, "z6readablefinalpage/finalpage-0000", 0).await; + let (holder_id, oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6finalpage", + "holder", + b"on the last page\n", + ) + .await; + stamp_scan_order(&pool, &holder_id, 1).await; + let cid = seed_legacy_pin_for_oid(&state, &oid).await; + + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.162:5000".parse().unwrap(); + + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), None)) + .await + .unwrap(), + ) + .await; + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "a probe ceiling on the final page is an incomplete search, not a verdict: {body}" + ); + let token = continuation_of(&body).unwrap_or_else(|| { + panic!( + "a ceiling reached on the final page must still carry a continuation; \ + a tokenless shed tells the caller their ladder is over: {body}" + ) + }); + let (status, body) = status_and_body( + router + .oneshot(get_cid_scan(&cid, Some(peer), Some(&token))) + .await + .unwrap(), + ) + .await; + assert_eq!( + status, + StatusCode::OK, + "echoing the final-page continuation must reach the holder: {body}" + ); + } + + /// A ceiling on the final page of a RESUMED scan must keep its continuation. + /// + /// `pager.resumed && pager.exhausted` is the wrapped-scan tail: the caller has walked + /// to the end of the table, so there is nothing left to resume and the absent token + /// is the signal. That is only true when the walk actually reached the end. A ceiling + /// stopping part way through the last page leaves rows unwalked in front of the + /// cursor, and clearing the seal there strands them exactly as a tokenless shed does. + /// + /// Four rows, three per page, one probe: the third rung is the one that resumes into + /// a short page and stops on the holder. + /// + /// MUTATION (RED): drop `scan_continuation.is_none()` from the wrap clause. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_ceiling_on_a_resumed_final_page_keeps_its_continuation(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool.clone()); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 3; + state.ipfs_max_legacy_scan_rows = 1024; + state.ipfs_max_legacy_probes = 1; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1024, std::time::Duration::from_secs(3600)); + + seed_root_readable_repos(&state, "wrapguard", 3).await; + for i in 0..3 { + stamp_scan_order(&pool, &format!("z6readablewrapguard/wrapguard-{i:04}"), i).await; + } + let (holder_id, oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6wrapguard", + "holder", + b"behind a resumed ceiling\n", + ) + .await; + stamp_scan_order(&pool, &holder_id, 3).await; + let cid = seed_legacy_pin_for_oid(&state, &oid).await; + + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.163:5000".parse().unwrap(); + + let mut token: Option = None; + let mut served_at = None; + for step in 1..=6 { + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), token.as_deref())) + .await + .unwrap(), + ) + .await; + if status == StatusCode::OK { + served_at = Some(step); + break; + } + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "an intermediate rung is the retryable 503 (step {step}): {body}" + ); + token = Some( + continuation_of(&body) + .unwrap_or_else(|| panic!("rung {step} must carry a continuation: {body}")), + ); + } + assert!( + served_at.is_some(), + "a ceiling that stops part way through the last page of a resumed scan must \ + still ladder; the wrap tail is for a walk that reached the end" + ); + } + + /// A CID with several oid candidates must ladder to a holder only a LATER candidate + /// can serve. + /// + /// `pinned_cids` is unique on the oid, not the cid, so one CID resolves to several + /// candidates and every one of them shares the request's pager, budgets, and resume + /// slot. With a single shared slot the first candidate's truncation seals a row the + /// SECOND candidate never examined: the next rung resumes past the holder, the scan + /// wraps, and the tokenless shed tells the caller the ladder is over. The holder is + /// then unreachable on every retry, because the inventory never changes. + /// + /// Two rows and a two-probe ceiling, with the holder on the second row. The absent + /// candidate sorts first (`oids_for_cid` orders by hex), so it is the one that spends + /// the budget and the holder is reachable only through candidate 2. + /// + /// PRE-FIX (observed RED): rung 1 sheds a token sealing row 1, rung 2 resumes past it, + /// wraps, and sheds with NO token; the holder is never served. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_multi_oid_ladder_reaches_a_holder_only_a_later_candidate_serves( + pool: sqlx::PgPool, + ) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool.clone()); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 1024; + // Two probes: exactly the two rows, so candidate 1 spends the whole budget and + // candidate 2 cannot probe anything this rung. + state.ipfs_max_legacy_probes = 2; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1024, std::time::Duration::from_secs(3600)); + + seed_root_readable_repos(&state, "multioid", 1).await; + stamp_scan_order(&pool, "z6readablemultioid/multioid-0000", 0).await; + let (holder_id, holder_oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6multioid", + "holder", + b"only the later candidate can serve this\n", + ) + .await; + stamp_scan_order(&pool, &holder_id, 1).await; + let cid = seed_legacy_pin_for_oid(&state, &holder_oid).await; + + // A second, absent candidate under the SAME cid, sorting ahead of the holder's + // oid so the ordered candidate list puts it first. + let absent_first = "00".repeat(32); + state + .db + .record_pinned_cid(&absent_first, &cid, None) + .await + .expect("co-locate a second source-less oid under the same cid"); + let candidates = state.db.oids_for_cid(&cid).await.unwrap(); + assert_eq!( + candidates, + vec![absent_first.clone(), holder_oid.clone()], + "precondition: the holder's oid must be the SECOND candidate, or the \ + starvation this test is about never happens" + ); + + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.164:5000".parse().unwrap(); + + let mut token: Option = None; + let mut served_at = None; + for step in 1..=8 { + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), token.as_deref())) + .await + .unwrap(), + ) + .await; + if status == StatusCode::OK { + served_at = Some(step); + break; + } + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "an intermediate rung is the retryable 503 (step {step}): {body}" + ); + token = Some(continuation_of(&body).unwrap_or_else(|| { + panic!( + "rung {step} shed with no continuation, which tells the caller the \ + ladder is over while a later candidate still holds the object: {body}" + ) + })); + } + assert!( + served_at.is_some(), + "a holder reachable only through a later oid candidate must be served by \ + driving the ladder, not stranded behind the first candidate's seal" + ); + } + + /// Seed `n` root-readable filler rows in scan order, at ascending stamps from + /// `first`. They pass the root gate and hold nothing, so each costs exactly one probe + /// and reaches a clean absent verdict, which is what drives a scan to its probe + /// ceiling on a known row. + async fn seed_ladder_filler( + state: &crate::state::AppState, + pool: &sqlx::PgPool, + prefix: &str, + n: usize, + first: usize, + ) { + seed_root_readable_repos(state, prefix, n).await; + for i in 0..n { + stamp_scan_order( + pool, + &format!("z6readable{prefix}/{prefix}-{i:04}"), + first + i, + ) + .await; + } + } + + /// Open a continuation the node just minted, under the node's own key. The ladder + /// tests that assert WHICH candidate a rung names need the position itself; the status + /// code alone cannot tell "advanced to the next candidate" from "sealed a row of the + /// current one that happens to work". + fn opened(key: &[u8; 32], cid: &str, token: &str) -> gitlawb_core::scan_token::ScanPosition { + gitlawb_core::scan_token::open_scan_token(key, cid, token, chrono::Utc::now().timestamp()) + .expect("the node's own token must open under the node's own key") + } + + /// Mint a continuation the handler will accept, for the fixtures that need to start + /// mid-ladder rather than drive every rung to get there. + fn minted(key: &[u8; 32], cid: &str, sha256_hex: &str, row: (&str, &str)) -> String { + gitlawb_core::scan_token::seal_scan_token( + key, + cid, + &gitlawb_core::scan_token::ScanPosition { + created_at_key: row.0.to_string(), + id: row.1.to_string(), + sha256_hex: sha256_hex.to_string(), + }, + chrono::Utc::now().timestamp() + 300, + ) + .expect("seal a continuation for the fixture") + } + + /// The multi-candidate ladder TERMINATES, and the terminating shed lands exactly on + /// the rung in which the FINAL candidate reaches the end of the table. + /// + /// Every rung must make progress of one of two kinds: advance the row within the + /// resumed candidate, or advance to the next candidate. Neither an endless ladder nor + /// a rung that hands back a token it already issued is acceptable, and a tokenless + /// shed before the last candidate has been walked is the starvation bug wearing the + /// "ladder over" signal. + /// + /// Four rows at two per page against a two-probe ceiling, and two candidates neither + /// of which can serve. The ladder is then fully determined: candidate A takes rungs + /// 1 and 2 on rows (0,1) and (2,3), rung 3 walks A off the end and advances to B, + /// rungs 4 and 5 repeat the table for B, and rung 6 walks B off the end. There are no + /// provenance sources anywhere, so the visit budget is untouched when the scan starts + /// and the settled-no-row shed cannot fire here; the ONLY tokenless rung is the last. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_multi_oid_ladder_ends_when_the_final_candidate_reaches_the_end( + pool: sqlx::PgPool, + ) { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 1024; + state.ipfs_max_legacy_probes = 2; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1024, std::time::Duration::from_secs(3600)); + + seed_ladder_filler(&state, &pool, "term", 4, 0).await; + let first = "00".repeat(32); + let second = "11".repeat(32); + let cid = seed_legacy_pin(&state, &first).await; + state + .db + .record_pinned_cid(&second, &cid, None) + .await + .expect("co-locate a second source-less oid under the same cid"); + assert_eq!( + state.db.oids_for_cid(&cid).await.unwrap(), + vec![first, second], + "precondition: two candidates in a known order" + ); + + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.165:5000".parse().unwrap(); + + let mut token: Option = None; + let mut seen: Vec = Vec::new(); + let mut tokenless_at = None; + for step in 1..=12 { + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), token.as_deref())) + .await + .unwrap(), + ) + .await; + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "no candidate can serve, so every rung is the truncated-search 503 \ + (step {step}): {body}" + ); + assert_eq!(body["error"], "search_incomplete", "{body}"); + match continuation_of(&body) { + Some(t) => { + assert!( + !seen.contains(&t), + "rung {step} handed back a token it already issued, which is the \ + ladder spinning in place rather than advancing" + ); + seen.push(t.clone()); + token = Some(t); + } + None => { + tokenless_at = Some(step); + break; + } + } + } + assert_eq!( + tokenless_at, + Some(6), + "the ladder must end on the rung where the SECOND candidate walks off the end \ + of the table: two rungs of rows plus one wrap rung per candidate. An earlier \ + tokenless rung means a candidate was abandoned unexamined" + ); + } + + /// The tokenless shed did not widen: a single-candidate resumed scan that wraps with + /// nothing sealed still ends the ladder exactly as before. + /// + /// This is the negative control for the advance. The advance mints a token whenever a + /// finished candidate has a successor, so an implementation that forgets the successor + /// check would keep minting forever and the caller would never learn the search is + /// over. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_single_candidate_wrap_still_sheds_tokenless(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 1024; + state.ipfs_max_legacy_probes = 2; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1024, std::time::Duration::from_secs(3600)); + + seed_ladder_filler(&state, &pool, "solowrap", 2, 0).await; + let cid = seed_legacy_pin(&state, &absent_oid()).await; + assert_eq!( + state.db.oids_for_cid(&cid).await.unwrap().len(), + 1, + "precondition: exactly one candidate, so no advance is ever available" + ); + + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.166:5000".parse().unwrap(); + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), None)) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "{body}"); + let token = continuation_of(&body).expect("the probe ceiling mints rung 1"); + + let (status, body) = status_and_body( + router + .oneshot(get_cid_scan(&cid, Some(peer), Some(&token))) + .await + .unwrap(), + ) + .await; + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "a resumed scan that ran off the end has not covered the rows before the \ + token, so it is still the retryable shed: {body}" + ); + assert!( + body["message"] + .as_str() + .is_some_and(|m| m.contains("scan-wrapped")), + "and the reason must still be the wrap, not an advance: {body}" + ); + assert_eq!( + continuation_of(&body), + None, + "with no later candidate the wrap ends the ladder, and the absent token is \ + what tells the caller so: {body}" + ); + } + + /// The advance names the NEXT candidate at the front-of-table sentinel. + /// + /// Asserted on the token's contents rather than on the ladder's outcome, because the + /// outcome alone cannot tell "advanced to candidate B" from "sealed some row of + /// candidate A that happens to work". The sentinel matters on its own: candidate B has + /// walked nothing, so resuming it anywhere but the front skips rows for it. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_finished_candidate_advances_to_the_next_at_the_front_sentinel( + pool: sqlx::PgPool, + ) { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 1024; + state.ipfs_max_legacy_probes = 2; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1024, std::time::Duration::from_secs(3600)); + + seed_ladder_filler(&state, &pool, "advance", 2, 0).await; + let first = "00".repeat(32); + let second = "11".repeat(32); + let cid = seed_legacy_pin(&state, &first).await; + state + .db + .record_pinned_cid(&second, &cid, None) + .await + .unwrap(); + + let key = state.ipfs_scan_token_key.clone(); + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.167:5000".parse().unwrap(); + + let (_, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), None)) + .await + .unwrap(), + ) + .await; + let rung1 = continuation_of(&body).expect("rung 1 mints on the probe ceiling"); + let pos = gitlawb_core::scan_token::open_scan_token( + &key, + &cid, + &rung1, + chrono::Utc::now().timestamp(), + ) + .expect("the node's own token opens under the node's own key"); + assert_eq!( + pos.sha256_hex, first, + "rung 1 seals the candidate that was actually walking" + ); + assert!( + !pos.created_at_key.is_empty(), + "and it seals a real row, not the sentinel" + ); + + let (_, body) = status_and_body( + router + .oneshot(get_cid_scan(&cid, Some(peer), Some(&rung1))) + .await + .unwrap(), + ) + .await; + let rung2 = continuation_of(&body) + .expect("the finished candidate must advance the ladder, not end it"); + let pos = gitlawb_core::scan_token::open_scan_token( + &key, + &cid, + &rung2, + chrono::Utc::now().timestamp(), + ) + .expect("the advance token opens"); + assert_eq!( + pos.sha256_hex, second, + "the finished candidate hands the ladder to the NEXT candidate" + ); + assert_eq!( + (pos.created_at_key.as_str(), pos.id.as_str()), + ("", ""), + "at the front-of-table sentinel: the next candidate has walked nothing, so \ + any row cursor would skip rows for it" + ); + } + + /// On a FRONT-STARTED request a later candidate's stop is honest coverage, and it is + /// what mints rung 1 when the first candidate wraps under budget. + /// + /// The rule that silences later candidates is keyed on where the REQUEST started, not + /// on which candidate is walking. On a resumed request the pager holds only the suffix + /// from the caller's cursor, so a later candidate's walk covers a suffix and must not + /// seal. Front-started, the pager starts at the front and every candidate's row loop + /// covers the fetched table from the beginning, so the first candidate that has NOT + /// finished owns the seal, later candidates included. + /// + /// Three rows at three per page against a four-probe ceiling: candidate A walks all + /// three, wraps on the empty page after them, and seals nothing; candidate B spends + /// the fourth probe on row 0 and stops on row 1, with row 0 settled behind it. The + /// holder is row 2, reachable only for B. + /// + /// Over-applying the resumed-only rule here sheds a tainted TOKENLESS 503 at rung 1 + /// and the holder is never served. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_front_started_later_candidate_still_seals_its_stop_row(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool.clone()); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 3; + state.ipfs_max_legacy_scan_rows = 1024; + // One more probe than candidate A spends walking the whole table, so A wraps + // UNTRUNCATED and B gets exactly one probe before the ceiling stops it. + state.ipfs_max_legacy_probes = 4; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1024, std::time::Duration::from_secs(3600)); + + seed_ladder_filler(&state, &pool, "frontprop", 2, 0).await; + let (holder_id, holder_oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6frontprop", + "holder", + b"reachable only for the later candidate\n", + ) + .await; + stamp_scan_order(&pool, &holder_id, 2).await; + let cid = seed_legacy_pin_for_oid(&state, &holder_oid).await; + let absent_first = "00".repeat(32); + state + .db + .record_pinned_cid(&absent_first, &cid, None) + .await + .unwrap(); + assert_eq!( + state.db.oids_for_cid(&cid).await.unwrap(), + vec![absent_first, holder_oid.clone()], + "precondition: the holder's oid is the SECOND candidate" + ); + + let key = state.ipfs_scan_token_key.clone(); + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.168:5000".parse().unwrap(); + + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), None)) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "{body}"); + let rung1 = continuation_of(&body).unwrap_or_else(|| { + panic!( + "the first candidate wrapped under budget and sealed nothing, so the \ + LATER candidate's ceiling stop is the only thing that can mint rung 1; \ + a tokenless shed here ends a ladder that works today: {body}" + ) + }); + let pos = opened(&key, &cid, &rung1); + assert_eq!( + pos.sha256_hex, holder_oid, + "rung 1 belongs to the candidate that actually stopped" + ); + assert!( + !pos.created_at_key.is_empty(), + "and it seals that candidate's own stop row, not the front sentinel: it \ + walked from the front, so there is nothing to restart" + ); + + let (status, body) = status_and_body( + router + .oneshot(get_cid_scan(&cid, Some(peer), Some(&rung1))) + .await + .unwrap(), + ) + .await; + assert_eq!( + status, + StatusCode::OK, + "echoing rung 1 must reach the holder: {body}" + ); + } + + /// A candidate that a ceiling stopped PART WAY through the last page has not wrapped, + /// however the shared pager's exhausted flag reads. + /// + /// `pager.exhausted` is per REQUEST and is set the moment any short page comes back, + /// so it is true while rows the ceiling refused are still sitting in front of the + /// cursor. Reading it at the tail as the wrap witness marks the truncated candidate + /// finished, advances the ladder to the next one, and strands those rows forever. The + /// witness has to be the per-candidate exit the row loop actually took. + /// + /// Four rows at three per page against a one-probe ceiling. Rung 3 resumes into a + /// SHORT page (two rows), spends its probe on the first and is stopped on the second, + /// so the request ends with `exhausted` set and a row unwalked. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_truncation_on_an_exhausted_page_does_not_advance_the_candidate( + pool: sqlx::PgPool, + ) { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 3; + state.ipfs_max_legacy_scan_rows = 1024; + state.ipfs_max_legacy_probes = 1; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1024, std::time::Duration::from_secs(3600)); + + seed_ladder_filler(&state, &pool, "wrapwitness", 4, 0).await; + let first = "00".repeat(32); + let second = "11".repeat(32); + let cid = seed_legacy_pin(&state, &first).await; + state + .db + .record_pinned_cid(&second, &cid, None) + .await + .unwrap(); + + let key = state.ipfs_scan_token_key.clone(); + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.169:5000".parse().unwrap(); + + let mut token: Option = None; + for step in 1..=3 { + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), token.as_deref())) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "{body}"); + let t = continuation_of(&body) + .unwrap_or_else(|| panic!("rung {step} must carry a continuation: {body}")); + let pos = opened(&key, &cid, &t); + assert_eq!( + pos.sha256_hex, first, + "rung {step} stopped the FIRST candidate at a ceiling, so it is still \ + that candidate's rung. Rung 3 is the one that matters: it resumes into \ + a short page, so the shared exhausted flag is set while a row it refused \ + is still unwalked, and an implementation reading that flag as the wrap \ + witness advances here and strands the row" + ); + assert!( + !pos.created_at_key.is_empty(), + "rung {step} seals a real row, not the sentinel" + ); + token = Some(t); + } + + let (status, body) = status_and_body( + router + .oneshot(get_cid_scan(&cid, Some(peer), token.as_deref())) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "{body}"); + let pos = opened( + &key, + &cid, + &continuation_of(&body).expect("rung 4 walks the first candidate off the end"), + ); + assert_eq!( + (pos.sha256_hex.as_str(), pos.created_at_key.as_str()), + (second.as_str(), ""), + "only once the first candidate has actually walked every fetched row does \ + the ladder advance, and then to the front of the next candidate" + ); + } + + /// A resumed rung does not re-run the scans of candidates earlier rungs already + /// finished, and the skip lands after the provenance phase, not before it. + /// + /// `walk.probes` has no test seam, so the observable is the marker-query pair the + /// fallback gate runs per candidate that reaches `needs_scan`. Both candidates carry + /// recorded sources marked incomplete, so both would bump the counter if both were + /// scanned; resuming at the second must leave it at one. + /// + /// The counter also pins the skip's exact position. Skipping at the top of the oid + /// loop would cut off the provenance phase, which can serve outright; skipping inside + /// `needs_scan` would charge the skipped candidate two lookups for nothing and read 2 + /// here. + #[sqlx::test] + async fn get_by_cid_resumed_rung_skips_the_scans_of_earlier_candidates(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 1024; + state.ipfs_max_legacy_probes = 1024; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1024, std::time::Duration::from_secs(3600)); + + // One private repo, used both as the scan inventory and as the recorded pin + // source for each candidate: it denies at the root gate either way, so the + // provenance phase runs and serves nothing. + seed_root_denying_repos(&state, "skipearlier", 2, 0).await; + let source = "skipearlier-0000".to_string(); + let first = "00".repeat(32); + let second = "11".repeat(32); + let cid = seed_legacy_pin(&state, &first).await; + state + .db + .record_pinned_cid(&second, &cid, None) + .await + .unwrap(); + for oid in [&first, &second] { + state.db.record_pin_source(oid, &source).await.unwrap(); + // Incomplete keeps `needs_scan` true past a non-empty source set, which is + // what puts the marker pair on the path for every candidate that is NOT + // skipped. + state.db.mark_pin_sources_incomplete(oid, "").await.unwrap(); + } + + let key = state.ipfs_scan_token_key.clone(); + let token = minted(&key, &cid, &second, ("", "")); + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.170:5000".parse().unwrap(); + + crate::api::ipfs::reset_marker_queries(); + let (status, body) = status_and_body( + router + .oneshot(get_cid_scan(&cid, Some(peer), Some(&token))) + .await + .unwrap(), + ) + .await; + assert_eq!( + crate::api::ipfs::marker_queries(), + 1, + "only the resumed candidate owes a scan this rung; the one before it was \ + finished by an earlier rung and must not pay the fallback gate again: {body}" + ); + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "the resumed candidate walked the table from the sentinel but the rows \ + before the ladder started were skipped, so the honest tail is the retryable \ + shed: {body}" + ); + } + + /// A resumed request still lets LATER candidates serve off the pages it already + /// bought. They are silenced for sealing, not deferred. + /// + /// Skipping them would waste page fetches the caller has already paid for and would + /// turn a rung that could have ended the ladder outright into another 503. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_resumed_rung_still_serves_from_a_later_candidate(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool.clone()); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 1024; + state.ipfs_max_legacy_probes = 1024; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1024, std::time::Duration::from_secs(3600)); + + seed_ladder_filler(&state, &pool, "opportune", 1, 0).await; + let (holder_id, holder_oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6opportune", + "holder", + b"served off a page the resumed candidate bought\n", + ) + .await; + stamp_scan_order(&pool, &holder_id, 1).await; + let cid = seed_legacy_pin_for_oid(&state, &holder_oid).await; + let absent_first = "00".repeat(32); + state + .db + .record_pinned_cid(&absent_first, &cid, None) + .await + .unwrap(); + + let key = state.ipfs_scan_token_key.clone(); + let token = minted(&key, &cid, &absent_first, ("", "")); + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.171:5000".parse().unwrap(); + + let (status, body) = status_and_body( + router + .oneshot(get_cid_scan(&cid, Some(peer), Some(&token))) + .await + .unwrap(), + ) + .await; + assert_eq!( + status, + StatusCode::OK, + "a later candidate that can serve off the already-fetched rows must serve in \ + this same rung: {body}" + ); + } + + /// A token naming a candidate that is no longer pinned degrades to a front restart. + /// + /// The hex is sealed by the node so it cannot be forged, but an unpin between rungs + /// can retire it. The open path must then treat the token as absent: never resume + /// some other candidate at that row, never fabricate a 404 out of a table this + /// request has not looked at, and never panic on a lookup that misses. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_token_naming_an_unpinned_candidate_restarts_at_the_front( + pool: sqlx::PgPool, + ) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool.clone()); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 1024; + state.ipfs_max_legacy_probes = 1024; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1024, std::time::Duration::from_secs(3600)); + + seed_ladder_filler(&state, &pool, "stalehex", 1, 0).await; + let (holder_id, holder_oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6stalehex", + "holder", + b"still reachable after the sealed candidate went away\n", + ) + .await; + stamp_scan_order(&pool, &holder_id, 1).await; + let cid = seed_legacy_pin_for_oid(&state, &holder_oid).await; + + let key = state.ipfs_scan_token_key.clone(); + // A well-formed token under the node's own key, naming an oid the CID no longer + // resolves to, sealed at a row PAST the holder. Resuming it against the wrong + // candidate would skip the holder; treating it as absent restarts at the front. + let token = minted( + &key, + &cid, + &"cc".repeat(32), + (&scan_order_stamp(9).to_rfc3339(), "zzz/zzz"), + ); + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.172:5000".parse().unwrap(); + + let (status, body) = status_and_body( + router + .oneshot(get_cid_scan(&cid, Some(peer), Some(&token))) + .await + .unwrap(), + ) + .await; + assert_eq!( + status, + StatusCode::OK, + "a stale candidate identity restarts the scan at the front, so the holder is \ + still found: {body}" + ); + } + + /// The ladder only ever names the resumed candidate, or the one immediately after it. + /// + /// Three candidates, resumed at the first with ceilings that truncate it. Every rung + /// until the first candidate finishes must keep naming it, and the rung that finally + /// moves must hand the ladder to candidate 2 at the front, never skip to candidate 3. + /// Skipping one would mark it finished over a table it never walked. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_ladder_never_skips_a_candidate(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 1024; + state.ipfs_max_legacy_probes = 2; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1024, std::time::Duration::from_secs(3600)); + + seed_ladder_filler(&state, &pool, "noskip", 4, 0).await; + let first = "00".repeat(32); + let second = "11".repeat(32); + let third = "22".repeat(32); + let cid = seed_legacy_pin(&state, &first).await; + for oid in [&second, &third] { + state.db.record_pinned_cid(oid, &cid, None).await.unwrap(); + } + assert_eq!( + state.db.oids_for_cid(&cid).await.unwrap(), + vec![first.clone(), second.clone(), third.clone()], + "precondition: three candidates in a known order" + ); + + let key = state.ipfs_scan_token_key.clone(); + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.173:5000".parse().unwrap(); + + let mut token = Some(minted(&key, &cid, &first, ("", ""))); + let mut moved_to = None; + for step in 1..=8 { + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), token.as_deref())) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "{body}"); + let t = continuation_of(&body) + .unwrap_or_else(|| panic!("rung {step} must carry a continuation: {body}")); + let pos = opened(&key, &cid, &t); + assert_ne!( + pos.sha256_hex, third, + "rung {step} handed the ladder to the THIRD candidate while the second \ + had not been walked; that marks it finished over a table it never saw" + ); + if pos.sha256_hex != first { + moved_to = Some((pos.sha256_hex.clone(), pos.created_at_key.clone())); + break; + } + token = Some(t); + } + assert_eq!( + moved_to, + Some((second, String::new())), + "the ladder moves one candidate at a time, to the front of the next" + ); + } + + /// R11: a four-candidate CID must be served inside the client's resume budget. + /// + /// `gl ipfs get` stops after `MAX_SCAN_RESUMES` resumes (see + /// `crates/gl/src/ipfs_cmd.rs`; it is private to that crate, so the 8 is repeated + /// here and a change to the cap should bring you to this fixture). Ladder length + /// scales with candidate count, so this is the shape that pins the cost. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_four_candidates_serve_within_the_client_resume_budget(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool.clone()); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 1024; + state.ipfs_max_legacy_probes = 2; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1024, std::time::Duration::from_secs(3600)); + + seed_ladder_filler(&state, &pool, "fourcand", 1, 0).await; + let (holder_id, holder_oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6fourcand", + "holder", + b"four candidates deep\n", + ) + .await; + stamp_scan_order(&pool, &holder_id, 1).await; + let cid = seed_legacy_pin_for_oid(&state, &holder_oid).await; + // Three absent candidates, all sorting ahead of the holder's oid, so the holder + // is reachable only through the LAST of the four. + for oid in ["00", "11", "22"] { + state + .db + .record_pinned_cid(&oid.repeat(32), &cid, None) + .await + .unwrap(); + } + let candidates = state.db.oids_for_cid(&cid).await.unwrap(); + assert_eq!(candidates.len(), 4, "precondition: four candidates"); + assert_eq!( + candidates[3], holder_oid, + "precondition: the holder's oid sorts last" + ); + + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.174:5000".parse().unwrap(); + + // One initial request plus at most MAX_SCAN_RESUMES echoes, exactly as the client + // drives it. + let mut token: Option = None; + let mut served_at = None; + for step in 1..=9 { + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), token.as_deref())) + .await + .unwrap(), + ) + .await; + if status == StatusCode::OK { + served_at = Some(step); + break; + } + token = Some( + continuation_of(&body) + .unwrap_or_else(|| panic!("rung {step} must carry a continuation: {body}")), + ); + } + assert!( + served_at.is_some_and(|s| s <= 9), + "a four-candidate CID must be served inside the client's 8-resume budget, \ + got {served_at:?}" + ); + } + + /// A resumed candidate that owes NO scan still advances the ladder. + /// + /// Finished means covered, and a candidate whose recorded provenance is complete is + /// covered without a single row being walked. Gating the advance on the row loop + /// having wrapped leaves that candidate permanently unfinished: the rung sheds with + /// no token and every candidate behind it is never examined, which is the starvation + /// bug in a third shape. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_resumed_candidate_owing_no_scan_still_advances(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool.clone()); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 1024; + state.ipfs_max_legacy_probes = 1; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1024, std::time::Duration::from_secs(3600)); + + seed_ladder_filler(&state, &pool, "noscan", 1, 0).await; + let (holder_id, holder_oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6noscan", + "holder", + b"behind a candidate that owes no scan\n", + ) + .await; + stamp_scan_order(&pool, &holder_id, 1).await; + let cid = seed_legacy_pin_for_oid(&state, &holder_oid).await; + + // The first candidate has a COMPLETE recorded source that denies, so its + // provenance phase answers for it and `needs_scan` is false: no row loop runs and + // it can never wrap. + let absent_first = "00".repeat(32); + state + .db + .record_pinned_cid(&absent_first, &cid, None) + .await + .unwrap(); + seed_root_denying_repos(&state, "noscansrc", 1, 0).await; + state + .db + .record_pin_source(&absent_first, "noscansrc-0000") + .await + .unwrap(); + + let key = state.ipfs_scan_token_key.clone(); + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.175:5000".parse().unwrap(); + + let mut token = Some(minted(&key, &cid, &absent_first, ("", ""))); + let mut served_at = None; + for step in 1..=6 { + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), token.as_deref())) + .await + .unwrap(), + ) + .await; + if status == StatusCode::OK { + served_at = Some(step); + break; + } + token = Some(continuation_of(&body).unwrap_or_else(|| { + panic!( + "rung {step} shed with no continuation: a candidate that owes no scan \ + is finished, and finished must hand the ladder on: {body}" + ) + })); + } + assert!( + served_at.is_some(), + "the ladder must reach the holder behind the no-scan candidate" + ); + } + + /// Resuming the FINAL candidate at the front sentinel keeps the retryable shed. + /// + /// Under the sentinel the row walk really does start at the front, so it is tempting + /// to treat the request as front-started. It is not: this rung SKIPPED every candidate + /// before the sealed one, so absence has not been proven within it and the definitive + /// 404 is not available. + #[sqlx::test] + async fn get_by_cid_front_sentinel_resume_keeps_the_retryable_shed(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 1024; + state.ipfs_max_legacy_probes = 1024; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1024, std::time::Duration::from_secs(3600)); + + seed_root_denying_repos(&state, "sentinelshed", 2, 0).await; + let first = "00".repeat(32); + let second = "11".repeat(32); + let cid = seed_legacy_pin(&state, &first).await; + state + .db + .record_pinned_cid(&second, &cid, None) + .await + .unwrap(); + + let key = state.ipfs_scan_token_key.clone(); + let token = minted(&key, &cid, &second, ("", "")); + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.176:5000".parse().unwrap(); + + let (status, body) = status_and_body( + router + .oneshot(get_cid_scan(&cid, Some(peer), Some(&token))) + .await + .unwrap(), + ) + .await; + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "the candidates before the sealed one were skipped this request, so their \ + absence is unproven and the 404 is not available: {body}" + ); + assert!( + body["message"] + .as_str() + .is_some_and(|m| m.contains("scan-wrapped")), + "{body}" + ); + assert_eq!( + continuation_of(&body), + None, + "the last candidate reached the end of the table, so the ladder is over: {body}" + ); + } + + /// A rung that advanced nothing must not hand the caller back the token they sent. + /// + /// `walk.visits` is charged by the provenance phase as well as by the scan, so a CID + /// whose recorded sources spend the whole visit budget reaches the scan's top-of-loop + /// visit arm before a single page has been fetched. On a RESUMED request `pager.cursor` + /// is still the caller's own incoming position at that moment, so sealing it emits + /// their own token back verbatim. `gl` echoes a token up to `MAX_SCAN_RESUMES` times + /// inside its deadline, and every one of those rungs re-runs the whole provenance phase + /// (up to `MAX_PIN_SOURCES` acquires and `cat-file` subprocesses) to arrive at the same + /// place, so one anonymous request becomes nine and the token makes the spin look like + /// progress. + /// + /// The STATUS is asserted, not just the missing token. The documented precedence is + /// truncation 503 over throttle 429 over the definitive 404, and a dropped seal must + /// leave the truncation tail standing rather than fall through to either lower one. + /// + /// MUTATION (RED): drop the strictly-ahead filter at the mint site, and the shed + /// carries a continuation that opens to the identical position that was sent. + #[sqlx::test] + async fn get_by_cid_visit_starved_resume_does_not_echo_the_callers_own_token( + pool: sqlx::PgPool, + ) { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 1024; + // Probes must NOT bind: the visit budget, spent before the scan starts, is what + // stops this request. + state.ipfs_max_legacy_probes = 1024; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1024, std::time::Duration::from_secs(3600)); + let mut cfg = (*state.config).clone(); + cfg.ipfs_max_repo_visits = 2; + state.config = std::sync::Arc::new(cfg); + + // Four root-readable rows. The first two double as the candidate's recorded pin + // sources: each passes the root gate, so each is charged a visit, and the pair + // spends the ceiling before the scan fetches its first page. + seed_ladder_filler(&state, &pool, "visitstarve", 4, 0).await; + let oid = absent_oid(); + let cid = seed_legacy_pin(&state, &oid).await; + for i in 0..2 { + state + .db + .record_pin_source(&oid, &format!("z6readablevisitstarve/visitstarve-{i:04}")) + .await + .unwrap(); + } + // A non-empty source set only reaches the scan when it may be INCOMPLETE, and the + // scan is what this rung has to be starved out of. + state + .db + .mark_pin_sources_incomplete(&oid, "") + .await + .unwrap(); + + let key = state.ipfs_scan_token_key.clone(); + let start = ( + scan_order_stamp(0).to_rfc3339(), + "z6readablevisitstarve/visitstarve-0000".to_string(), + ); + let token = minted(&key, &cid, &oid, (start.0.as_str(), start.1.as_str())); + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.177:5000".parse().unwrap(); + + let (status, body) = status_and_body( + router + .oneshot(get_cid_scan(&cid, Some(peer), Some(&token))) + .await + .unwrap(), + ) + .await; + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "the search was cut short, so the truncation 503 stands; dropping the seal \ + must not let the tail fall through to the throttle or the definitive 404: \ + {body}" + ); + // Rendered as the position rather than the opaque token so a failure NAMES the + // defect: the echoed triple is byte for byte the one the request carried in. + let echoed = continuation_of(&body).map(|t| { + let pos = opened(&key, &cid, &t); + (pos.sha256_hex, pos.created_at_key, pos.id) + }); + assert_eq!( + echoed, None, + "this rung reached no row the caller had not already been given, so it owes \ + no continuation; echoing {start:?} back under {oid} spins the ladder for \ + another eight amplified requests and calls it progress" + ); + } + + /// The filter drops a seal that stood still, never one that moved. + /// + /// Two properties in one fixture, because they are the two halves of "does not + /// over-drop". Rung 1 is FRONT-STARTED, where the request's start is before every row, + /// so its seal must pass the filter untouched; rung 2 resumes from it, walks two more + /// rows, and its seal must pass because it is strictly ahead. + /// + /// MUTATION (RED): compare the proposal against the start with `>=` instead of `>` + /// and rung 2 keeps its token, so this stays green; compare with `<` and both rungs + /// lose theirs. The filter's job is the middle case, and this fixture is what keeps + /// it from swallowing the other two. + #[sqlx::test] + async fn get_by_cid_a_rung_that_advances_a_row_still_mints(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 1024; + state.ipfs_max_legacy_probes = 2; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1024, std::time::Duration::from_secs(3600)); + + seed_ladder_filler(&state, &pool, "advances", 4, 0).await; + let oid = absent_oid(); + let cid = seed_legacy_pin(&state, &oid).await; + + let key = state.ipfs_scan_token_key.clone(); + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.178:5000".parse().unwrap(); + + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), None)) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "{body}"); + let rung1 = continuation_of(&body).expect( + "a front-started request starts before every row, so its probe-ceiling seal \ + is strictly ahead by construction and must still mint", + ); + let first = opened(&key, &cid, &rung1); + + let (status, body) = status_and_body( + router + .oneshot(get_cid_scan(&cid, Some(peer), Some(&rung1))) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "{body}"); + let rung2 = continuation_of(&body) + .expect("the resumed rung walked two more rows, so it has somewhere to seal"); + let second = opened(&key, &cid, &rung2); + assert_eq!( + (second.sha256_hex.as_str(), first.sha256_hex.as_str()), + (oid.as_str(), oid.as_str()), + "one candidate, so both rungs name it" + ); + assert!( + (second.created_at_key.clone(), second.id.clone()) + > (first.created_at_key.clone(), first.id.clone()), + "a rung that reached rows the caller had not seen must seal one of them: \ + {first:?} then {second:?}" + ); + } + + /// The advance to the next candidate is not "backwards", and the filter must know it. + /// + /// The advance seals the front-of-table sentinel, an empty row pair that sorts BELOW + /// every real key. A filter that compared only the row would read the ladder's one real + /// forward step as a step back and drop it, ending every multi-candidate ladder at the + /// rung that was about to hand over. What makes it forward is the candidate: a + /// different hex can only come from the finished-candidate advance, which is ahead by + /// construction. + /// + /// MUTATION (RED): compare rows without first comparing the candidate, and the + /// handover token disappears. + #[sqlx::test] + async fn get_by_cid_the_advance_to_the_next_candidate_survives_the_filter(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 1024; + state.ipfs_max_legacy_probes = 1024; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1024, std::time::Duration::from_secs(3600)); + + seed_root_denying_repos(&state, "advfilter", 2, 0).await; + let first = "00".repeat(32); + let second = "11".repeat(32); + let cid = seed_legacy_pin(&state, &first).await; + state + .db + .record_pinned_cid(&second, &cid, None) + .await + .unwrap(); + + let key = state.ipfs_scan_token_key.clone(); + // Resumed at the LAST row of the table, so the first candidate's keyset fetch comes + // back empty, it wraps, and the rung's whole job is the handover. + let token = minted( + &key, + &cid, + &first, + (&scan_order_stamp(1).to_rfc3339(), "advfilter-0001"), + ); + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.179:5000".parse().unwrap(); + + let (status, body) = status_and_body( + router + .oneshot(get_cid_scan(&cid, Some(peer), Some(&token))) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "{body}"); + let handover = continuation_of(&body).expect( + "the finished candidate hands the ladder on, and the sentinel it seals is \ + ahead by candidate even though the row pair sorts below the start", + ); + let pos = opened(&key, &cid, &handover); + assert_eq!( + ( + pos.sha256_hex.as_str(), + pos.created_at_key.as_str(), + pos.id.as_str() + ), + (second.as_str(), "", ""), + "the next candidate, at the front of the table" + ); + } + + /// A ceiling that stops mid-page seals the last row it SETTLED, which is progress. + /// + /// This is the arm the tokenless shed must not swallow. A resumed scan only ever walks + /// rows past its start cursor, so any row it settled is strictly ahead, and the rung + /// that settles two rows before a ceiling refuses the third owes the caller the second + /// one. Only a rung that settled NOTHING sheds tokenless, because there the spender is + /// the provenance phase, which runs identically on every retry. + /// + /// MUTATION (RED): seal the request's start instead of the settled row, and the filter + /// (correctly) drops it, so this ladder loses its token and stalls. + #[sqlx::test] + async fn get_by_cid_resumed_ceiling_seals_the_last_row_it_settled(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // A page wide enough that the probe ceiling binds INSIDE the row loop rather than + // at the top of it: that is the arm whose position is the last settled row. + state.ipfs_legacy_scan_page_rows = 4; + state.ipfs_max_legacy_scan_rows = 1024; + state.ipfs_max_legacy_probes = 2; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1024, std::time::Duration::from_secs(3600)); + + seed_ladder_filler(&state, &pool, "settled", 5, 0).await; + let oid = absent_oid(); + let cid = seed_legacy_pin(&state, &oid).await; + + let key = state.ipfs_scan_token_key.clone(); + let start = ( + scan_order_stamp(0).to_rfc3339(), + "z6readablesettled/settled-0000".to_string(), + ); + let token = minted(&key, &cid, &oid, (start.0.as_str(), start.1.as_str())); + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.180:5000".parse().unwrap(); + + let (status, body) = status_and_body( + router + .oneshot(get_cid_scan(&cid, Some(peer), Some(&token))) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "{body}"); + let pos = opened( + &key, + &cid, + &continuation_of(&body).expect( + "the rung settled two rows before the ceiling refused the third, so it \ + has real progress to seal", + ), + ); + assert_eq!( + (pos.created_at_key.as_str(), pos.id.as_str()), + ( + scan_order_stamp(2).to_rfc3339().as_str(), + "z6readablesettled/settled-0002" + ), + "the seal is the last row the ceiling let this rung settle, not the row it \ + refused and not the caller's own start" + ); + assert!( + (pos.created_at_key.clone(), pos.id.clone()) > start, + "and it is strictly ahead of the position the request came in with" + ); + } + + /// The VISIT ceiling must advance the ladder too, for the same reason as the probe + /// ceiling: it is the sibling arm, it fires on the same root-readable inventory, and + /// a tokenless shed there strands everything behind it just as permanently. + /// + /// MUTATION (RED): drop the continuation from the visit-ceiling arm. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_visit_ceiling_ladders_to_a_holder_past_it(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 1024; + // Probes must NOT bind: the visit ceiling is the one under test. + state.ipfs_max_legacy_probes = 1024; + let mut cfg = (*state.config).clone(); + cfg.ipfs_max_repo_visits = 2; + state.config = std::sync::Arc::new(cfg); + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1024, std::time::Duration::from_secs(3600)); + + seed_root_readable_repos(&state, "visit", 6).await; + let (_, oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6visit", + "holder", + b"past the visits\n", + ) + .await; + let cid = seed_legacy_pin_for_oid(&state, &oid).await; + + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.153:5000".parse().unwrap(); + let bound = 6usize.div_ceil(2) + 1; + + let mut token: Option = None; + let mut served_at = None; + for step in 1..=bound { + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), token.as_deref())) + .await + .unwrap(), + ) + .await; + if status == StatusCode::OK { + served_at = Some(step); + break; + } + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "an intermediate rung is the retryable 503 (step {step}): {body}" + ); + token = Some(continuation_of(&body).unwrap_or_else(|| { + panic!( + "the visit-ceiling shed at step {step} must carry a continuation: \ + {body}" + ) + })); + } + assert!( + served_at.is_some(), + "a holder past the visit ceiling must be reached within {bound} \ + token-echoing requests, not stranded forever" + ); + } + + /// Scenario 6: the page toll accumulates ACROSS requests. + /// + /// Every page the scan buys is charged to the caller's per-IP work bucket, so a + /// denial-only inventory cannot be re-paged for free by re-requesting. A bucket + /// sized to 4 pages admits four requests' worth of paging and then sheds the fifth + /// with 429, buying NO page (the `preload_queries()` count stalls) and carrying NO + /// token. The caller's PREVIOUS token still resumes them once the bucket refills. + /// + /// MUTATION D (RED): drop the page toll and the pages are free again, so the fifth + /// request buys its page and never 429s. + #[sqlx::test] + async fn get_by_cid_denial_only_requests_throttle_across_requests(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 2; + // Four pages of allowance: above the derived floor's page term for this fixture + // and still small enough that a handful of requests exhausts it. + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(4, std::time::Duration::from_secs(3600)); + seed_root_denying_repos(&state, "toll", 20, 0).await; + let cid = seed_legacy_pin(&state, &absent_oid()).await; + + let router = ipfs_router(state.clone()); + let peer: SocketAddr = "203.0.113.145:5000".parse().unwrap(); + + crate::api::ipfs::reset_preload_queries(); + let mut last_token = None; + for step in 1..=4 { + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), None)) + .await + .unwrap(), + ) + .await; + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "request {step} is within the bucket and must buy its page: {body}" + ); + last_token = continuation_of(&body); + } + let last_token = last_token.expect("a tolled-but-admitted request still emits a token"); + let pages_before = crate::api::ipfs::preload_queries(); + + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), None)) + .await + .unwrap(), + ) + .await; + assert_eq!( + status, + StatusCode::TOO_MANY_REQUESTS, + "a spent work bucket must brake the next denial-only request with 429 \ + rather than sell it another page: {body}" + ); + assert_eq!( + crate::api::ipfs::preload_queries(), + pages_before, + "and the braked request must buy NO page; a 429 that still paged would \ + leave the amplification exactly where it was" + ); + assert!( + continuation_of(&body).is_none(), + "the 429 carries no token: the caller's own bucket, not the node's search, \ + stopped them, and their previous token is still valid: {body}" + ); + + // Bucket refilled (a fresh limiter is the window elapsing). The token the caller + // already holds still resumes them: the throttle cost them a page, not their + // place in the ladder. + let mut refilled = state.clone(); + refilled.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(4, std::time::Duration::from_secs(3600)); + crate::api::ipfs::reset_scan_rows(); + let (status, body) = status_and_body( + ipfs_router(refilled) + .oneshot(get_cid_scan(&cid, Some(peer), Some(&last_token))) + .await + .unwrap(), + ) + .await; + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "the previously issued token must still resume after a refill: {body}" + ); + assert_eq!( + crate::api::ipfs::scan_rows(), + 2, + "and it must resume at the sealed position, one ceiling's worth of rows \ + read, not a restart at the front" + ); + } + + /// Scenario 7: the RULES ceiling. The row ceiling bounds the row count but not the + /// memory each row drags in: the pager retains every fetched page's rules for the + /// whole request. A window of rule-heavy repos must taint at the rules ceiling with + /// the row count still well under the row ceiling, on the same 503-with-token + /// contract. + #[sqlx::test] + async fn get_by_cid_rules_ceiling_stops_scan_with_token(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + // Rows are NOT the binding ceiling here. + state.ipfs_max_legacy_scan_rows = 1000; + state.ipfs_max_legacy_scan_rule_bytes = 3; + seed_root_denying_repos(&state, "rules", 8, 2).await; + let cid = seed_legacy_pin(&state, &absent_oid()).await; + + let peer: SocketAddr = "203.0.113.146:5000".parse().unwrap(); + crate::api::ipfs::reset_scan_rows(); + let (status, body) = status_and_body( + ipfs_router(state) + .oneshot(get_cid_scan(&cid, Some(peer), None)) + .await + .unwrap(), + ) + .await; + + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "a rules-ceiling truncation sheds the same retryable 503: {body}" + ); + assert!( + body["message"] + .as_str() + .unwrap_or_default() + .contains("rules-ceiling"), + "the shed must name the rules ceiling so an operator can tell it from a row \ + truncation: {body}" + ); + let rows = crate::api::ipfs::scan_rows(); + assert!( + rows < 1000, + "the rules ceiling must fire with rows still under the row ceiling, or it is \ + not the guard being exercised; read {rows} rows" + ); + assert!( + continuation_of(&body).is_some(), + "a rules truncation carries a continuation too: {body}" + ); + } + + /// A SINGLE page whose rules exceed the ceiling truncates the request that bought + /// it. + /// + /// The ceiling bounds retained MEMORY, and the thing it has to bound is bytes: there + /// is no per-repo cap on `visibility_rules`, and an owner controls both how many + /// rules their repos carry and how long each `reader_dids` list is. Counted in rules + /// and checked only between pages, one page could carry arbitrarily many bytes and + /// the guard would not notice until it was asked for the NEXT page, which on a scan + /// that ends there is never. + /// + /// The fixture is calibrated so page one alone clears the byte ceiling while its + /// four rules are far under any plausible rule COUNT, which is what makes the unit + /// the thing under test. + /// + /// MUTATION (RED): move the check back between pages and the request that bought the + /// oversized page runs on to a clean 404. + #[sqlx::test] + async fn get_by_cid_one_oversized_page_truncates_its_own_request(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + // Neither rows nor probes may bind: this is the rule-bytes guard alone. + state.ipfs_max_legacy_scan_rows = 1000; + // Under the byte ceiling one page (2 rows x 2 rules, each rule carrying its repo + // id, its glob and a reader DID) is already over. Under a RULE count of 200 that + // same page is four. + state.ipfs_max_legacy_scan_rule_bytes = 200; + seed_root_denying_repos(&state, "bytes", 6, 2).await; + let cid = seed_legacy_pin(&state, &absent_oid()).await; + + let peer: SocketAddr = "203.0.113.152:5000".parse().unwrap(); + crate::api::ipfs::reset_scan_rows(); + let (status, body) = status_and_body( + ipfs_router(state) + .oneshot(get_cid_scan(&cid, Some(peer), None)) + .await + .unwrap(), + ) + .await; + + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "the page that blew the retained-byte ceiling must truncate its OWN request, \ + not run on to a 404: {body}" + ); + assert!( + body["message"] + .as_str() + .unwrap_or_default() + .contains("rules-ceiling"), + "the shed names the rule-bytes ceiling: {body}" + ); + assert_eq!( + crate::api::ipfs::scan_rows(), + 2, + "and it stops on the page that bought the bytes, not a page later" + ); + assert!( + continuation_of(&body).is_some(), + "a rule-bytes truncation carries a continuation like every other ceiling: \ + {body}" + ); + } + + /// The rule-bytes ceiling must be enforced by the QUERY, not by summing the page + /// after it has been transferred and allocated. + /// + /// A repo owner controls how many rules their repos carry and how long each + /// `reader_dids` list is, so a post-fetch sum truncates the REQUEST while leaving the + /// WORK unbounded: the oversized page is already in memory by the time the guard + /// fires. INV-10 bounds work done, never results measured afterwards, and the caller + /// here is an anonymous `/ipfs/{legacy-cid}` request holding one of the scarce walk + /// permits. + /// + /// The assertion is on the number of rule ROWS the query actually returned, not on + /// the status: the status is identical either way, which is exactly why the old shape + /// looked correct. + /// + /// MUTATION (RED): drop the query bound and sum the rules after the fetch, and the + /// whole page's rules are materialized (16 rows here against a budget that admits + /// one repo's two). + #[sqlx::test] + async fn get_by_cid_rule_bytes_bounded_in_the_query_not_after_the_page(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // One page holds every seeded repo, so nothing but the rule budget can bind. + state.ipfs_legacy_scan_page_rows = 8; + state.ipfs_max_legacy_scan_rows = 1000; + // Under this budget a single repo's pair of rules is already over, so at most one + // repo may be loaded and the page's remaining seven must never leave the database. + state.ipfs_max_legacy_scan_rule_bytes = 200; + seed_root_denying_repos(&state, "querybound", 8, 2).await; + let cid = seed_legacy_pin(&state, &absent_oid()).await; + + let peer: SocketAddr = "203.0.113.171:5000".parse().unwrap(); + crate::api::ipfs::reset_scan_rule_rows(); + let (status, body) = status_and_body( + ipfs_router(state) + .oneshot(get_cid_scan(&cid, Some(peer), None)) + .await + .unwrap(), + ) + .await; + + let rule_rows = crate::api::ipfs::scan_rule_rows(); + assert!( + rule_rows <= 4, + "the byte budget must bound the QUERY: at most one repo's rules may be \ + materialized under a 200-byte budget, but {rule_rows} rule rows were pulled \ + (the whole page is 16)" + ); + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "reaching the query bound is the ceiling condition and sheds the retryable \ + 503, exactly as the post-fetch sum did: {body}" + ); + assert!( + body["message"] + .as_str() + .unwrap_or_default() + .contains("rules-ceiling"), + "the shed still names the rule-bytes ceiling: {body}" + ); + assert!( + continuation_of(&body).is_some(), + "and it still mints a continuation, or the repos behind the cut are \ + unreachable: {body}" + ); + } + + /// The property the old `!exhausted` condition protected, restated for the query + /// bound: a scan that genuinely covered the table must answer 404, never a permanent + /// 503. + /// + /// Under the query bound the taint no longer keys on "the page was short" but on + /// "the query left repos unloaded". A short final page whose rules all fit leaves + /// nothing unloaded, so it stays a complete scan and the absent object is a clean + /// 404. The budget here is finite and set by the fixture, so this is the guard being + /// exercised rather than the 4 MiB default never coming near. + #[sqlx::test] + async fn get_by_cid_short_final_page_under_the_rule_budget_still_404s(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 8; + state.ipfs_max_legacy_scan_rows = 1000; + // Roomy enough for all four repos' rules together, so no cut is possible. + state.ipfs_max_legacy_scan_rule_bytes = 64 * 1024; + seed_root_denying_repos(&state, "shortfit", 4, 1).await; + let cid = seed_legacy_pin(&state, &absent_oid()).await; + + let peer: SocketAddr = "203.0.113.172:5000".parse().unwrap(); + let (status, body) = status_and_body( + ipfs_router(state) + .oneshot(get_cid_scan(&cid, Some(peer), None)) + .await + .unwrap(), + ) + .await; + + assert_eq!( + status, + StatusCode::NOT_FOUND, + "a complete scan of an absent object is a verdict; turning it into a 503 \ + would make the object permanently unresolvable: {body}" + ); + } + + /// The ladder MAKES PROGRESS under the query bound: every rung consumes at least one + /// repo, so the continuation always advances and the scan terminates. + /// + /// This is the failure mode the query bound could have introduced. If a page whose + /// FIRST repo alone exceeds the remaining budget loaded nothing, the cut would sit at + /// the cursor, the next request would reproduce it exactly, and the caller would be + /// wedged on a 503 forever for an object the node could otherwise settle. The bound + /// therefore always admits the first rule-carrying repo of a page whatever its size. + /// + /// The ladder ends on the tokenless shed, which is the design's "your ladder is + /// over" answer for a RESUMED scan (absence was only ever proven over + /// `[token, end)`), not on a 404. + #[sqlx::test] + async fn get_by_cid_rule_bytes_ladder_advances_to_a_tokenless_shed(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 1000; + // Every repo's rules alone clear the budget, so every page is cut at its first + // repo: the worst case for progress. + state.ipfs_max_legacy_scan_rule_bytes = 1; + let repos = 8usize; + seed_root_denying_repos(&state, "ladder", repos, 2).await; + let cid = seed_legacy_pin(&state, &absent_oid()).await; + + let peer: SocketAddr = "203.0.113.173:5000".parse().unwrap(); + let router = ipfs_router(state); + let mut token: Option = None; + let mut rungs = 0usize; + let bound = repos + 2; + loop { + rungs += 1; + assert!( + rungs <= bound, + "the ladder must consume at least one repo per rung and terminate within \ + {bound} rungs; a rung that loaded nothing would repeat forever" + ); + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), token.as_deref())) + .await + .unwrap(), + ) + .await; + if status == StatusCode::NOT_FOUND { + break; + } + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "rung {rungs} must be the retryable 503: {body}" + ); + let next = continuation_of(&body); + if next.is_none() { + break; + } + assert_ne!( + next, token, + "rung {rungs} handed back the SAME continuation it was given, so the scan \ + made no progress and the caller is wedged: {body}" + ); + token = next; + } + } + + /// Scenario 8: interleaved callers stay isolated. Two source keys alternate + /// token-echoing ladders against the same denial-heavy inventory with the holder + /// past the ceiling; each must reach its own 200 within its own bound. + /// + /// Isolation is STRUCTURAL under this design (each ladder's entire state rides in + /// its own tokens and the node holds none), so this is the executed confirmation + /// rather than a mutant target. It is what rules out the rejected designs: a + /// node-global persisted cursor lets these two advance each other's window, and a + /// per-caller server-side map lets one evict the other. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_interleaved_callers_each_reach_their_holder(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 4; + // The production toll, generous enough that neither caller's ladder is braked; + // this scenario is about isolation, not the toll. + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(600, std::time::Duration::from_secs(3600)); + + seed_root_denying_repos(&state, "interleave", 10, 0).await; + let (_, oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6interleave", + "holder", + b"shared holder\n", + ) + .await; + let cid = seed_legacy_pin_for_oid(&state, &oid).await; + + let router = ipfs_router(state); + let peers: [SocketAddr; 2] = [ + "203.0.113.147:5000".parse().unwrap(), + "203.0.113.148:5000".parse().unwrap(), + ]; + let bound = 10usize.div_ceil(4) + 1; + let mut tokens: [Option; 2] = [None, None]; + let mut served = [None, None]; + + for step in 1..=bound { + for (i, peer) in peers.iter().enumerate() { + if served[i].is_some() { + continue; + } + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(*peer), tokens[i].as_deref())) + .await + .unwrap(), + ) + .await; + if status == StatusCode::OK { + served[i] = Some(step); + continue; + } + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "caller {i} rung {step} must be the retryable 503: {body}" + ); + tokens[i] = Some( + continuation_of(&body) + .unwrap_or_else(|| panic!("caller {i} rung {step} needs a token: {body}")), + ); + } + } + + assert!( + served[0].is_some() && served[1].is_some(), + "both interleaved callers must reach the holder within their own bound of \ + {bound}; got {served:?}. A shared server-side cursor would let one caller's \ + progress skip the other's coverage" + ); + } + + /// Seed `n` PRIVATE repos whose id, owner DID, and `created_at` are all + /// high-entropy MARKERS, so a substring search over an emitted token is a real + /// test. Returns the markers in scan order. + async fn seed_marked_withheld_repos( + state: &crate::state::AppState, + n: usize, + ) -> Vec<(String, String, String)> { + const OWNER: &str = "did:key:z6MkWithheldOwnerMarkerQQQQQQQQQQQQQQQQ"; + let mut out = Vec::new(); + for i in 0..n { + let at = scan_order_stamp(i); + let id = format!("marker-repo-XZXZ{i:04}"); + // Every other row is a quarantined mirror instead of a private repo, so + // both withholding classes sit in the window the token is minted from. + if i % 2 == 1 { + state + .db + .upsert_mirror_repo(OWNER, &id, &format!("/nonexistent/{id}"), None, true) + .await + .expect("seed a quarantined marker row"); + // `upsert_mirror_repo` stamps `now` and derives its own id, so re-read + // the row the scan will actually see. + let rec = state + .db + .get_repo(OWNER, &id) + .await + .unwrap() + .expect("the quarantined marker row exists"); + out.push((rec.id, OWNER.to_string(), rec.created_at.to_rfc3339())); + continue; + } + state + .db + .create_repo(&crate::db::RepoRecord { + id: id.clone(), + name: id.clone(), + owner_did: OWNER.to_string(), + description: None, + is_public: false, + default_branch: "main".to_string(), + created_at: at, + updated_at: at, + disk_path: format!("/nonexistent/{id}"), + forked_from: None, + machine_id: None, + }) + .await + .expect("seed a private marker row"); + out.push((id, OWNER.to_string(), at.to_rfc3339())); + } + out + } + + /// Scenario 9, the INV-13 guard: the emitted continuation leaks no withheld field. + /// + /// A denial-only scan fetches nothing BUT withheld rows, so the row its token seals + /// is by construction a private or quarantined repo the caller may not read. Its + /// `created_at` leaks a hidden repo's creation time and its `id` carries the owner's + /// DID. Base64 is transport, not confidentiality (this is the exact shape #134 + /// shipped and INV-13 records), so the token must be AEAD-SEALED. + /// + /// The fixture is arranged so the row at the truncation boundary (the row the token + /// seals) IS one of the poisoned withheld repos. Stated because it is load-bearing: + /// a future edit seeding a READABLE repo at the boundary would leave mutation E + /// green and this guard would silently stop proving anything. + /// + /// The last assertion is the one the substring checks structurally cannot make. + /// AEAD ciphertext is plaintext-length plus the tag, and both halves of a scan + /// position vary in length, so without fixed-width padding the token LENGTH is a + /// side channel for the sealed row. + /// + /// MUTATION E (RED): seal by base64-of-plaintext and the markers decode straight out. + /// MUTATION G (RED): drop the fixed-width padding and the two lengths diverge. + /// + /// Like the two token guards below it, this has no pre-fix RED: its assertions call + /// `seal_scan_token` / `open_scan_token`, which do not exist on the pre-fix head, so + /// the only failure available there is a compile error. Mutations E and G are its + /// REDs, and each injects precisely the encoding INV-13 forbids rather than merely + /// removing the code, which is the stronger observation. + #[sqlx::test] + async fn scan_token_leaks_no_withheld_fields(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 2; + let markers = seed_marked_withheld_repos(&state, 6).await; + let cid = seed_legacy_pin(&state, &absent_oid()).await; + let key = state.ipfs_scan_token_key.clone(); + + let peer: SocketAddr = "203.0.113.150:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid_scan(&cid, Some(peer), None)) + .await + .unwrap(); + let raw_body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .expect("read body"); + let body_text = String::from_utf8_lossy(&raw_body).to_string(); + let body: serde_json::Value = serde_json::from_slice(&raw_body).expect("json body"); + let token = continuation_of(&body).expect("the truncation must emit a token"); + + // Fixture precondition, on the FIXTURE rather than on the token: every seeded + // row is withheld (private or quarantined) and the scan stopped after exactly + // one ceiling's worth, so the row the token seals is a withheld row. Stated + // without opening the token so the leak assertions below are what fires when the + // seal is replaced by an encoding, rather than a precondition panic. + assert_eq!( + crate::api::ipfs::scan_rows(), + 2, + "the truncation boundary must sit inside the seeded withheld window" + ); + + let decoded = base64::Engine::decode( + &base64::engine::general_purpose::URL_SAFE_NO_PAD, + token.as_bytes(), + ) + .expect("the token is base64url"); + let decoded_text = String::from_utf8_lossy(&decoded).to_string(); + + for (id, owner, created) in &markers { + for (what, marker) in [ + ("repo id", id), + ("owner did", owner), + ("created_at", created), + ] { + assert!( + !body_text.contains(marker.as_str()), + "the response body must not carry a withheld repo's {what} ({marker}): \ + {body_text}" + ); + assert!( + !decoded_text.contains(marker.as_str()), + "the token's DECODED bytes must not carry a withheld repo's {what} \ + ({marker}); base64 is transport, not confidentiality (INV-13)" + ); + assert!( + decoded + .windows(marker.len()) + .all(|w| w != marker.as_bytes()), + "the token's raw bytes must not carry a withheld repo's {what} ({marker})" + ); + } + } + + // And the row it actually seals IS one of the poisoned withheld rows, checked + // after the leak assertions so a broken seal is reported as a leak, not as a + // fixture failure. Load-bearing: seeding a READABLE repo at the boundary would + // leave mutation E green and this whole guard would stop proving anything. + let sealed = gitlawb_core::scan_token::open_scan_token( + &key, + &cid, + &token, + chrono::Utc::now().timestamp(), + ) + .expect("the node's own key opens its own token"); + assert!( + markers + .iter() + .any(|(id, _, created)| *id == sealed.id && *created == sealed.created_at_key), + "the row at the truncation boundary must be one of the poisoned withheld \ + repos; sealed {sealed:?}" + ); + + // A different key must not open it: the seal, not an encoding, is what withholds. + let other = gitlawb_core::scan_token::new_key(); + assert!( + gitlawb_core::scan_token::open_scan_token( + &other, + &cid, + &token, + chrono::Utc::now().timestamp() + ) + .is_none(), + "a token that opens under any key but the node's own is not sealed" + ); + + // Token LENGTH must not vary with the sealed row. + let now = chrono::Utc::now().timestamp(); + let short = gitlawb_core::scan_token::seal_scan_token( + &key, + &cid, + &gitlawb_core::scan_token::ScanPosition { + created_at_key: "2020-01-01T00:00:00+00:00".into(), + id: "a/b".into(), + sha256_hex: absent_oid(), + }, + now + 60, + ) + .unwrap(); + let long = gitlawb_core::scan_token::seal_scan_token( + &key, + &cid, + &gitlawb_core::scan_token::ScanPosition { + created_at_key: "2020-01-01T00:00:00+00:00".into(), + id: format!("did:key:z6MkAVeryLongOwnerKeyIdentifier/{}", "n".repeat(48)), + sha256_hex: absent_oid(), + }, + now + 60, + ) + .unwrap(); + assert_eq!( + short.len(), + long.len(), + "tokens sealing rows of very different id lengths must be byte-identical in \ + length, or the length is a side channel for the withheld row, which the \ + substring assertions above structurally cannot see" + ); + } + + /// Scenario 10: tampered, foreign-CID, and expired tokens are ABSENT, uniformly. + /// + /// Each of the three failure classes must produce exactly the front-started response + /// a tokenless request gets: same status, same body shape, and (the decisive part) + /// an emitted continuation sealing the FRONT window's last row, not the row the + /// rejected token named. Never an error, never a resumed position, and no way to + /// tell the three classes apart. + /// + /// The "front-started" half is asserted by opening the EMITTED token and checking + /// which row it seals, which looks over-elaborate until you try the obvious thing. + /// `scan_rows()` cannot separate the two states: a front start reads rows 1-2 and a + /// resume from the rejected position reads rows 3-4, so the counter says 2 either + /// way. The sealed position is the only thing that differs, and without checking it + /// the foreign-CID leg passes under mutation F. + /// + /// No pre-fix RED, for the same reason as the guard above: the probes are minted + /// with `seal_scan_token`, which does not exist pre-fix. Mutation F is its RED. + /// + /// MUTATION F (RED): drop the CID from the associated data and the foreign-CID leg's + /// token is honoured, so the scan resumes at the foreign position. + #[sqlx::test] + async fn scan_token_invalid_variants_start_at_front(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 2; + seed_root_denying_repos(&state, "front", 6, 0).await; + let cid = seed_legacy_pin(&state, &absent_oid()).await; + let key = state.ipfs_scan_token_key.clone(); + let now = chrono::Utc::now().timestamp(); + + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.151:5000".parse().unwrap(); + + // Baseline: a tokenless request reads the front window and seals row 2. + let (base_status, base_body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), None)) + .await + .unwrap(), + ) + .await; + let base_token = continuation_of(&base_body).expect("baseline emits a token"); + let front = gitlawb_core::scan_token::open_scan_token(&key, &cid, &base_token, now) + .expect("baseline token opens"); + assert_eq!( + front.id, "front-0001", + "fixture precondition: the front window ends at the second seeded row" + ); + + // Probe 1: a byte-flipped token. + let mut bytes: Vec = base_token.bytes().collect(); + let last = bytes.len() - 1; + bytes[last] = if bytes[last] == b'A' { b'B' } else { b'A' }; + let tampered = String::from_utf8(bytes).unwrap(); + + // Probe 2: a well-formed token minted for a DIFFERENT CID, at a position deep + // in the table so honouring it would be unmistakable. + let elsewhere = cid_for_oid(&"f4".repeat(32)); + let foreign = gitlawb_core::scan_token::seal_scan_token( + &key, + &elsewhere, + &gitlawb_core::scan_token::ScanPosition { + created_at_key: scan_order_stamp(3).to_rfc3339(), + id: "front-0003".into(), + sha256_hex: absent_oid(), + }, + now + 3600, + ) + .unwrap(); + + // Probe 3: a token for this CID whose expiry is already past. + let expired = gitlawb_core::scan_token::seal_scan_token( + &key, + &cid, + &gitlawb_core::scan_token::ScanPosition { + created_at_key: scan_order_stamp(3).to_rfc3339(), + id: "front-0003".into(), + sha256_hex: absent_oid(), + }, + now - 1, + ) + .unwrap(); + + for (what, probe) in [ + ("tampered", tampered), + ("foreign-CID", foreign), + ("expired", expired), + ] { + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), Some(&probe))) + .await + .unwrap(), + ) + .await; + assert_eq!( + status, base_status, + "a {what} token must answer exactly as a tokenless request does, never \ + an error and never a distinguishable status: {body}" + ); + assert_eq!( + body["error"], base_body["error"], + "a {what} token must not change the body shape: {body}" + ); + assert_eq!( + body["message"], base_body["message"], + "a {what} token must not change the message: {body}" + ); + let token = continuation_of(&body) + .unwrap_or_else(|| panic!("the {what} probe answers like a front start: {body}")); + let pos = gitlawb_core::scan_token::open_scan_token(&key, &cid, &token, now) + .expect("the emitted token opens"); + assert_eq!( + pos.id, front.id, + "a {what} token must be treated as ABSENT and the scan must start at the \ + FRONT; resuming from it would honour a position the caller was never \ + handed for this CID" + ); + } + } + + /// Scenario 11: every seal draws a FRESH nonce. + /// + /// This is the property the whole confidentiality claim rests on and the one the + /// other two token guards cannot see: both of them pass unchanged under a constant + /// nonce. Under a stream cipher a repeated nonce repeats the keystream, so two + /// tokens sealed under one nonce XOR to the difference of their plaintexts, and an + /// attacker who can force the node to seal a position they know then recovers a + /// withheld row's fields in full. That is strictly worse than the base64 defect + /// INV-13 records. + /// + /// No pre-fix RED, like the two guards above: it seals through an API that does not + /// exist on the pre-fix head. Mutation H is its RED, and H reddens nothing else, + /// which is the same fact stated from the other side. + /// + /// MUTATION H (RED): fix the nonce to a constant and the two tokens are identical. + #[test] + fn scan_token_seals_are_nonce_fresh() { + let key = gitlawb_core::scan_token::new_key(); + let pos = gitlawb_core::scan_token::ScanPosition { + created_at_key: "2020-01-01T00:00:07+00:00".into(), + id: "did:key:z6MkHiddenOwner/withheld-repo".into(), + sha256_hex: "f2".repeat(32), + }; + let expires = chrono::Utc::now().timestamp() + 3600; + let first = + gitlawb_core::scan_token::seal_scan_token(&key, "bafkcid", &pos, expires).unwrap(); + let second = + gitlawb_core::scan_token::seal_scan_token(&key, "bafkcid", &pos, expires).unwrap(); + + assert_ne!( + first, second, + "sealing the same position twice must produce different bytes; identical \ + tokens mean a reused nonce, and a reused nonce under a stream cipher leaks \ + the withheld plaintext to anyone holding two tokens" + ); + let now = chrono::Utc::now().timestamp(); + for token in [&first, &second] { + let opened = gitlawb_core::scan_token::open_scan_token(&key, "bafkcid", token, now) + .expect("both tokens must still open"); + assert_eq!( + opened, pos, + "nonce freshness must not cost correctness: both seals open to the same \ + position" + ); + } + } + + /// Scenario 12: the degenerate ZERO-ROW resume never 404s. + /// + /// A row count that is an EXACT multiple of the ceiling is the shape whose last + /// emitted token points AT the final row, so the next resume fetches an empty page. + /// The wrap taint is evaluated on `pager.exhausted`, not at any particular break + /// site, so this is covered by construction: an implementation that keys the taint + /// on having fetched a page passes every other scenario here and converts an + /// incomplete search into a false 404 exactly here. + #[sqlx::test] + async fn scan_token_at_table_end_wraps_not_404(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 2; + // 4 rows at ceiling 2: the second rung's token points at the last row. + seed_root_denying_repos(&state, "endstop", 4, 0).await; + let cid = seed_legacy_pin(&state, &absent_oid()).await; + + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.149:5000".parse().unwrap(); + + let mut token: Option = None; + let mut last = None; + for step in 1..=4 { + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), token.as_deref())) + .await + .unwrap(), + ) + .await; + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "no rung of this ladder may 404, least of all the zero-row one \ + (step {step}): {body}" + ); + match continuation_of(&body) { + Some(next) => token = Some(next), + None => { + last = Some(body); + break; + } + } + } + let last = last.expect("the ladder must terminate at the table end"); + assert_eq!(last["error"], "search_incomplete", "{last}"); + assert!( + last["message"] + .as_str() + .unwrap_or_default() + .contains("scan-wrapped"), + "a resume landing on or past the last row fetches an empty page, which sets \ + `exhausted` and must taint scan-wrapped: {last}" + ); + assert!( + continuation_of(&last).is_none(), + "a wrapped scan emits no token: {last}" + ); + } + + /// F3 budget expiry mid-loop: one absolute request budget + /// (`ipfs_request_budget_secs`) bounds the whole admitted scan; per-repo + /// stages may not each draw a fresh timeout past it. Budget 1s, per-iteration + /// acquire timeout 2s; the FIRST-iterated row is a Tigris-backed ghost (no local + /// copy, silent local endpoint) whose acquire stalls, the row behind it is a plain + /// public repo carrying the blob. The ghost's acquire runs clamped to the ~1s + /// remainder and times out; at the next repo the budget gate sees zero + /// remaining, taints "budget", and STOPS the scan, so the blob repo is never + /// visited (a visit would probe the healthy public copy and serve 200, which + /// the 503 assertion rules out) and the shed names the budget. Without the + /// budget the acquire would time out at its own 2s, the scan would continue, + /// and the buried blob would serve 200 (the recorded RED). MUTATION (RED): + /// remove the `request_deadline` capture (or make the remaining budget + /// infinite) and this serves 200 again. + #[sqlx::test] + async fn get_by_cid_request_budget_expiry_stops_scan_with_503(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // Seed through a LOCAL-ONLY store first, so seeding never consults the + // (deliberately unreachable) Tigris endpoint. The ghost row goes in FIRST: the + // paged scan orders on `(created_at, id)` ASC (#173, jatmn), so the row created + // first is the row iterated first. + state.repo_store = + crate::git::repo_store::RepoStore::for_testing(repos_dir.clone(), pool.clone()); + state + .db + .upsert_mirror_repo("z6f3budget", "ghost", "/unused-ghost", None, false) + .await + .unwrap(); + let (_, oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6f3budget", + "buried", + b"budget expiry proof\n", + ) + .await; + // Swap in a Tigris-backed store over the SAME repos_dir (the seeded bare + // repo stays a fast local hit). The ghost has no local copy, so its acquire + // consults the silent local endpoint and stalls past the budget + // (endpoint-pinned test client, no AWS_* env reads). + let endpoint = crate::test_support::silent_http_endpoint().await; + let tigris = + crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint) + .await; + state.repo_store = crate::git::repo_store::RepoStore::new(repos_dir, Some(tigris), pool); + let mut cfg = (*state.config).clone(); + cfg.ipfs_request_budget_secs = 1; + cfg.git_acquire_timeout_secs = 2; + state.config = Arc::new(cfg); + + let peer: SocketAddr = "203.0.113.70:5000".parse().unwrap(); + let cid = seed_legacy_pin_for_oid(&state, &oid).await; + let resp = ipfs_router(state) + .oneshot(get_cid(&cid, Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted request budget must stop the scan with a retryable 503; \ + scanning on into the later public blob repo would have served 200" + ); + assert_eq!( + resp.headers() + .get("retry-after") .and_then(|h| h.to_str().ok()), Some("1"), "the budget-truncation 503 must carry Retry-After" @@ -1821,7 +8005,7 @@ mod tests { /// the recorded pid is already dead: a tokio abort would have left it /// running), the log shows the walk started but never completed, and the /// request sheds the terminal budget-truncated 503 without ever reaching the - /// OLDER public copy of the same blob (which would have served 200). After + /// public copy of the same blob behind it (which would have served 200). After /// the response the permit is free: the spawn_blocking closure genuinely /// returned. MUTATION (RED): drop the `min` clamp on `walk_timeout` and the /// walk runs its full 8s sleep (elapsed and log-completion assertions fail). @@ -1872,13 +8056,13 @@ mod tests { cfg.ipfs_request_budget_secs = 2; state.config = Arc::new(cfg); - // Older row: a plain public copy of the same blob, which must never be - // reached. Newer row: path-scoped, so its blob costs the clamped walk. + // First-iterated row (seeded first, `(created_at, id)` ASC): path-scoped, so + // its blob costs the clamped walk. Behind it, a plain public copy of the same + // blob which must never be reached. let content = b"budget walk clamp proof\n"; - let (_, oid) = - seed_repo_with_blob(&state, tmp.path(), "z6f3clamp", "pubcopy", content).await; - let (walk_id, _) = + let (walk_id, oid) = seed_repo_with_blob(&state, tmp.path(), "z6f3clamp", "gated", content).await; + seed_repo_with_blob(&state, tmp.path(), "z6f3clamp", "pubcopy", content).await; state .db .set_visibility_rule( @@ -1892,10 +8076,11 @@ mod tests { .unwrap(); let sem = state.git_ipfs_walk_semaphore.clone(); + let cid = seed_legacy_pin_for_oid(&state, &oid).await; let router = ipfs_router(state); let started = std::time::Instant::now(); let peer: SocketAddr = "203.0.113.71:5000".parse().unwrap(); - let mut fut = Box::pin(router.oneshot(get_cid(&cid_for_oid(&oid), Some(peer)))); + let mut fut = Box::pin(router.oneshot(get_cid(&cid, Some(peer)))); // Drive until the fake git's rev-list records its pid: the walk is now in // the blocking pool and the request future is `.await`ing its join. Stop @@ -2071,9 +8256,10 @@ mod tests { let peer: SocketAddr = "203.0.113.72:5000".parse().unwrap(); // The request must return in bounded time: the reaped probe sheds a 503; a // bare unbounded probe would block on the FIFO forever (no feeder frees it). + let cid = seed_legacy_pin_for_oid(&state, &oid).await; let resp = tokio::time::timeout( std::time::Duration::from_secs(15), - ipfs_router(state).oneshot(get_cid(&cid_for_oid(&oid), Some(peer))), + ipfs_router(state).oneshot(get_cid(&cid, Some(peer))), ) .await .expect("the hung probe must be reaped, not block the handler") @@ -2255,34 +8441,33 @@ mod tests { drop(held); } - /// Retain-through-blocking (R3, the load-bearing async property): the walk - /// admission is held until the `spawn_blocking` walk actually RETURNS, not when a - /// tokio timeout fires. With the global pool at size 1, drive a request until its - /// walk (a fake git that hangs on `rev-list`) is in flight; the slot must stay held - /// (`available_permits() == 0`) and a replacement from a DIFFERENT source must shed - /// 503 for as long as the blocking walk runs — even though the request future is - /// only `.await`ing the blocking join. When the blocking walk ends the permit frees - /// and a replacement is admitted. The permit lives INSIDE the handler across the - /// blocking `.await`; move it out (drop before the walk) and the replacement would - /// be admitted while the walk still burns a blocking thread (the bug this guards). + /// Build the shared `/ipfs` TREE-walk fixture. A fake `git` whose `rev-list` records + /// its pid then sleeps ~6s (so the tree walk blocks deterministically inside + /// `run_bounded_git`) and whose `cat-file -t` answers "tree" (so the bounded + /// object-type probe, `object_type_bounded` on `state.git_bin`, routes into the + /// tree-gate arm); a real SHA-256 bare repo with a committed `src/` tree pinned WITH + /// provenance; and a path-scoped rule so the gate takes the tree-walk branch. Returns + /// the tempdir (keep it alive for the whole test), the state (the caller sets the walk + /// semaphores), the requested CID, and the rev-list pidfile path. #[cfg(unix)] - #[sqlx::test] - async fn get_by_cid_walk_permit_held_through_blocking_walk(pool: sqlx::PgPool) { + async fn seed_tree_walk_fixture( + pool: sqlx::PgPool, + ) -> ( + tempfile::TempDir, + crate::state::AppState, + String, + std::path::PathBuf, + ) { use std::process::Command; let tmp = tempfile::TempDir::new().unwrap(); let revlist_pid = tmp.path().join("revlist.pid"); - // Fake git for the /ipfs WALK only (object_type/read_object_content use the real - // `git`, so the object must genuinely exist below). Empty refs (so - // assert_all_refs_are_commits returns Ok without the peel), `rev-parse` resolves, - // and `rev-list` records its pid then sleeps ~6s so the walk BLOCKS - // deterministically. The sleep bounds the walk so a broken fix cannot wedge the - // suite. let body = format!( "#!/bin/sh\n\ case \"$1\" in\n\ for-each-ref) : ;;\n\ rev-parse) echo deadbeef ;;\n\ + cat-file) if [ \"$2\" = \"-t\" ]; then echo tree; fi ;;\n\ rev-list) echo $$ > \"{}\"; sleep 6 ;;\n\ *) : ;;\n\ esac\n\ @@ -2302,25 +8487,17 @@ mod tests { let repos_dir = tmp.path().join("repos"); std::fs::create_dir_all(&repos_dir).unwrap(); state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); - // Isolate the global walk pool at size 1; per-source cap permissive so only the - // held global permit can shed the replacement. - state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(1)); - state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); state.git_bin = git_path.to_str().unwrap().to_string(); state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; - let owner = "z6ipfs1"; - let name = "ip1"; + let owner = "z6ipfstree"; + let name = "iptree"; state .db .upsert_mirror_repo(owner, name, "/unused", None, false) .await .unwrap(); let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); - // The exact bare path the handler's `acquire` resolves. Build a REAL SHA-256 bare - // repo there with a committed blob under `src/`, so real `git cat-file -t ` classifies it as a blob (the CID digest IS the sha256 object id in - // object-format=sha256) and the handler reaches the path-scoped walk branch. let bare = state .repo_store .acquire(&rec.owner_did, &rec.name) @@ -2342,7 +8519,11 @@ mod tests { }; let work = tmp.path().join("work"); std::fs::create_dir_all(work.join("src")).unwrap(); - std::fs::write(work.join("src/secret.txt"), b"ipfs walk retain proof\n").unwrap(); + std::fs::write( + work.join("src/secret.txt"), + b"ipfs tree walk retain proof\n", + ) + .unwrap(); run( &["init", "-q", "--object-format=sha256", "-b", "main"], &work, @@ -2361,43 +8542,71 @@ mod tests { ], tmp.path(), ); - // The blob's SHA-256 object id (= the CID's digest); build the CID from it. - let oid = { + let tree_oid = { let out = Command::new("git") - .args(["rev-parse", "HEAD:src/secret.txt"]) + .args(["rev-parse", "HEAD:src"]) .current_dir(&work) .output() .expect("git rev-parse runs"); assert!(out.status.success(), "rev-parse failed"); String::from_utf8_lossy(&out.stdout).trim().to_string() }; - let oid_bytes = gitlawb_core::cid::sha256_hex_to_bytes(&oid).unwrap(); - let cid = gitlawb_core::cid::Cid::from_sha256_bytes(&oid_bytes) - .as_str() - .to_string(); - // Precondition: real git classifies the object as a blob (so the handler reaches - // the walk branch, not an early `continue`). assert_eq!( - crate::git::store::object_type(&bare, &oid) + crate::git::store::object_type(&bare, &tree_oid) .unwrap() .as_deref(), - Some("blob"), - "the seeded sha256 blob must exist so the handler reaches the walk" + Some("tree"), + "the seeded sha256 tree must exist so the handler reaches the tree walk" ); - // A path-scoped rule so has_path_scoped_rule() is true (the walk branch) without - // denying the "/" gate on the public repo. + let (_ty, raw) = crate::git::store::read_object(&bare, &tree_oid) + .unwrap() + .expect("tree object readable"); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(&raw).to_string(); + state + .db + .record_pinned_cid(&tree_oid, &cid, Some(&rec.id)) + .await + .unwrap(); state .db .set_visibility_rule( &rec.id, - "src/**", + "/src/**", crate::db::VisibilityMode::B, - &["did:key:z6MkU3IpfsReaderAAAAAAAAAAAAAAAAAAAAAAAA".to_string()], + &["did:key:z6MkF5IpfsTreeReaderAAAAAAAAAAAAAAAAAAAA".to_string()], &rec.owner_did, ) .await .unwrap(); + (tmp, state, cid, revlist_pid) + } + + /// Retain-through-blocking (#174 F5, the load-bearing async property, on the + /// NEWLY-BOUNDED TREE path): the walk admission is held until the `spawn_blocking` + /// walk actually RETURNS, not when a tokio timeout fires. The requested CID + /// resolves to a TREE object under a path-scoped rule, so the gate runs + /// `allowed_tree_set_for_caller_bounded` — the walk this integration converts to + /// `run_bounded_git` — rather than the blob walk #174 already proved. With the + /// global pool at size 1, drive a request until its walk (a fake git that hangs on + /// `rev-list`) is in flight; the slot must stay held (`available_permits() == 0`) + /// and a replacement from a DIFFERENT source must shed 503 for as long as the + /// blocking walk runs — even though the request future is only `.await`ing the + /// blocking join. When the blocking walk ends the permit frees and a replacement + /// is admitted. The permit lives INSIDE the handler across the blocking `.await`; + /// move it out (drop before the walk) and the replacement would be admitted while + /// the walk still burns a blocking thread (the bug this guards). + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_walk_permit_held_through_bounded_tree_walk(pool: sqlx::PgPool) { + let (tmp, mut state, cid, revlist_pid) = seed_tree_walk_fixture(pool).await; + // Isolate the global walk pool at size 1; per-source cap permissive so only the + // held global permit can shed the replacement. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(1)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + // Keep the fixture tempdir alive for the whole test (its Drop removes the repos). + let _tmp = tmp; + let sem = state.git_ipfs_walk_semaphore.clone(); assert_eq!( sem.available_permits(), @@ -2418,10 +8627,10 @@ mod tests { let peer: SocketAddr = "203.0.113.81:5000".parse().unwrap(); let mut fut = Box::pin(router.clone().oneshot(make_req(peer))); - // Drive until the fake git's rev-list records its pid — the walk is now in the - // blocking pool and the request future is `.await`ing its join, holding the walk - // permit. Stop polling the instant the future completes (re-polling a completed - // oneshot panics). + // Drive until the fake git's rev-list records its pid — the TREE walk is now in + // the blocking pool and the request future is `.await`ing its join, holding the + // walk permit. Stop polling the instant the future completes (re-polling a + // completed oneshot panics). let mut walk_pid: Option = None; let mut early = None; for _ in 0..500 { @@ -2451,20 +8660,20 @@ mod tests { } let _cleanup = ReapOnDrop(pid); - // Load-bearing: while the blocking walk runs, the slot is HELD and a replacement - // from a DIFFERENT source sheds 503 — proving the permit is retained across the - // spawn_blocking join, not freed by a tokio timeout. + // Load-bearing: while the blocking TREE walk runs, the slot is HELD and a + // replacement from a DIFFERENT source sheds 503 — proving the permit is + // retained across the spawn_blocking join, not freed by a tokio timeout. assert_eq!( sem.available_permits(), 0, - "the walk slot must be held while the spawn_blocking walk runs" + "the walk slot must be held while the spawn_blocking tree walk runs" ); let peer2: SocketAddr = "203.0.113.82:5000".parse().unwrap(); let resp = router.clone().oneshot(make_req(peer2)).await.unwrap(); assert_eq!( resp.status(), StatusCode::SERVICE_UNAVAILABLE, - "a replacement must shed 503 while the prior request's blocking walk still runs" + "a replacement must shed 503 while the prior request's blocking tree walk still runs" ); // Drop the in-flight request — a client disconnect. The detached blocking walk @@ -2499,17 +8708,122 @@ mod tests { unsafe { libc::kill(pid, libc::SIGKILL); } - let mut freed = false; + let mut freed = false; + for _ in 0..400 { + if sem.available_permits() == 1 { + freed = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + assert!( + freed, + "once the blocking walk tears down, the last admission clone drops and frees the slot" + ); + assert_eq!( + sem.available_permits(), + 1, + "admission released exactly once — the single slot is back, not double-freed" + ); + } + + /// Amplification negative (#173 round-10, R1): sequential cancel-spam from ONE source + /// cannot hold more than the per-source cap of concurrent walks. An abandoned + /// blocking walk keeps its per-source permit until its bounded work finishes (up + /// to `git_service_timeout_secs`), so with a per-source cap of 1 a second request from + /// the SAME source sheds 503 even though the GLOBAL pool has room — the source cannot + /// amplify its concurrent walk children past the cap by dropping-and-retrying. (The + /// worst case: an abandoned walk can occupy its global/per-source permit for one + /// bound-interval, so distributed cancel-spam can hold the global pool that long — the + /// accepted bounded-admission tradeoff, not a leak.) + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_cancel_spam_bounded_by_per_source_cap(pool: sqlx::PgPool) { + let (tmp, mut state, cid, revlist_pid) = seed_tree_walk_fixture(pool).await; + // Global pool has ample room (4); the per-source cap is 1. So any shed of a + // same-source replacement is the PER-SOURCE cap, never global exhaustion. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(4)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + let _tmp = tmp; + + let sem = state.git_ipfs_walk_semaphore.clone(); + let per_caller = state.git_ipfs_walk_per_caller.clone(); + let router = ipfs_router(state); + let make_req = |peer: SocketAddr| { + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + req + }; + + // Source S fires request 1; drive until its tree walk is in flight (the task now + // holds source S's single per-source permit). + let source_s: SocketAddr = "203.0.113.71:5000".parse().unwrap(); + let mut fut = Box::pin(router.clone().oneshot(make_req(source_s))); + let mut walk_pid: Option = None; + for _ in 0..500 { + let _ = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + if let Some(p) = std::fs::read_to_string(&revlist_pid) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + walk_pid = Some(p); + break; + } + } + let pid = walk_pid.expect("the fake git rev-list must have spawned"); + struct ReapOnDrop(i32); + impl Drop for ReapOnDrop { + fn drop(&mut self) { + unsafe { + libc::kill(self.0, libc::SIGKILL); + } + } + } + let _cleanup = ReapOnDrop(pid); + + // Cancel-spam: drop request 1's future. The uncancellable blocking walk keeps + // running and KEEPS holding source S's single per-source permit. + drop(fut); + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + // A SECOND request from source S sheds 503. The global pool still has room (only 1 + // of 4 taken), so this is the per-source cap, not global exhaustion. + let resp = router.clone().oneshot(make_req(source_s)).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a same-source cancel-spam replacement must shed 503 on the per-source cap \ + while the abandoned walk still holds the source's permit" + ); + assert!( + sem.available_permits() >= 3, + "the shed was the per-source cap, not global exhaustion (global pool still has room)" + ); + assert_eq!( + per_caller.tracked_keys(), + 1, + "exactly one per-source permit is outstanding for the one source — no amplification" + ); + + // Tear the walk down; the closure returns and releases source S's permit + // (tracked_keys returns to 0), so the source is no longer over the cap. + unsafe { + libc::kill(pid, libc::SIGKILL); + } + let mut released = false; for _ in 0..400 { - if sem.available_permits() == 1 { - freed = true; + if per_caller.tracked_keys() == 0 { + released = true; break; } tokio::time::sleep(std::time::Duration::from_millis(50)).await; } assert!( - freed, - "once the blocking walk ends the walk permit must free the global slot" + released, + "once the blocking walk tears down it releases source S's per-source permit" ); } @@ -2521,11 +8835,17 @@ mod tests { /// a deny VERDICT) and the second is cut by the cap (no verdict), so the fake git's /// `rev-list` runs exactly once and the request sheds a retryable 503 + Retry-After /// — never the old false 404 (the blob genuinely sits in the second repo). - /// MUTATION (RED): remove the `repos_walked >= cap` skip and both repos are walked - /// (count 2); drop the truncation taint on the skip and the 503 decays to a 404. + /// This drives the GITLAWB_IPFS_MAX_REPOS_WALKED knob specifically. The merge left + /// two walk caps in play, this one and the branch's own history-walk ceiling, and + /// the gate takes the tighter of the two; setting this knob to 1 is what makes it + /// the binding one here. A sibling case covers the ceiling. + /// + /// MUTATION (RED): drop `config.ipfs_max_repos_walked` from the `min()` in the walk + /// gate and both repos are walked (count 2); drop the truncation taint on the skip + /// and the 503 decays to a 404. #[cfg(unix)] #[sqlx::test] - async fn get_by_cid_caps_repos_walked_per_request(pool: sqlx::PgPool) { + async fn get_by_cid_caps_repos_walked_knob_bounds_the_walks(pool: sqlx::PgPool) { use std::process::Command; let tmp = tempfile::TempDir::new().unwrap(); @@ -2638,119 +8958,508 @@ mod tests { .await .unwrap(); } - let oid_bytes = gitlawb_core::cid::sha256_hex_to_bytes(&oid).unwrap(); - let cid = gitlawb_core::cid::Cid::from_sha256_bytes(&oid_bytes) - .as_str() - .to_string(); + // The resolver maps a requested CID back to an oid through the CID index, so a + // bare digest-as-oid CID resolves to nothing and 404s before any repo is + // visited. Register a legacy NULL-provenance row, which is also what routes the + // request to the bounded legacy scan this cap governs. Neither repo serves, so + // the key need not be the content CID. + let cid = seed_legacy_pin(&state, &oid).await; + + let peer: SocketAddr = "203.0.113.90:5000".parse().unwrap(); + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + let resp = ipfs_router(state).oneshot(req).await.unwrap(); + // The first repo's walk yields the empty allowed-set (deny verdict); the second + // repo NEEDS a walk the cap forbids, so the scan is truncated without a verdict + // on it: retryable 503, never a false 404 for the blob it genuinely carries. + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a walk-cap truncation must shed a retryable 503, not report the object absent" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the truncation 503 must carry Retry-After" + ); + + let walks = std::fs::read_to_string(&walk_log) + .map(|s| s.lines().count()) + .unwrap_or(0); + assert_eq!( + walks, 1, + "with the per-request repo-walk cap at 1, only the first candidate repo is \ + walked (the second is cut by the cap), so exactly one walk runs; got {walks}" + ); + } + + /// Route rate limit is WIRED (not a silent no-op): the production `build_router` + /// attaches an `IpRateLimiter` extension to the `/ipfs/{cid}` route, so a per-IP + /// flood is braked with 429. A bare `rate_limit_by_ip` layer with no extension does + /// nothing, so this proves the extension is attached. Drive it through the real + /// router with a tight limiter (1/hr): the second request from the same IP is 429. + /// MUTATION (RED): drop the `axum::Extension(ipfs_limiter)` layer in `server.rs` and + /// the second request is no longer braked (it reaches the handler, 404, not 429). + #[sqlx::test] + async fn ipfs_route_ip_rate_limit_is_attached(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool).await; + // Tight per-IP /ipfs bucket so the second request from one IP trips 429. + state.ipfs_rate_limiter = + crate::rate_limit::RateLimiter::new(1, std::time::Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let router = crate::server::build_router(state); + let cid = valid_cid(); + let make = |peer: SocketAddr| { + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + req + }; + let peer: SocketAddr = "203.0.113.99:5000".parse().unwrap(); + + // First request from this IP passes the brake and reaches the handler (404 — no + // such object anywhere), debiting the single-slot bucket. + let resp = router.clone().oneshot(make(peer)).await.unwrap(); + assert_ne!( + resp.status(), + StatusCode::TOO_MANY_REQUESTS, + "the first /ipfs request from an IP must pass the rate brake" + ); + // Second request from the SAME IP is braked with 429 — proving the limiter + // extension is attached (a bare no-op layer would let it through to 404). + let resp = router.clone().oneshot(make(peer)).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::TOO_MANY_REQUESTS, + "an exhausted per-IP /ipfs bucket must brake with 429 — the IpRateLimiter \ + extension must be attached to the route" + ); + // A DIFFERENT IP still has its own budget (independent bucket). + let other: SocketAddr = "203.0.113.100:5000".parse().unwrap(); + let resp = router.oneshot(make(other)).await.unwrap(); + assert_ne!( + resp.status(), + StatusCode::TOO_MANY_REQUESTS, + "a different IP must not be braked by another IP's exhausted bucket" + ); + } + + /// F6/KTD-5: the legacy scan's page queries (`list_repos_page_for_scan`, + /// `list_visibility_rules_for_repos`) run AFTER the scarce walk permits are + /// acquired (held RAII for the whole request) but BEFORE the per-repo loop's + /// first budget gate. Pre-fix they were bare awaits with no deadline, so a query + /// blocked in Postgres pinned the walk slot for the whole stall, past the request + /// budget. Here we hold an ACCESS EXCLUSIVE lock on `repos` so the page query + /// blocks; with the budget clamp the request sheds a retryable budget 503 within + /// ~budget and FREES the walk permit, and a follow-up (lock released) is served. + /// + /// Load-bearing: pre-fix the bare await blocks on the lock until the 10s wrapping + /// timeout fires (RED — "never returned within budget"). After the fix it returns + /// the 503 at ~1s and the permit is free again. MUTATION (RED): drop the + /// `tokio::time::timeout` around `list_repos_page_for_scan` and this hangs past + /// the wrap. + #[sqlx::test] + async fn get_by_cid_stalled_metadata_query_frees_walk_permit(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // Global walk pool of 1 so the held/freed permit is directly observable; + // per-source cap permissive so only the global pool matters. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(1)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + let mut cfg = (*state.config).clone(); + cfg.ipfs_request_budget_secs = 1; + state.config = Arc::new(cfg); + + let sem = state.git_ipfs_walk_semaphore.clone(); + let cid = seed_legacy_pin(&state, &absent_oid()).await; + let router = ipfs_router(state); + + // Hold an ACCESS EXCLUSIVE lock on `repos` on a dedicated pooled connection: + // the page SELECT needs ACCESS SHARE, which conflicts, so it blocks at lock + // acquisition regardless of row count. + let mut lock_conn = pool.acquire().await.unwrap(); + sqlx::raw_sql("BEGIN; LOCK TABLE repos IN ACCESS EXCLUSIVE MODE;") + .execute(&mut *lock_conn) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.80:5000".parse().unwrap(); + let started = std::time::Instant::now(); + let resp = tokio::time::timeout( + std::time::Duration::from_secs(10), + router.clone().oneshot(get_cid(&cid, Some(peer))), + ) + .await + .expect("the budget clamp must return within budget; a bare await hangs on the lock") + .unwrap(); + let elapsed = started.elapsed(); + + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a metadata query blocked past the request budget must shed a retryable 503" + ); + assert!( + elapsed < std::time::Duration::from_secs(3), + "the clamp must end the request at ~budget (1s); got {elapsed:?} \ + (pre-fix the bare await blocks on the lock for the whole stall)" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("budget"), + "the shed must name the budget taint so it maps to \ + GITLAWB_IPFS_REQUEST_BUDGET_SECS; got: {body}" + ); + // The scarce walk permit was RAII-dropped on the early return, not pinned for + // the stall: the slot is free again the instant the request returns. + assert_eq!( + sem.available_permits(), + 1, + "the walk permit must be freed on the budget-shed path, not held for the stall" + ); + + // Release the lock; a follow-up request is now SERVED (404 — empty DB), never + // capacity-503'd, proving the slot was not left pinned. + sqlx::raw_sql("ROLLBACK") + .execute(&mut *lock_conn) + .await + .unwrap(); + drop(lock_conn); + let resp2 = router.oneshot(get_cid(&cid, Some(peer))).await.unwrap(); + assert_eq!( + resp2.status(), + StatusCode::NOT_FOUND, + "with the permit freed and the lock released, a follow-up is served (404), \ + not capacity-503'd" + ); + } + + /// F2 (#174): `oids_for_cid` is the FIRST DB await inside the admission-held + /// region, and pre-fix it was a bare await with no deadline. A query blocked in + /// Postgres there pinned both walk permits for the whole stall, past the request + /// budget, so later /ipfs requests took capacity 503s long after + /// GITLAWB_IPFS_REQUEST_BUDGET_SECS elapsed, reachable by any unauthenticated + /// caller. Here `pinned_cids` (the only table `oids_for_cid` reads) is held + /// ACCESS EXCLUSIVE so the query blocks at lock acquisition. + /// + /// The follow-up after ROLLBACK is a 404 rather than a 200 because this scenario + /// seeds no pin at all; "admitted and answered, never capacity-503'd" is what + /// proves the permit came back. + /// + /// Load-bearing: pre-fix the bare await blocks on the lock until the 10s wrapping + /// timeout fires (RED). MUTATION (RED): drop the `tokio::time::timeout` around + /// `oids_for_cid` and this hangs past the wrap. + #[sqlx::test] + async fn get_by_cid_stalled_oids_query_frees_walk_permit(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // Global walk pool of 1 so the held/freed permit is directly observable; + // per-source cap permissive so only the global pool matters. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(1)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + let mut cfg = (*state.config).clone(); + cfg.ipfs_request_budget_secs = 1; + state.config = Arc::new(cfg); + + let sem = state.git_ipfs_walk_semaphore.clone(); + // A well-formed CID with no `pinned_cids` row: the request still runs the + // `oids_for_cid` lookup, which is the await under test. + let cid = cid_for_oid(&absent_oid()); + let router = ipfs_router(state); + + let mut lock_conn = pool.acquire().await.unwrap(); + sqlx::raw_sql("BEGIN; LOCK TABLE pinned_cids IN ACCESS EXCLUSIVE MODE;") + .execute(&mut *lock_conn) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.81:5000".parse().unwrap(); + let started = std::time::Instant::now(); + let resp = tokio::time::timeout( + std::time::Duration::from_secs(10), + router.clone().oneshot(get_cid(&cid, Some(peer))), + ) + .await + .expect("the budget clamp must return within budget; a bare await hangs on the lock") + .unwrap(); + let elapsed = started.elapsed(); + + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an oid lookup blocked past the request budget must shed a retryable 503" + ); + assert!( + elapsed < std::time::Duration::from_secs(3), + "the clamp must end the request at ~budget (1s); got {elapsed:?} \ + (pre-fix the bare await blocks on the lock for the whole stall)" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("budget"), + "the shed must name the budget taint so it maps to \ + GITLAWB_IPFS_REQUEST_BUDGET_SECS; got: {body}" + ); + assert_eq!( + sem.available_permits(), + 1, + "the walk permit must be freed on the budget-shed path, not held for the stall" + ); + + sqlx::raw_sql("ROLLBACK") + .execute(&mut *lock_conn) + .await + .unwrap(); + drop(lock_conn); + let resp2 = router.oneshot(get_cid(&cid, Some(peer))).await.unwrap(); + assert_eq!( + resp2.status(), + StatusCode::NOT_FOUND, + "with the permit freed and the lock released, a follow-up is ADMITTED and \ + answers 404 (no pin was seeded), never capacity-503'd" + ); + } + + /// F4 (#174 round 13): the pre-walk resolve carries its OWN short budget. The + /// clamp above proves `oids_for_cid` cannot run unbounded, but its deadline is the + /// 600s request budget, and a CID with no `pinned_cids` row does zero probe and + /// zero walk work. Under a stalled pool such a request held the scarce walk slot + /// for that whole window while nothing walked, so requests from enough distinct + /// source keys capacity-503'd every real `/ipfs` retrieval at admission. + /// + /// The request budget is left at its 600s DEFAULT here on purpose: that is the + /// whole point of the scenario, since the short resolve budget, not the long + /// request budget, is what must end this request. + /// + /// Load-bearing: without the resolve clamp the stalled lookup runs to the 600s + /// request budget and blows past the 10s wrap (RED). MUTATION (RED): revert the + /// clamp to `remaining()` only, the pre-fix shape. + #[sqlx::test] + async fn get_by_cid_stalled_resolve_frees_walk_permit_within_resolve_budget( + pool: sqlx::PgPool, + ) { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // Global walk pool of 1 so the held/freed permit is directly observable; + // per-source cap permissive so only the global pool matters. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(1)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + let mut cfg = (*state.config).clone(); + cfg.ipfs_resolve_budget_secs = 1; + assert_eq!( + cfg.ipfs_request_budget_secs, 600, + "the request budget stays at its default: this scenario proves the SHORT \ + budget is what sheds" + ); + state.config = Arc::new(cfg); - let peer: SocketAddr = "203.0.113.90:5000".parse().unwrap(); - let mut req = Request::builder() - .method(Method::GET) - .uri(format!("/ipfs/{cid}")) - .body(Body::empty()) + let sem = state.git_ipfs_walk_semaphore.clone(); + // A well-formed CID with no `pinned_cids` row: an anonymous caller's request + // that will do no admitted work at all once the lookup answers. + let cid = cid_for_oid(&absent_oid()); + let router = ipfs_router(state); + + let mut lock_conn = pool.acquire().await.unwrap(); + sqlx::raw_sql("BEGIN; LOCK TABLE pinned_cids IN ACCESS EXCLUSIVE MODE;") + .execute(&mut *lock_conn) + .await .unwrap(); - req.extensions_mut().insert(ConnectInfo(peer)); - let resp = ipfs_router(state).oneshot(req).await.unwrap(); - // The first repo's walk yields the empty allowed-set (deny verdict); the second - // repo NEEDS a walk the cap forbids, so the scan is truncated without a verdict - // on it: retryable 503, never a false 404 for the blob it genuinely carries. + + let peer: SocketAddr = "203.0.113.85:5000".parse().unwrap(); + let started = std::time::Instant::now(); + let resp = tokio::time::timeout( + std::time::Duration::from_secs(10), + router.clone().oneshot(get_cid(&cid, Some(peer))), + ) + .await + .expect( + "the resolve clamp must return within the SHORT budget; on the request budget \ + alone the stalled lookup holds for 600s", + ) + .unwrap(); + let elapsed = started.elapsed(); + assert_eq!( resp.status(), StatusCode::SERVICE_UNAVAILABLE, - "a walk-cap truncation must shed a retryable 503, not report the object absent" + "a resolve blocked past the resolve budget must shed a retryable 503" + ); + assert!( + elapsed < std::time::Duration::from_secs(3), + "the clamp must end the request at ~the resolve budget (1s); got {elapsed:?} \ + (on the request budget alone it runs for 600s)" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + // INV-24: the knob an operator can turn must be the one the shed names. The two + // budgets are separately settable, so a body naming the request budget here + // would point at a knob that did nothing. + assert!( + body.contains("resolve budget"), + "the shed must name the RESOLVE budget so it maps to \ + GITLAWB_IPFS_RESOLVE_BUDGET_SECS; got: {body}" + ); + assert!( + !body.contains("request budget"), + "the resolve shed must not name the request budget, which is untouched at \ + 600s here and would send an operator to the wrong knob; got: {body}" ); assert_eq!( - resp.headers() - .get("retry-after") - .and_then(|h| h.to_str().ok()), - Some("1"), - "the truncation 503 must carry Retry-After" + sem.available_permits(), + 1, + "the walk permit must be freed on the resolve-budget shed path, not held \ + for the stall" ); - let walks = std::fs::read_to_string(&walk_log) - .map(|s| s.lines().count()) - .unwrap_or(0); + sqlx::raw_sql("ROLLBACK") + .execute(&mut *lock_conn) + .await + .unwrap(); + drop(lock_conn); + let resp2 = router.oneshot(get_cid(&cid, Some(peer))).await.unwrap(); assert_eq!( - walks, 1, - "with the per-request repo-walk cap at 1, only the first candidate repo is \ - walked (the second is cut by the cap), so exactly one walk runs; got {walks}" + resp2.status(), + StatusCode::NOT_FOUND, + "with the permit freed and the lock released, a follow-up is ADMITTED and \ + answers 404 (no pin was seeded), never capacity-503'd" ); } - /// Route rate limit is WIRED (not a silent no-op): the production `build_router` - /// attaches an `IpRateLimiter` extension to the `/ipfs/{cid}` route, so a per-IP - /// flood is braked with 429. A bare `rate_limit_by_ip` layer with no extension does - /// nothing, so this proves the extension is attached. Drive it through the real - /// router with a tight limiter (1/hr): the second request from the same IP is 429. - /// MUTATION (RED): drop the `axum::Extension(ipfs_limiter)` layer in `server.rs` and - /// the second request is no longer braked (it reaches the handler, 404, not 429). + /// F4 must-not (#174 round 13): the short resolve budget bounds the RESOLVE and + /// nothing else. A request whose resolve answers promptly and then spends real time + /// in an admitted visibility walk is PROGRESSING, and shedding it would convert a + /// slow-but-correct retrieval into a 503 on a box that is merely loaded. + /// + /// The resolve budget is 1s while the walk sleeps ~2s per `rev-list`, so any + /// deadline that reaches past `oids_for_cid` ends this request before it can serve. + /// The shim execs the REAL git after sleeping, so the allowed-set the walk produces + /// is the repo's genuine one and the 200 is a real serve, not an artifact. + /// + /// MUTATION (RED): anchor the region's `remaining()` on the resolve deadline (the + /// plausible over-wide re-implementation: one short admission-anchored clock for + /// the whole permit-held region) and this 200 becomes a budget 503. + #[cfg(unix)] #[sqlx::test] - async fn ipfs_route_ip_rate_limit_is_attached(pool: sqlx::PgPool) { - let mut state = crate::test_support::test_state(pool).await; - // Tight per-IP /ipfs bucket so the second request from one IP trips 429. - state.ipfs_rate_limiter = - crate::rate_limit::RateLimiter::new(1, std::time::Duration::from_secs(3600)); + async fn get_by_cid_slow_walk_not_shed_by_resolve_budget(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(1)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); - let router = crate::server::build_router(state); - let cid = valid_cid(); - let make = |peer: SocketAddr| { - let mut req = Request::builder() - .method(Method::GET) - .uri(format!("/ipfs/{cid}")) - .body(Body::empty()) - .unwrap(); - req.extensions_mut().insert(ConnectInfo(peer)); - req - }; - let peer: SocketAddr = "203.0.113.99:5000".parse().unwrap(); + let content = b"slow but progressing\n"; + let (repo_id, oid) = + seed_repo_with_blob(&state, tmp.path(), "z6slowwalk", "holder", content).await; + // Path-scoped, so serving costs a real reachability walk: the rule withholds a + // path the seeded blob is NOT under (it lives at `src/secret.txt`), so anon is + // allowed and the request must reach a 200 the slow way. + state + .db + .set_visibility_rule( + &repo_id, + "withheld/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkU3IpfsReaderDDDDDDDDDDDDDDDDDDDDDDDD".to_string()], + "z6slowwalk", + ) + .await + .unwrap(); + let cid = seed_legacy_pin_for_oid(&state, &oid).await; - // First request from this IP passes the brake and reaches the handler (404 — no - // such object anywhere), debiting the single-slot bucket. - let resp = router.clone().oneshot(make(peer)).await.unwrap(); - assert_ne!( - resp.status(), - StatusCode::TOO_MANY_REQUESTS, - "the first /ipfs request from an IP must pass the rate brake" + // Sleep only on the reachability walk, then exec the real git, so the delay + // lands inside the admitted region and after the resolve has already answered. + let shim = tmp.path().join("slowwalkgit"); + std::fs::write( + &shim, + "#!/bin/sh\n\ + case \"$1\" in\n\ + rev-list) sleep 2 ;;\n\ + esac\n\ + exec git \"$@\"\n", + ) + .unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&shim).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&shim, perm).unwrap(); + } + state.git_bin = shim.to_str().unwrap().to_string(); + + let mut cfg = (*state.config).clone(); + cfg.ipfs_resolve_budget_secs = 1; + state.config = Arc::new(cfg); + + let peer: SocketAddr = "203.0.113.86:5000".parse().unwrap(); + let started = std::time::Instant::now(); + let resp = ipfs_router(state) + .oneshot(get_cid(&cid, Some(peer))) + .await + .unwrap(); + let elapsed = started.elapsed(); + let status = resp.status(); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + assert_eq!( + status, + StatusCode::OK, + "a walk that outlives the SHORT resolve budget must still serve: the resolve \ + budget bounds the pre-walk lookup, never admitted walk work. Got: {}", + String::from_utf8_lossy(&body) ); - // Second request from the SAME IP is braked with 429 — proving the limiter - // extension is attached (a bare no-op layer would let it through to 404). - let resp = router.clone().oneshot(make(peer)).await.unwrap(); assert_eq!( - resp.status(), - StatusCode::TOO_MANY_REQUESTS, - "an exhausted per-IP /ipfs bucket must brake with 429 — the IpRateLimiter \ - extension must be attached to the route" + &body[..], + content.as_slice(), + "the served bytes must be the seeded object's" ); - // A DIFFERENT IP still has its own budget (independent bucket). - let other: SocketAddr = "203.0.113.100:5000".parse().unwrap(); - let resp = router.oneshot(make(other)).await.unwrap(); - assert_ne!( - resp.status(), - StatusCode::TOO_MANY_REQUESTS, - "a different IP must not be braked by another IP's exhausted bucket" + // Anti-vacuity: without a walk that genuinely outlives the 1s resolve budget, + // the 200 above would prove nothing about the boundary. + assert!( + elapsed >= std::time::Duration::from_secs(2), + "the request must actually have spent longer than the 1s resolve budget in \ + the walk; got {elapsed:?}, so the shim's sleep never ran" ); } - /// F6/KTD-5: the two initial metadata queries (`list_all_repos`, - /// `list_visibility_rules_for_repos`) run AFTER the scarce walk permits are - /// acquired (held RAII for the whole request) but BEFORE the per-repo loop's - /// first budget gate. Pre-fix they were bare awaits with no deadline, so a query - /// blocked in Postgres pinned the walk slot for the whole stall, past the request - /// budget. Here we hold an ACCESS EXCLUSIVE lock on `repos` so `list_all_repos` - /// blocks; with the budget clamp the request sheds a retryable budget 503 within - /// ~budget and FREES the walk permit, and a follow-up (lock released) is served. + /// F2 (#174), second lockable site: `pin_sources_for_oid` runs once per candidate + /// oid, still inside the admission-held region, and was likewise a bare await. /// - /// Load-bearing: pre-fix the bare await blocks on the lock until the 10s wrapping - /// timeout fires (RED — "never returned within budget"). After the fix it returns - /// the 503 at ~1s and the permit is free again. MUTATION (RED): drop the - /// `tokio::time::timeout` around `list_all_repos` and this hangs past the wrap. + /// The lock isolates it from the first await: `oids_for_cid` reads only + /// `pinned_cids`, which stays unlocked, so it completes and the handler reaches + /// the per-oid loop; `pin_sources_for_oid` also reads `pin_repo_sources`, which is + /// held ACCESS EXCLUSIVE, so it is the query that blocks. A seeded legacy pin is + /// what gives the loop an oid to iterate. + /// + /// Load-bearing: pre-fix the bare await blocks on the lock until the 10s wrap + /// fires (RED). MUTATION (RED): drop the `tokio::time::timeout` around + /// `pin_sources_for_oid`; that mutation must leave the `oids_for_cid` scenario + /// above GREEN, which is what proves the two tests isolate their own queries. #[sqlx::test] - async fn get_by_cid_stalled_metadata_query_frees_walk_permit(pool: sqlx::PgPool) { + async fn get_by_cid_stalled_pin_sources_query_frees_walk_permit(pool: sqlx::PgPool) { let mut state = crate::test_support::test_state(pool.clone()).await; state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; - // Global walk pool of 1 so the held/freed permit is directly observable; - // per-source cap permissive so only the global pool matters. state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(1)); state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); let mut cfg = (*state.config).clone(); @@ -2758,22 +9467,20 @@ mod tests { state.config = Arc::new(cfg); let sem = state.git_ipfs_walk_semaphore.clone(); + let cid = seed_legacy_pin(&state, &absent_oid()).await; let router = ipfs_router(state); - // Hold an ACCESS EXCLUSIVE lock on `repos` on a dedicated pooled connection: - // `list_all_repos`' SELECT needs ACCESS SHARE, which conflicts, so it blocks - // at lock acquisition regardless of row count. let mut lock_conn = pool.acquire().await.unwrap(); - sqlx::raw_sql("BEGIN; LOCK TABLE repos IN ACCESS EXCLUSIVE MODE;") + sqlx::raw_sql("BEGIN; LOCK TABLE pin_repo_sources IN ACCESS EXCLUSIVE MODE;") .execute(&mut *lock_conn) .await .unwrap(); - let peer: SocketAddr = "203.0.113.80:5000".parse().unwrap(); + let peer: SocketAddr = "203.0.113.82:5000".parse().unwrap(); let started = std::time::Instant::now(); let resp = tokio::time::timeout( std::time::Duration::from_secs(10), - router.clone().oneshot(get_cid(&valid_cid(), Some(peer))), + router.clone().oneshot(get_cid(&cid, Some(peer))), ) .await .expect("the budget clamp must return within budget; a bare await hangs on the lock") @@ -2783,7 +9490,7 @@ mod tests { assert_eq!( resp.status(), StatusCode::SERVICE_UNAVAILABLE, - "a metadata query blocked past the request budget must shed a retryable 503" + "a pin-source lookup blocked past the request budget must shed a retryable 503" ); assert!( elapsed < std::time::Duration::from_secs(3), @@ -2799,30 +9506,23 @@ mod tests { "the shed must name the budget taint so it maps to \ GITLAWB_IPFS_REQUEST_BUDGET_SECS; got: {body}" ); - // The scarce walk permit was RAII-dropped on the early return, not pinned for - // the stall: the slot is free again the instant the request returns. assert_eq!( sem.available_permits(), 1, "the walk permit must be freed on the budget-shed path, not held for the stall" ); - // Release the lock; a follow-up request is now SERVED (404 — empty DB), never - // capacity-503'd, proving the slot was not left pinned. sqlx::raw_sql("ROLLBACK") .execute(&mut *lock_conn) .await .unwrap(); drop(lock_conn); - let resp2 = router - .oneshot(get_cid(&valid_cid(), Some(peer))) - .await - .unwrap(); + let resp2 = router.oneshot(get_cid(&cid, Some(peer))).await.unwrap(); assert_eq!( resp2.status(), StatusCode::NOT_FOUND, - "with the permit freed and the lock released, a follow-up is served (404), \ - not capacity-503'd" + "with the permit freed and the lock released, a follow-up is served (404 \ + against an empty repo set), never capacity-503'd" ); } @@ -2832,7 +9532,9 @@ mod tests { /// listing — exposing a public repo's path-restricted blob. Here a PUBLIC repo /// carries the blob under a path-scoped rule that denies anon; `visibility_rules` /// is locked ACCESS EXCLUSIVE so the rule query blocks. The fix returns the budget - /// 503 BEFORE the loop, so the handler NEVER serves (never 200). + /// 503 BEFORE the loop, so the handler NEVER serves (never 200). Since the scan was + /// paged (#173, jatmn) the rules are fetched per PAGE, so this covers the clamp on + /// every page rather than on one whole-inventory load. /// /// Load-bearing: pre-fix the bare await blocks on the lock until the 10s wrap fires /// (RED). After the fix it sheds the 503 at ~1s. The `assert_ne!(200)` is the @@ -2871,9 +9573,10 @@ mod tests { cfg.ipfs_request_budget_secs = 1; state.config = Arc::new(cfg); let sem = state.git_ipfs_walk_semaphore.clone(); + let cid = seed_legacy_pin_for_oid(&state, &oid).await; let router = ipfs_router(state); - // Lock `visibility_rules` ACCESS EXCLUSIVE: list_all_repos (on `repos`) still + // Lock `visibility_rules` ACCESS EXCLUSIVE: the page query (on `repos`) still // succeeds, but list_visibility_rules_for_repos blocks on the rule query. let mut lock_conn = pool.acquire().await.unwrap(); sqlx::raw_sql("BEGIN; LOCK TABLE visibility_rules IN ACCESS EXCLUSIVE MODE;") @@ -2885,7 +9588,7 @@ mod tests { let started = std::time::Instant::now(); let resp = tokio::time::timeout( std::time::Duration::from_secs(10), - router.oneshot(get_cid(&cid_for_oid(&oid), Some(peer))), + router.oneshot(get_cid(&cid, Some(peer))), ) .await .expect("the budget clamp must return within budget; a bare await hangs on the lock") @@ -2929,4 +9632,156 @@ mod tests { .unwrap(); drop(lock_conn); } + + /// Seed a PROVENANCED pin whose single recorded source repo does not exist, and + /// return its CID. `pin_sources_for_oid` therefore comes back NON-EMPTY (so + /// `needs_scan` cannot short-circuit on `sources.is_empty()`) while the per-source + /// loop takes the `get_repo_by_id -> None` arm and falls straight through to the + /// marker pair. `pin_repo_sources` stays empty, so `pin_sources_at_cap` is `false` + /// and the `||` goes on to evaluate `pin_sources_incomplete`. + async fn seed_provenanced_pin_with_missing_source( + state: &crate::state::AppState, + oid: &str, + ) -> String { + let cid = cid_for_oid(oid); + state + .db + .record_pinned_cid(oid, &cid, Some("repo-id-that-does-not-exist")) + .await + .expect("seed a provenanced pin row"); + cid + } + + /// Shared body for the two marker-query stall cases. Arms the seam for `which`, + /// drives one request, and asserts the whole budget-shed contract: a 503 inside the + /// budget, a body naming the budget taint, the walk permit BACK in the pool rather + /// than pinned for the stall, and a follow-up that is ADMITTED (404 here, since the + /// fixture seeds no servable object) instead of capacity-shed. + async fn assert_marker_query_stall_frees_walk_permit( + pool: sqlx::PgPool, + which: MarkerQuery, + peer: SocketAddr, + label: &str, + ) { + let mut state = crate::test_support::test_state(pool).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // Global walk pool of 1 so the held/freed permit is directly observable; + // per-source cap permissive so only the global pool matters. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(1)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + let mut cfg = (*state.config).clone(); + cfg.ipfs_request_budget_secs = 1; + state.config = Arc::new(cfg); + + let sem = state.git_ipfs_walk_semaphore.clone(); + let cid = seed_provenanced_pin_with_missing_source(&state, &absent_oid()).await; + let router = ipfs_router(state); + + // 30s so the stall is decided by the 1s budget clamp, never by the sleep + // finishing on its own. + arm_marker_query_stall(which, std::time::Duration::from_secs(30)); + let started = std::time::Instant::now(); + let resp = tokio::time::timeout( + std::time::Duration::from_secs(10), + router.clone().oneshot(get_cid(&cid, Some(peer))), + ) + .await + .unwrap_or_else(|_| { + disarm_marker_query_stall(); + panic!("{label}: the budget clamp must return within budget; a bare await hangs") + }) + .unwrap(); + let elapsed = started.elapsed(); + disarm_marker_query_stall(); + + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "{label}: a marker query blocked past the request budget must shed a retryable 503" + ); + assert!( + elapsed < std::time::Duration::from_secs(3), + "{label}: the clamp must end the request at ~budget (1s); got {elapsed:?} \ + (an unclamped await runs the full 30s stall)" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("budget"), + "{label}: the shed must name the budget taint so it maps to \ + GITLAWB_IPFS_REQUEST_BUDGET_SECS; got: {body}" + ); + // The scarce walk permit was RAII-dropped on the early return, not pinned for + // the stall: the slot is free again the instant the request returns. + assert_eq!( + sem.available_permits(), + 1, + "{label}: the walk permit must be freed on the budget-shed path, not held \ + for the stall" + ); + + // With the seam disarmed the follow-up is ADMITTED and answered (404, since the + // fixture seeds no servable object), never capacity-503'd, which is what proves + // the slot came back rather than staying pinned. + let resp2 = router.oneshot(get_cid(&cid, Some(peer))).await.unwrap(); + assert_eq!( + resp2.status(), + StatusCode::NOT_FOUND, + "{label}: with the permit freed, a follow-up is admitted and answers 404, \ + never capacity-503'd" + ); + } + + /// F2 (#174), marker pair, first query: `pin_sources_at_cap` runs on a provenance + /// MISS, still inside the admission-held region, and pre-fix was a bare await. A + /// query blocked in Postgres there pinned the walk permits for the whole stall, + /// past the request budget, capacity-503'ing later requests from any + /// unauthenticated caller. + /// + /// This one needs the `#[cfg(test)]` fault-injection seam rather than a `LOCK TABLE` + /// fixture: it reads `pin_repo_sources`, which `pin_sources_for_oid` already read + /// earlier in the same region, so a table lock stalls that earlier await and the RED + /// lands on the wrong clamp. See `MARKER_QUERY_STALL` for the full reasoning. The + /// injected sleep sits INSIDE the `tokio::time::timeout`, so the clamp is what ends + /// the request. + /// + /// MUTATION (RED): replace the clamp around `pin_sources_at_cap` with the bare + /// await, keeping the seam, and the request runs the full 30s stall past the 10s + /// wrap. + #[sqlx::test] + async fn get_by_cid_stalled_pin_sources_at_cap_frees_walk_permit(pool: sqlx::PgPool) { + assert_marker_query_stall_frees_walk_permit( + pool, + MarkerQuery::AtCap, + "203.0.113.83:5000".parse().unwrap(), + "pin_sources_at_cap", + ) + .await; + } + + /// F2 (#174), marker pair, second query: `pin_sources_incomplete` is a SEPARATE + /// clamp, evaluated only when `pin_sources_at_cap` came back `false`, and carries + /// the same pre-fix bare-await exposure. The fixture leaves `pin_repo_sources` + /// empty so `at_cap` is `false` and the `||` actually reaches this query; the seam + /// is armed for `Incomplete` only, so the first query is untouched and the RED is + /// attributable to this clamp alone. + /// + /// It reads `pinned_cids`, which `oids_for_cid` and `pin_sources_for_oid` already + /// read, so it is unlockable for the same reason as its sibling above. + /// + /// MUTATION (RED): replace the clamp around `pin_sources_incomplete` with the bare + /// await, keeping the seam, and the request runs the full 30s stall past the 10s + /// wrap. + #[sqlx::test] + async fn get_by_cid_stalled_pin_sources_incomplete_frees_walk_permit(pool: sqlx::PgPool) { + assert_marker_query_stall_frees_walk_permit( + pool, + MarkerQuery::Incomplete, + "203.0.113.84:5000".parse().unwrap(), + "pin_sources_incomplete", + ) + .await; + } } diff --git a/crates/gitlawb-node/src/api/issues.rs b/crates/gitlawb-node/src/api/issues.rs index 17acf9af..0eacfa72 100644 --- a/crates/gitlawb-node/src/api/issues.rs +++ b/crates/gitlawb-node/src/api/issues.rs @@ -61,11 +61,14 @@ pub async fn create_issue( let json_str = serde_json::to_string(&issue) .map_err(|e| AppError::BadRequest(format!("serialization error: {e}")))?; + // Shed 503 + Retry-After on an exhausted write-lock POOL instead of a generic + // git 500 (#173 F1). This path holds no admission permit, so it reaches the pool + // unthrottled; reuse the push handler's mapping so the two cannot drift. let guard = state .repo_store .acquire_write(&record.owner_did, &record.name) .await - .map_err(|e| AppError::Git(e.to_string()))?; + .map_err(|e| crate::api::repos::acquire_write_app_error(&e, &repo))?; let disk_path = guard.path().to_path_buf(); let create_result = git_issues::create_issue(&disk_path, &issue_id, &json_str); @@ -229,11 +232,13 @@ pub async fn close_issue( .await? .ok_or_else(|| AppError::RepoNotFound(format!("{owner}/{repo}")))?; + // Same capacity shed as create_issue above (#173 F1): an exhausted write-lock + // pool is a 503 + Retry-After, not a 500 git error. let guard = state .repo_store .acquire_write(&record.owner_did, &record.name) .await - .map_err(|e| AppError::Git(e.to_string()))?; + .map_err(|e| crate::api::repos::acquire_write_app_error(&e, &repo))?; let disk_path = guard.path().to_path_buf(); // Owner OR issue author may close. The author lives in the issue's git-JSON @@ -279,3 +284,168 @@ pub async fn close_issue( Ok(Json(issue)) } + +/// #173 F1 follow-up: the two issue write paths reach `acquire_write` holding NO +/// admission permit (unlike the push handler, which is capped by the git-push +/// semaphore), so they are the callers most likely to meet an exhausted write-lock +/// POOL under load. An exhausted pool is a capacity signal, so both must shed +/// 503 + Retry-After (`AppError::Overloaded`) the way the push handler does, not +/// report the generic 500 git error that says nothing about retrying. +#[cfg(test)] +mod lock_pool_shed_tests { + use super::*; + use axum::response::IntoResponse; + use sqlx::PgPool; + + fn seed_repo(owner_did: &str, name: &str) -> crate::db::RepoRecord { + let now = Utc::now(); + crate::db::RepoRecord { + id: Uuid::new_v4().to_string(), + name: name.to_string(), + owner_did: owner_did.to_string(), + description: None, + is_public: true, + default_branch: "main".to_string(), + created_at: now, + updated_at: now, + disk_path: format!("/tmp/{name}"), + forked_from: None, + machine_id: None, + } + } + + /// State whose repo store draws write locks from a ONE-connection pool with a + /// short checkout timeout, so a single held guard exhausts it promptly rather + /// than at the pool default. + async fn one_connection_lock_pool_state(pool: &PgPool) -> AppState { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.repo_store = crate::git::repo_store::RepoStore::new( + std::path::PathBuf::from("/tmp/gitlawb-issues-lockpool"), + None, + crate::git::repo_store::build_lock_pool(pool, 1, std::time::Duration::from_secs(1)), + ); + state + } + + /// The shed must be a real 503 carrying Retry-After, not just an internal enum + /// variant: assert on the rendered response so a remapping of `Overloaded` is + /// caught here too. + fn assert_sheds_503_with_retry_after(err: AppError, what: &str) { + let resp = err.into_response(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "{what}: an exhausted write-lock pool must shed 503, not a 500 git error" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .map(|v| v.to_str().unwrap()), + Some("1"), + "{what}: a capacity shed must tell the client when to retry" + ); + } + + /// RED-before/GREEN-after for `create_issue`. Both directions: the shed while the + /// only lock-pool connection is held by a guard on a DIFFERENT repo (so this is + /// pool capacity, not advisory-lock contention on this repo), and the must-not + /// case once that connection is back. + #[sqlx::test] + async fn create_issue_lock_pool_exhaustion_sheds_503_not_500(pool: PgPool) { + let owner = "did:key:zISSUECREATELOCKPOOLAAAAAAAAAAAAAAAAAAAA"; + let state = one_connection_lock_pool_state(&pool).await; + state + .db + .create_repo(&seed_repo(owner, "lp-create")) + .await + .expect("seed repo"); + + let held = state + .repo_store + .acquire_write(owner, "other-repo") + .await + .expect("the first write takes the only lock-pool connection"); + + let shed = create_issue( + State(state.clone()), + Extension(AuthenticatedDid(owner.to_string())), + Path((owner.to_string(), "lp-create".to_string())), + Json(CreateIssueRequest { + title: "t".to_string(), + body: None, + signed_payload: None, + }), + ) + .await; + let err = shed.expect_err("an exhausted lock pool must fail the call"); + assert_sheds_503_with_retry_after(err, "create_issue"); + + // MUST-NOT: with the pool free again the call is not shed as capacity (it + // fails later on the nonexistent on-disk repo, which is a git 500). + held.release(false).await; + let admitted = create_issue( + State(state.clone()), + Extension(AuthenticatedDid(owner.to_string())), + Path((owner.to_string(), "lp-create".to_string())), + Json(CreateIssueRequest { + title: "t".to_string(), + body: None, + signed_payload: None, + }), + ) + .await; + assert!( + !matches!(admitted, Err(AppError::Overloaded(_))), + "with the lock pool free, create_issue must not be shed as capacity; got {:?}", + admitted.err() + ); + } + + /// RED-before/GREEN-after for `close_issue`, same two directions. + #[sqlx::test] + async fn close_issue_lock_pool_exhaustion_sheds_503_not_500(pool: PgPool) { + let owner = "did:key:zISSUECLOSELOCKPOOLBBBBBBBBBBBBBBBBBBBBB"; + let state = one_connection_lock_pool_state(&pool).await; + state + .db + .create_repo(&seed_repo(owner, "lp-close")) + .await + .expect("seed repo"); + + let held = state + .repo_store + .acquire_write(owner, "other-repo") + .await + .expect("the first write takes the only lock-pool connection"); + + let shed = close_issue( + State(state.clone()), + Extension(AuthenticatedDid(owner.to_string())), + Path(( + owner.to_string(), + "lp-close".to_string(), + "deadbeef".to_string(), + )), + ) + .await; + let err = shed.expect_err("an exhausted lock pool must fail the call"); + assert_sheds_503_with_retry_after(err, "close_issue"); + + held.release(false).await; + let admitted = close_issue( + State(state.clone()), + Extension(AuthenticatedDid(owner.to_string())), + Path(( + owner.to_string(), + "lp-close".to_string(), + "deadbeef".to_string(), + )), + ) + .await; + assert!( + !matches!(admitted, Err(AppError::Overloaded(_))), + "with the lock pool free, close_issue must not be shed as capacity; got {:?}", + admitted.err() + ); + } +} diff --git a/crates/gitlawb-node/src/api/mod.rs b/crates/gitlawb-node/src/api/mod.rs index 71bfa43c..df10175a 100644 --- a/crates/gitlawb-node/src/api/mod.rs +++ b/crates/gitlawb-node/src/api/mod.rs @@ -208,11 +208,15 @@ mod authz_guard { (issues, "create_issue", "authorize_repo_read("), (bounties, "create_bounty", "authorize_repo_read("), (repos, "fork_repo", "authorize_repo_read("), - // get_by_cid gates each iterated repo row directly via visibility_check - // (KTD2a: it must NOT route through authorize_repo_read's fuzzy re-resolve). - (ipfs, "get_by_cid", "visibility_check("), - // #94 sibling read surfaces: gate private-repo metadata on read - // visibility (public repos stay anonymous; private repos 404). + // get_by_cid resolves each candidate (provenance path + legacy scan) through + // the shared `gate_and_serve`; the gate markers themselves are asserted + // below. This row proves the delegation is real, so the gate is actually + // reached rather than dead code. The delegated gate still calls + // `visibility_check` directly and never `authorize_repo_read`, so it keeps + // the property the pre-merge marker enforced: no fuzzy re-resolve. + (ipfs, "get_by_cid", "gate_and_serve("), + // Sibling read surfaces: gate private-repo metadata on read visibility + // (public repos stay anonymous; private repos 404). (replicas, "list_replicas", "authorize_repo_read("), (protect, "list_protected_branches", "authorize_repo_read("), (labels, "list_labels", "authorize_repo_read("), @@ -250,6 +254,22 @@ mod authz_guard { "visibility::require_owner must use did_matches for DID-safe owner matching" ); + // The CID read surface (#173) enforces its gate inside the shared + // `gate_and_serve`, which BOTH the provenance path and the legacy scan call, so + // the markers must live there (the get_by_cid row above only proves delegation). + // The repo's own "/" visibility check (KTD2a — never authorize_repo_read's fuzzy + // re-resolve) and the quarantine hard-drop BEFORE visibility (INV-11) are both + // load-bearing: removing either re-opens a leak on the provenance path. + let gate_body = fn_body(ipfs, "gate_and_serve"); + assert!( + gate_body.contains("visibility_check("), + "gate_and_serve must gate the CID read surface via visibility_check (KTD2a)" + ); + assert!( + gate_body.contains("if quarantined"), + "gate_and_serve must hard-drop a quarantined repo before the visibility gate (INV-11)" + ); + for (src, func, marker) in rows { let body = fn_body(src, func); assert!( diff --git a/crates/gitlawb-node/src/api/pulls.rs b/crates/gitlawb-node/src/api/pulls.rs index 26be6109..6255ef24 100644 --- a/crates/gitlawb-node/src/api/pulls.rs +++ b/crates/gitlawb-node/src/api/pulls.rs @@ -209,11 +209,14 @@ pub async fn merge_pr( return Err(AppError::BadRequest(format!("PR is already {}", pr.status))); } + // Shed 503 + Retry-After on an exhausted write-lock POOL instead of a generic + // git 500 (#173 F1). Merging holds no admission permit, so it reaches the pool + // unthrottled; reuse the push handler's mapping so the two cannot drift. let guard = state .repo_store .acquire_write(&record.owner_did, &record.name) .await - .map_err(|e| AppError::Git(e.to_string()))?; + .map_err(|e| crate::api::repos::acquire_write_app_error(&e, &name))?; let disk_path = guard.path().to_path_buf(); let merger_did = auth.0; let merge_result = store::merge_branch( @@ -424,3 +427,116 @@ pub async fn list_comments( let comments = state.db.list_pr_comments(&pr.id).await?; Ok(Json(serde_json::json!({ "comments": comments }))) } + +/// #173 F1 follow-up: `merge_pr` reaches `acquire_write` holding NO admission permit +/// (unlike the push handler, which is capped by the git-push semaphore), so it is one +/// of the callers most likely to meet an exhausted write-lock POOL under load. An +/// exhausted pool is a capacity signal, so the merge must shed 503 + Retry-After +/// (`AppError::Overloaded`) the way the push handler does, not report the generic +/// 500 git error that says nothing about retrying. +#[cfg(test)] +mod lock_pool_shed_tests { + use super::*; + use axum::response::IntoResponse; + use sqlx::PgPool; + + fn seed_repo(owner_did: &str, name: &str) -> crate::db::RepoRecord { + let now = Utc::now(); + crate::db::RepoRecord { + id: Uuid::new_v4().to_string(), + name: name.to_string(), + owner_did: owner_did.to_string(), + description: None, + is_public: true, + default_branch: "main".to_string(), + created_at: now, + updated_at: now, + disk_path: format!("/tmp/{name}"), + forked_from: None, + machine_id: None, + } + } + + /// RED-before/GREEN-after for `merge_pr`. Both directions: the shed while the only + /// lock-pool connection is held by a guard on a DIFFERENT repo (so this is pool + /// capacity, not advisory-lock contention on this repo), and the must-not case + /// once that connection is back. + #[sqlx::test] + async fn merge_pr_lock_pool_exhaustion_sheds_503_not_500(pool: PgPool) { + let owner = "did:key:zMERGELOCKPOOLOWNERAAAAAAAAAAAAAAAAAAAAA"; + let mut state = crate::test_support::test_state(pool.clone()).await; + // One lock-pool connection with a short checkout timeout, so a single held + // guard exhausts it promptly rather than at the pool default. + state.repo_store = crate::git::repo_store::RepoStore::new( + std::path::PathBuf::from("/tmp/gitlawb-pulls-lockpool"), + None, + crate::git::repo_store::build_lock_pool(&pool, 1, std::time::Duration::from_secs(1)), + ); + + let repo = seed_repo(owner, "lp-merge"); + let repo_id = repo.id.clone(); + state.db.create_repo(&repo).await.expect("seed repo"); + let now = Utc::now().to_rfc3339(); + state + .db + .create_pr(&PullRequest { + id: Uuid::new_v4().to_string(), + repo_id: repo_id.clone(), + number: 1, + title: "lp".to_string(), + body: None, + author_did: owner.to_string(), + source_branch: "feature".to_string(), + target_branch: "main".to_string(), + status: "open".to_string(), + merged_by_did: None, + merged_at: None, + created_at: now.clone(), + updated_at: now, + }) + .await + .expect("seed open PR"); + + let held = state + .repo_store + .acquire_write(owner, "other-repo") + .await + .expect("the first write takes the only lock-pool connection"); + + let shed = merge_pr( + State(state.clone()), + Extension(AuthenticatedDid(owner.to_string())), + Path((owner.to_string(), "lp-merge".to_string(), 1)), + ) + .await; + let err = shed.expect_err("an exhausted lock pool must fail the call"); + let resp = err.into_response(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "merge_pr: an exhausted write-lock pool must shed 503, not a 500 git error" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .map(|v| v.to_str().unwrap()), + Some("1"), + "merge_pr: a capacity shed must tell the client when to retry" + ); + + // MUST-NOT: with the pool free again the merge is not shed as capacity (it + // fails later on the nonexistent on-disk repo, which is a git 500). + held.release(false).await; + let admitted = merge_pr( + State(state.clone()), + Extension(AuthenticatedDid(owner.to_string())), + Path((owner.to_string(), "lp-merge".to_string(), 1)), + ) + .await; + assert!( + !matches!(admitted, Err(AppError::Overloaded(_))), + "with the lock pool free, merge_pr must not be shed as capacity; got {:?}", + admitted.err() + ); + } +} diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index b09cb6da..7b3676db 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -912,6 +912,226 @@ async fn run_encrypt_pin_task( } } +/// Test-only entry point: build an [`EncryptTaskCtx`] from a test `AppState` (with +/// an overridable `ipfs_api` for a mock Kubo server and an explicit `disk_path` for +/// the fixture repo) and run the real drain task. Keeps `EncryptTaskCtx` and +/// `run_encrypt_pin_task` private to this module. +/// +/// `owner_did` and `repo_name` must name the real seeded row: the drain re-fetches +/// the record by owner/name every lap, so a blank name resolves `Gone` and every +/// lap would pin nothing. +#[cfg(test)] +#[allow(clippy::too_many_arguments)] +pub(crate) async fn run_encrypt_pin_task_for_test( + state: &AppState, + guard: crate::state::EncryptInflightGuard, + disk_path: std::path::PathBuf, + repo_id: String, + owner_did: String, + repo_name: String, + ipfs_api: String, + snapshot_objects: Vec, + snapshot_rules: Option>, + snapshot_is_public: bool, +) { + let ctx = EncryptTaskCtx { + ipfs_api, + repo_path: disk_path, + db: state.db.clone(), + repo_id, + owner_did, + repo_name, + irys_url: String::new(), + http_client: std::sync::Arc::clone(&state.http_client), + node_did: state.node_did.to_string(), + node_keypair: std::sync::Arc::clone(&state.node_keypair), + git_bin: state.git_bin.clone(), + git_timeout: std::time::Duration::from_secs(state.config.git_service_timeout_secs), + encrypt_sem: state.git_encrypt_semaphore.clone(), + pin_sem: state.pin_semaphore.clone(), + }; + run_encrypt_pin_task( + ctx, + guard, + snapshot_objects, + snapshot_rules, + snapshot_is_public, + ) + .await; +} + +/// Test-only fault-injection seam for the drain re-reads in +/// `resolve_drain_object_list`. The behavior worth testing lives on the `Err` arm of +/// the two re-reads, which a real Postgres pool will not produce on demand, so the two +/// reads go through the wrappers below and consult this table first. Keyed by `repo_id` +/// (a fresh uuid per test) so tests running in parallel in one process cannot see each +/// other's injections, and it also records the ATTEMPT counts the retry-bound +/// assertions key on. +#[cfg(test)] +pub(crate) mod drain_faults { + use std::collections::HashMap; + use std::sync::{Mutex, OnceLock}; + + #[derive(Default, Clone, Copy, Debug)] + pub(crate) struct Counters { + pub(crate) repo_read_failures_left: usize, + pub(crate) rules_read_failures_left: usize, + pub(crate) repo_read_attempts: usize, + pub(crate) rules_read_attempts: usize, + } + + fn table() -> &'static Mutex> { + static TABLE: OnceLock>> = OnceLock::new(); + TABLE.get_or_init(|| Mutex::new(HashMap::new())) + } + + /// Make the next `repo_read_failures` repo re-reads and the next + /// `rules_read_failures` rule re-reads for `repo_id` return `Err`, then succeed. + pub(crate) fn inject(repo_id: &str, repo_read_failures: usize, rules_read_failures: usize) { + table().lock().unwrap().insert( + repo_id.to_string(), + Counters { + repo_read_failures_left: repo_read_failures, + rules_read_failures_left: rules_read_failures, + ..Default::default() + }, + ); + } + + /// Observed attempt counts (and remaining injections) for `repo_id`. + pub(crate) fn counters(repo_id: &str) -> Counters { + table() + .lock() + .unwrap() + .get(repo_id) + .copied() + .unwrap_or_default() + } + + /// Production-path hook: count one repo re-read attempt, return whether it must fail. + pub(crate) fn take_repo_read(repo_id: &str) -> bool { + let mut map = table().lock().unwrap(); + let c = map.entry(repo_id.to_string()).or_default(); + c.repo_read_attempts += 1; + if c.repo_read_failures_left > 0 { + c.repo_read_failures_left -= 1; + return true; + } + false + } + + /// Production-path hook: count one rules re-read attempt, return whether it must fail. + pub(crate) fn take_rules_read(repo_id: &str) -> bool { + let mut map = table().lock().unwrap(); + let c = map.entry(repo_id.to_string()).or_default(); + c.rules_read_attempts += 1; + if c.rules_read_failures_left > 0 { + c.rules_read_failures_left -= 1; + return true; + } + false + } +} + +/// The drain's repo re-read, behind the test-only fault seam above. +async fn drain_get_repo(ctx: &EncryptTaskCtx) -> anyhow::Result> { + #[cfg(test)] + if drain_faults::take_repo_read(&ctx.repo_id) { + return Err(anyhow::anyhow!("injected repo re-read failure")); + } + ctx.db.get_repo(&ctx.owner_did, &ctx.repo_name).await +} + +/// The drain's visibility-rule re-read, behind the test-only fault seam above. +async fn drain_list_rules( + ctx: &EncryptTaskCtx, + record_id: &str, +) -> anyhow::Result> { + #[cfg(test)] + if drain_faults::take_rules_read(&ctx.repo_id) { + return Err(anyhow::anyhow!("injected visibility-rule re-read failure")); + } + ctx.db.list_visibility_rules(record_id).await +} + +/// Attempts allowed for the drain re-read before the lap gives up. The coalesced +/// push's work is already out of the pending slot (`finish_or_take_pending` took it +/// in the same critical section that kept the key), and there is no reconciliation +/// sweep to re-derive it: a transient read error must be RETRIED here or that push's +/// pin/encrypt pass is gone. The bound keeps a sustained outage from spinning +/// forever; on exhaustion the work is still lost (the pre-existing residual), but +/// the give-up is logged at ERROR so it is observable instead of silent. +const DRAIN_REREAD_MAX_ATTEMPTS: usize = 3; + +/// Backoff before the next re-read attempt. Doubles per attempt. +const DRAIN_REREAD_BACKOFF: std::time::Duration = std::time::Duration::from_millis(50); + +/// The outcome of the drain's fresh state re-read, keeping the three cases distinct +/// that a single `Err => None` collapses into one: a usable refresh, a repo that +/// genuinely no longer exists (terminal, and NOT a retry), and a transient read +/// failure (retryable). +enum DrainRefresh { + State { + /// Boxed only to keep the enum small: `RepoRecord` dwarfs the other two + /// variants, which carry nothing (clippy::large_enum_variant). + record: Box, + rules: Vec, + }, + Gone, + Failed, +} + +/// Re-read repo state for a drain lap, retrying transient read errors. +/// +/// Both reads are retryable and neither may be read as an absence: an `Err` from the +/// repo row is not "the repo is gone", and an `Err` from the rule list is not "this +/// repo has no rules" (`.ok()` made those indistinguishable, and a `None` rule set +/// makes `replication_withheld_set` return `None`, which skips the entire lap). Only +/// `Ok(None)` on the repo row is a terminal absence, and it consumes no retry budget. +/// +/// The whole `RepoRecord` comes back, not just its rules and flags: the caller writes +/// against `record.id` from this FRESH re-fetch, never `ctx.repo_id` frozen at spawn. +async fn drain_refresh_state(ctx: &EncryptTaskCtx) -> DrainRefresh { + let mut backoff = DRAIN_REREAD_BACKOFF; + for attempt in 1..=DRAIN_REREAD_MAX_ATTEMPTS { + let record = match drain_get_repo(ctx).await { + Ok(Some(rec)) => Box::new(rec), + Ok(None) => return DrainRefresh::Gone, + Err(e) => { + tracing::warn!( + repo = %ctx.repo_id, err = %e, attempt, + "coalesced drain: repo re-read failed; retrying" + ); + tokio::time::sleep(backoff).await; + backoff *= 2; + continue; + } + }; + // record.id, never the spawn-time ctx.repo_id: the record above is re-fetched + // fresh by owner/name, and a delete+re-create between spawn and drain gives + // the row a NEW id - rules read against the stale id come back empty and + // would fail open for the new row. + match drain_list_rules(ctx, &record.id).await { + Ok(rules) => return DrainRefresh::State { record, rules }, + Err(e) => { + tracing::warn!( + repo = %ctx.repo_id, err = %e, attempt, + "coalesced drain: visibility-rule re-read failed; retrying" + ); + tokio::time::sleep(backoff).await; + backoff *= 2; + } + } + } + tracing::error!( + repo = %ctx.repo_id, + attempts = DRAIN_REREAD_MAX_ATTEMPTS, + "coalesced drain: re-read failed on every attempt; the coalesced push's \ + pin/encrypt pass is dropped (no reconciliation sweep re-derives it)" + ); + DrainRefresh::Failed +} + /// Resolve a coalesced-drain iteration's replicable object list. Re-fetches the /// repo record and visibility rules FRESH — rules tightened between the coalesced /// push and its drain must be honored, fail closed: a newly-withheld blob is not @@ -932,29 +1152,27 @@ async fn resolve_drain_object_list( Option>, bool, )> { - let record = match ctx.db.get_repo(&ctx.owner_did, &ctx.repo_name).await { - Ok(Some(r)) => r, - Ok(None) => { + // Both re-reads are bounded-retried: a transient blip must not discard the + // coalesced push's work (`finish_or_take_pending` already took it out of the + // pending slot and no sweep re-derives it). The rules come back from the same + // refresh, read against the FRESH record.id, never the spawn-time ctx.repo_id. + let (record, rules_opt) = match drain_refresh_state(ctx).await { + DrainRefresh::State { record, rules } => (*record, Some(rules)), + DrainRefresh::Gone => { tracing::warn!( repo = %ctx.repo_id, "coalesced drain: repo record is gone; dropping the pending work" ); return None; } - Err(e) => { + DrainRefresh::Failed => { tracing::warn!( repo = %ctx.repo_id, - err = %e, "coalesced drain: repo re-fetch failed; pinning nothing (fail closed)" ); return None; } }; - // record.id, never the spawn-time ctx.repo_id: the record above is re-fetched - // fresh by owner/name, and a delete+re-create between spawn and drain gives - // the row a NEW id — rules read against the stale id come back empty and - // would fail open for the new row. - let rules_opt = ctx.db.list_visibility_rules(&record.id).await.ok(); let (_announce, withheld) = replication_withheld_set( ctx.encrypt_sem.clone(), rules_opt.clone(), @@ -1141,12 +1359,27 @@ async fn pinata_object_list_for_refs( /// retained list memory is not bounded by this pool. Bounding that is a real change to /// the capture shape and is deliberately not attempted here; the Pinata twin below /// avoids it by acquiring BEFORE it derives its list. +// Nine because the merge of #173 and #174 landed both sets of arguments on one +// signature: #174's pin-admission permit plus #173's git seam and pin provenance. +// Each is a distinct value the pin loop needs and none is derivable from another, so +// a wrapper struct here would only rename the same nine fields. +// +// `batch_budget` is the ninth and is passed rather than read from the constant so the +// permit-release regression can drive a short budget. Hardcoding `PIN_BATCH_BUDGET` +// here made that test cost 120s of wall clock, and the only cheaper shapes were +// vacuous: a lock released before the budget returns at the same time whether or not +// the loop is bounded, which proves nothing. Production passes the constant. +#[allow(clippy::too_many_arguments)] async fn pin_new_objects_gated( pin_sem: &Arc, ipfs_api: &str, repo_path: &std::path::Path, + git_bin: &str, + git_timeout: std::time::Duration, object_list: Vec, db: &Arc, + repo_id: &str, + batch_budget: std::time::Duration, ) -> Vec<(String, String)> { // Nothing to pin: answer before taking a permit (#174 F2b). The permit bounds how // many pin loops run concurrently, and an empty list does no pinning, so parking @@ -1164,12 +1397,12 @@ async fn pin_new_objects_gated( crate::ipfs_pin::pin_new_objects( ipfs_api, repo_path, - // The literal, not `state.git_bin`: tests point that at a fake walk git, and - // this is the same choice `api/ipfs`'s bounded call sites already document. - "git", + git_bin, + git_timeout, object_list, db, - crate::ipfs_pin::PIN_BATCH_BUDGET, + repo_id, + batch_budget, ) .await } @@ -1189,8 +1422,19 @@ async fn pin_and_encrypt_objects( &ctx.pin_sem, &ctx.ipfs_api, &ctx.repo_path, + // The literal, not `ctx.git_bin`: that knob is the WALK binary, and tests point + // it at a fake that answers `rev-list` and friends. Routing the pin path's + // `cat-file` read through it would ask that fake to impersonate cat-file too. + // Same choice, for the same reason, as `api/ipfs`'s bounded call sites. The + // parameter stays so the seam is still drivable from a test. + "git", + ctx.git_timeout, object_list, &ctx.db, + // The drain's repo id, never `ctx.repo_id` frozen at spawn (#174 U3): pin + // provenance must name the row the reader will resolve against. + repo_id, + crate::ipfs_pin::PIN_BATCH_BUDGET, ) .await; if !pinned.is_empty() { @@ -1269,6 +1513,27 @@ async fn pin_and_encrypt_objects( } } +/// Map an `acquire_write` failure to the right `AppError`. An exhausted repo write-lock +/// POOL is a capacity signal, not a broken repo, so it sheds 503 + Retry-After the same +/// way the admission caps around it do; it used to fall into the generic git 500, which +/// tells the client nothing about retrying (#173 F1). Anything else stays a git error. +/// +/// Shared with the non-push `acquire_write` callers (`api/issues.rs`, `api/pulls.rs`) +/// rather than copied: those hold no admission permit, so they meet an exhausted pool +/// first, and a second copy of this mapping would be free to drift from the push path. +pub(crate) fn acquire_write_app_error(err: &anyhow::Error, repo: &str) -> AppError { + if err + .downcast_ref::() + .is_some() + { + tracing::warn!(repo = %repo, err = %err, "write-lock pool exhausted; shedding with 503"); + AppError::Overloaded("git write locks at capacity, retry shortly".into()) + } else { + tracing::error!(repo = %repo, err = %err, "acquire_write failed"); + AppError::Git(err.to_string()) + } +} + /// Map an error from a `smart_http` git service call to the right `AppError`: /// [`smart_http::GitServiceTimeout`] to 504, a malformed client request to 400, /// anything else to a 500 git error. Pure (no logging) so it is unit-testable; @@ -1925,7 +2190,10 @@ pub async fn git_receive_pack( // exhausted Postgres pool (so the 60-count never advances) — and the write permit // is held the whole time, draining the pool (#174 P1-2). The outer // `tokio::time::timeout` cancels a mid-sleep/mid-`fetch_one` future, so it bounds - // both the loop and a hung iteration without any repo_store.rs change (KTD3). The + // both the loop and a hung iteration. Cancelling here is only safe because + // acquire_write holds its advisory lock on a connection from a pool whose + // `after_release` hook unlocks (#173): the dropped future used to leave the lock + // held with no guard alive to release it, wedging later pushes to that repo. The // permit is a handler local here (moved into the AdmissionGuard only after this), // so the early return on timeout drops it and frees the slot; shed a bounded 503. let acquire_deadline = std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); @@ -1940,10 +2208,7 @@ pub async fn git_receive_pack( tracing::warn!(repo = %name, "acquire_write timed out; shedding with 503"); AppError::Overloaded("git service acquisition timed out, retry shortly".into()) })? - .map_err(|e| { - tracing::error!(repo = %name, err = %e, "acquire_write failed"); - AppError::Git(e.to_string()) - })?; + .map_err(|e| acquire_write_app_error(&e, name))?; let disk_path = guard.path().to_path_buf(); tracing::debug!(repo = %name, path = %disk_path.display(), "running git receive-pack"); let body_len = body.len(); @@ -1953,13 +2218,30 @@ pub async fn git_receive_pack( // instant a disconnect drops this future while the detached reaper runs (#174 P1-a). // The handler keeps no copy. This is independent of the write-lock `guard.release` // below: admission tracks the git process lifetime, the write lock tracks the repo. + // + // The WRITE LOCK rides the same seam (#173 F2). `guard.release(..)` below is only + // reached if `receive_pack` returns, so on a client disconnect the guard would drop + // with the future and the lock pool's `after_release` hook would free the advisory + // lock immediately, while `KillGroupOnDrop`'s detached reaper is still giving the + // group its ~2s SIGTERM grace. A second push admitted in that window puts two + // `git receive-pack` groups on one repo, which is exactly what the timeout path + // reaps to prevent ("a caller releasing a write lock can't race them"). Sharing the + // guard rather than moving it outright is what lets the SUCCESS path still reclaim + // it for the Tigris upload: the copy retained here can only DELAY release, never + // perform it early, because the handler reaches the take below only after + // `receive_pack` has returned (group reaped or disarmed). On the disconnect path + // this copy dies with the future and the reaper's copy is last, so the lock frees + // after the reap with no upload, which is the release(success = false) semantics an + // interrupted push must have. + let guard = std::sync::Arc::new(std::sync::Mutex::new(Some(guard))); // Clone (a) of the write lease rides this AdmissionGuard: on a client disconnect the // guard moves into KillGroupOnDrop's detached reaper, so the lease frees only after // the receive-pack group is reaped — NOT at the disconnect instant (which is exactly // when RepoWriteGuard::Drop frees the pg lock). Tying the lease to RepoWriteGuard // instead would drop it at disconnect and reopen the F3 race. - let admission = - smart_http::AdmissionGuard::new(_permit, _caller_permit).with_lease(lease.clone()); + let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit) + .with_hold(std::sync::Arc::clone(&guard)) + .with_lease(lease.clone()); let receive_result = smart_http::receive_pack( &state.git_bin, &disk_path, @@ -2025,12 +2307,21 @@ pub async fn git_receive_pack( // Always release the advisory lock — even on error — to prevent stale locks // from blocking subsequent pushes. Only upload to Tigris when the push // succeeded; uploading a half-applied repo would propagate corruption. - guard.release(push_succeeded).await; + // Reclaim the write lock from the shared cell (#173 F2). This is only reachable + // once `receive_pack` has returned, so the admission guard's copy can only ever + // DELAY release, never perform it early; on the disconnect path this line is not + // reached at all and the reaper's copy is last. + let reclaimed = guard + .lock() + .expect("repo write-lock mutex poisoned") + .take() + .expect("the write lock is only taken here, and only once"); + reclaimed.release(push_succeeded).await; // Clean path: clone (a) already dropped inside run_git_service when the receive-pack // group was reaped; clone (b) held here spanned the success-only Tigris upload that // ran inside release() above. Drop it now so a second same-repo push proceeds the // moment this write is durable, rather than at end of the (longer) handler tail. On - // the disconnect path this line is never reached — clone (a) rides the reaper (F3). + // the disconnect path this line is never reached: clone (a) rides the reaper (F3). drop(lease); let result = receive_result.map_err(|e| { @@ -2343,6 +2634,7 @@ async fn post_receive_replication_tail( let pinata_upload_url = state.config.pinata_upload_url.clone(); let repo_path_clone = disk_path.clone(); let db_clone = state.db.clone(); + let repo_id = record.id.clone(); let http_client = Arc::clone(&state.http_client); let node_did_str = state.node_did.to_string(); let repo_slug = format!( @@ -2431,8 +2723,10 @@ async fn post_receive_replication_tail( // The literal, not `state.git_bin`: tests point that at a fake // walk git, and this read must run the real one. "git", + pinata_git_timeout, object_list, &db_clone, + &repo_id, crate::ipfs_pin::PIN_BATCH_BUDGET, ) .await, @@ -5852,6 +6146,415 @@ mod tests { ); } + #[cfg(unix)] + fn pid_alive(pid: i32) -> bool { + // SAFETY: kill(2) with signal 0 only probes; it takes integers and borrows no + // Rust memory. + unsafe { libc::kill(pid, 0) == 0 } + } + + /// SIGKILL the recorded pids if the test unwinds, so a RED run leaks no orphan. + #[cfg(unix)] + struct KillOnPanic(Vec); + #[cfg(unix)] + impl Drop for KillOnPanic { + fn drop(&mut self) { + for pid in &self.0 { + // SAFETY: as above. + unsafe { + libc::kill(*pid, libc::SIGKILL); + } + } + } + } + + /// Is the repo write lock takeable from an INDEPENDENT session right now? Session + /// advisory locks are re-entrant within their own session, so this must not run on + /// any connection the code under test might be using. + #[cfg(unix)] + async fn write_lock_is_takeable(pool: &sqlx::PgPool, key: i64) -> bool { + let mut probe = pool.acquire().await.expect("probe connection"); + let taken: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut *probe) + .await + .expect("probe try-lock"); + if taken.0 { + sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *probe) + .await + .expect("probe unlock"); + } + taken.0 + } + + /// #173 F2 (RED-before/GREEN-after): on a CLIENT DISCONNECT the repo write lock must + /// stay held until the receive-pack process group is confirmed reaped. + /// + /// The handler's `guard.release(..)` line is only reached if `receive_pack` returns. + /// When the request future is dropped mid-push the guard drops instead, and (since + /// #173 U1 gave the lock pool an `after_release` hook) that FREES the advisory lock + /// immediately, while `KillGroupOnDrop`'s detached reaper is still giving the group + /// its ~2s SIGTERM grace. A second `acquire_write` admitted inside that window puts + /// two `git receive-pack` groups on one repo. `smart_http.rs` states the invariant + /// the other way round on the timeout path: "a caller releasing a write lock can't + /// race them". + /// + /// Real seam, not a stand-in: the production `git_receive_pack` handler, a fake git + /// whose descendant IGNORES SIGTERM (so the group genuinely survives the grace and + /// the window is ~2s wide, not a scheduling artifact), and the lock probed from an + /// independent session. RED before the fix: the lock is takeable while the group is + /// still alive. GREEN after: takeable only once the group is gone. + #[cfg(unix)] + #[sqlx::test] + async fn receive_pack_disconnect_holds_the_write_lock_until_the_group_is_reaped( + pool: sqlx::PgPool, + ) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + + let owner = "z6disc"; + let name = "dc1"; + let repos_dir = tempfile::TempDir::new().unwrap(); + let tmp = tempfile::TempDir::new().unwrap(); + let descfile = tmp.path().join("desc.pid"); + // The leader dies on the group SIGTERM; its descendant traps SIGTERM and loops + // (bounded at ~30s so a RED run leaks nothing permanent), so the group is only + // gone once the reaper escalates to SIGKILL. The descendant inherits the stdout + // pipe, which keeps drive_git_child's read_to_end pending until we drop. + let body = format!( + "#!/bin/sh\n\ + sh -c 'trap \"\" TERM; echo $$ > \"{}\"; i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done' &\n\ + wait\n", + descfile.display() + ); + let git_bin = write_fake_git(tmp.path(), &body); + + let mut state = crate::test_support::test_state(pool.clone()).await; + state.git_bin = git_bin; + state.repo_store = crate::git::repo_store::RepoStore::new( + repos_dir.path().to_path_buf(), + None, + crate::git::repo_store::build_lock_pool(&pool, 4, std::time::Duration::from_secs(5)), + ); + let mut cfg = (*state.config).clone(); + // Long enough that the git-service timeout is never what ends this push; the + // disconnect is. + cfg.git_service_timeout_secs = 600; + state.config = std::sync::Arc::new(cfg); + state + .db + .upsert_mirror_repo(owner, name, "/tmp/z6disc-dc1", None, false) + .await + .unwrap(); + + // The mirror row stores the short owner as owner_did, so the slug is the owner. + // Import the production derivation rather than hand-copying it: a local copy + // silently diverged when the key moved to SHA-256 (#210). + use crate::git::repo_store::advisory_lock_key; + let key = advisory_lock_key(&owner.replace([':', '/'], "_"), name); + // Probe from a pool that is NOT the store's lock pool and NOT the harness pool. + let probe = sqlx::postgres::PgPoolOptions::new() + .max_connections(2) + .connect_lazy_with((*pool.connect_options()).clone()); + + assert!( + write_lock_is_takeable(&probe, key).await, + "the write lock must be free before the push" + ); + + // Drive the handler a slice at a time until the fake git's SIGTERM-ignoring + // descendant records its pid, i.e. receive-pack is genuinely running under the + // write lock. `Ok(_)` means the handler returned early; stop polling then, since + // re-polling a completed future panics. + // + // Retried on a miss for the same reason `smart_http`'s disconnect tests retry: + // under `cargo test` fork-storm load a freshly written fake `git` can transiently + // fail to exec (ETXTBSY, a concurrent worker forked while its write fd was open), + // which leaves no pid. A losing attempt's future is dropped, which reaps whatever + // spawned and releases its write lock, so retries do not leak. The winning + // attempt's future is kept PENDING: dropping it below is the disconnect under test. + const SPAWN_ATTEMPTS: u64 = 12; + let (fut, desc) = { + let mut attempt = 0u64; + loop { + attempt += 1; + let _ = std::fs::remove_file(&descfile); + let mut fut = Box::pin(git_receive_pack( + State(state.clone()), + Path((owner.to_string(), name.to_string())), + Extension(crate::auth::AuthenticatedDid( + "did:key:z6MkDisconnectWriteLockProofDidAAAAAAAA".to_string(), + )), + crate::rate_limit::PeerAddr(Some( + "203.0.113.81:5000".parse::().unwrap(), + )), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + )); + let mut found: Option = None; + for _ in 0..500 { + let finished = + tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut) + .await + .is_ok(); + if let Some(p) = std::fs::read_to_string(&descfile) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + found = Some(p); + break; + } + if finished { + break; + } + } + match found { + Some(p) => break (fut, p), + None => { + drop(fut); + assert!( + attempt < SPAWN_ATTEMPTS, + "the push never reached receive-pack after {SPAWN_ATTEMPTS} \ + attempts (persistent failure, not a transient runner miss)" + ); + tokio::time::sleep(std::time::Duration::from_millis(100 * attempt)).await; + } + } + } + }; + let _cleanup = KillOnPanic(vec![desc]); + assert!( + pid_alive(desc), + "the receive-pack group must be running before the disconnect" + ); + assert!( + !write_lock_is_takeable(&probe, key).await, + "the write lock must be held while receive-pack runs" + ); + + // Client disconnect: drop the request future mid-receive-pack. + drop(fut); + + let mut takeable_while_group_alive = false; + let mut freed_after_reap = false; + for _ in 0..800 { + let takeable = write_lock_is_takeable(&probe, key).await; + let group_alive = pid_alive(desc); + if takeable && group_alive { + takeable_while_group_alive = true; + } + if takeable && !group_alive { + freed_after_reap = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + // Clean up regardless so a RED run leaves no orphan behind. + // SAFETY: kill(2) takes integers only. + unsafe { + libc::kill(desc, libc::SIGKILL); + } + assert!( + !takeable_while_group_alive, + "the repo write lock was takeable while a receive-pack group was still alive \ + on that repo: a second push can enter and two git receive-pack groups run \ + against one repo (#173 F2)" + ); + assert!( + freed_after_reap, + "the write lock must be released once the disconnected push's group is reaped" + ); + // The other half of the disconnect invariant, and the reason the guard rides the + // reaper rather than being released there: an interrupted push must not publish a + // half-applied repo. The guard is gone by now (the lock above only frees when it + // is), so the upload site has had its whole chance to be reached. The positive + // control is `receive_pack_success_reclaims_and_releases_the_write_lock`, which + // observes the same counter at 1: without it, a zero here would pass on any build + // where an upload is simply impossible. + assert_eq!( + state.repo_store.tigris_upload_site_reached(), + 0, + "a push interrupted by a client disconnect must not reach the Tigris upload \ + site: publishing a half-applied repo propagates it to every node that later \ + downloads it (#173 F2)" + ); + } + + /// #173 F2, the other half: carrying the write lock through the admission seam must + /// NOT cost the success path its `release(true)`. A push that completes normally has + /// to reclaim the lock and release it explicitly (that is what performs the Tigris + /// upload), synchronously, not leave it to the pool's `after_release` net. The lock + /// is probed immediately after the handler returns, with no polling, so a fix that + /// only ever dropped the guard would fail here. + #[cfg(unix)] + #[sqlx::test] + async fn receive_pack_success_reclaims_and_releases_the_write_lock(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + + let owner = "z6succ"; + let name = "sc1"; + let repos_dir = tempfile::TempDir::new().unwrap(); + let tmp = tempfile::TempDir::new().unwrap(); + // A receive-pack that succeeds. It DRAINS stdin first: exiting while the handler + // is still writing the request body would EPIPE that write, which + // `drive_git_child` surfaces as an error after a successful exit status, making + // the push fail for a reason that has nothing to do with the lock under test. + let git_bin = write_fake_git(tmp.path(), "#!/bin/sh\ncat >/dev/null\nexit 0\n"); + + let mut state = crate::test_support::test_state(pool.clone()).await; + state.git_bin = git_bin; + state.repo_store = crate::git::repo_store::RepoStore::new( + repos_dir.path().to_path_buf(), + None, + crate::git::repo_store::build_lock_pool(&pool, 4, std::time::Duration::from_secs(5)), + ); + state + .db + .upsert_mirror_repo(owner, name, "/tmp/z6succ-sc1", None, false) + .await + .unwrap(); + + use crate::git::repo_store::advisory_lock_key; + let key = advisory_lock_key(&owner.replace([':', '/'], "_"), name); + let probe = sqlx::postgres::PgPoolOptions::new() + .max_connections(2) + .connect_lazy_with((*pool.connect_options()).clone()); + + // Retried ONLY on the ETXTBSY exec race a freshly written fake `git` hits under + // fork-storm load (a concurrent test worker forked while its write fd was open). + // Narrow on purpose: any other failure still fails the assertion below loudly. + const SPAWN_ATTEMPTS: u64 = 12; + let mut result = None; + for attempt in 1..=SPAWN_ATTEMPTS { + let outcome = tokio::time::timeout( + std::time::Duration::from_secs(30), + git_receive_pack( + State(state.clone()), + Path((owner.to_string(), name.to_string())), + Extension(crate::auth::AuthenticatedDid( + "did:key:z6MkPushSuccessReleaseProofDidAAAAAAAA".to_string(), + )), + crate::rate_limit::PeerAddr(Some( + "203.0.113.83:5000".parse::().unwrap(), + )), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ), + ) + .await + .expect("the push must return"); + let exec_race = + matches!(&outcome, Err(AppError::Git(m)) if m.contains("Text file busy")); + if exec_race && attempt < SPAWN_ATTEMPTS { + tokio::time::sleep(std::time::Duration::from_millis(100 * attempt)).await; + continue; + } + result = Some(outcome); + break; + } + let result = result.expect("one attempt must have produced an outcome"); + assert!( + result.is_ok(), + "the fake receive-pack succeeds, so the handler must too; got {result:?}" + ); + + // No polling: `release` unlocks on the connection that took the lock, so the + // lock is free the instant the handler returns. Falling back to the async + // `after_release` net would not satisfy this. + assert!( + write_lock_is_takeable(&probe, key).await, + "a completed push must reclaim its write lock and release it synchronously" + ); + // POSITIVE CONTROL for the disconnect case's "no upload" assertion. A push that + // completed does reach the Tigris upload site, exactly once, so the zero the + // disconnect test observes is a real difference between the two paths rather than + // an artifact of tests running with no Tigris client configured. Exactly once, + // not at least once: a retried exec race releases with success = false and must + // not count. + assert_eq!( + state.repo_store.tigris_upload_site_reached(), + 1, + "a completed push must reach the Tigris upload site once" + ); + } + + /// #173 F1 (RED-before/GREEN-after): an exhausted repo write-lock POOL is a capacity + /// signal, so the push must shed 503 + Retry-After (Overloaded) like every other + /// admission path here, not report a 500 git error. Both directions: the shed with + /// the single lock-pool connection occupied by a guard on a DIFFERENT repo (so this + /// is pool capacity, not advisory-lock contention), and the must-not case once that + /// connection is back. Before the fix `acquire_write`'s checkout failure fell into + /// the generic `AppError::Git` arm (500, no Retry-After). + #[sqlx::test] + async fn receive_pack_lock_pool_exhaustion_sheds_503_not_500(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + + let owner = "z6lockpool"; + let name = "lp1"; + let mut state = crate::test_support::test_state(pool.clone()).await; + // One lock-pool connection, short checkout timeout so the exhaustion surfaces + // promptly rather than at the handler's own acquire deadline. + state.repo_store = crate::git::repo_store::RepoStore::new( + std::path::PathBuf::from("/tmp/gitlawb-lockpool-shed"), + None, + crate::git::repo_store::build_lock_pool(&pool, 1, std::time::Duration::from_secs(1)), + ); + state + .db + .upsert_mirror_repo(owner, name, "/tmp/z6lockpool-lp1", None, false) + .await + .unwrap(); + + let did = "did:key:z6MkLockPoolShedProofDidAAAAAAAAAAAAAAAAAA"; + let peer: SocketAddr = "203.0.113.71:5000".parse().unwrap(); + + // Occupy the only lock-pool connection with a write on an UNRELATED repo. + let held = state + .repo_store + .acquire_write(owner, "other-repo") + .await + .expect("the first write takes the only lock-pool connection"); + + let shed = git_receive_pack( + State(state.clone()), + Path((owner.to_string(), name.to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(peer)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await; + assert!( + matches!(shed, Err(AppError::Overloaded(_))), + "an exhausted lock pool must shed 503 + Retry-After, not a 500 git error; \ + got {shed:?}" + ); + + // MUST-NOT: with the pool free again, the push is not shed as capacity (it fails + // later on the nonexistent on-disk repo, which is a git error, not Overloaded). + held.release(false).await; + let admitted = git_receive_pack( + State(state.clone()), + Path((owner.to_string(), name.to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some("203.0.113.72:5000".parse().unwrap())), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await; + assert!( + !matches!(admitted, Err(AppError::Overloaded(_))), + "with the lock pool free, a push must not be shed as capacity; got {admitted:?}" + ); + } + /// #174 U5 (P1-e, RED-before/GREEN-after): the post-push encryption walk acquires a /// `git_encrypt_semaphore` permit before running, so completed pushes cannot spawn /// unbounded concurrent full-history walks. With the pool exhausted the gated walk @@ -6176,7 +6879,17 @@ mod tests { let objects = vec!["0123456789abcdef0123456789abcdef01234567".to_string()]; let blocked = tokio::time::timeout( std::time::Duration::from_millis(500), - pin_new_objects_gated(&pin_sem, "", tmp.path(), objects.clone(), &db), + pin_new_objects_gated( + &pin_sem, + "", + tmp.path(), + "git", + std::time::Duration::from_secs(30), + objects.clone(), + &db, + "repo-gated-a", + crate::ipfs_pin::PIN_BATCH_BUDGET, + ), ) .await; assert!( @@ -6188,13 +6901,106 @@ mod tests { drop(held); let out = tokio::time::timeout( std::time::Duration::from_secs(5), - pin_new_objects_gated(&pin_sem, "", tmp.path(), objects, &db), + pin_new_objects_gated( + &pin_sem, + "", + tmp.path(), + "git", + std::time::Duration::from_secs(30), + objects, + &db, + "repo-gated-b", + crate::ipfs_pin::PIN_BATCH_BUDGET, + ), ) .await .expect("the pin loop completes once admission frees"); assert!(out.is_empty(), "an empty ipfs_api pins nothing"); } + /// #173 F3, at the layer that actually owns the permit. `pin_new_objects_gated` + /// holds the global `pin_semaphore` across the whole `ipfs_pin::pin_new_objects` + /// call, so a DB call inside that loop with no deadline parked a global pin slot + /// for as long as the query was stuck; once every slot was so held, post-push + /// replication stopped for every repo on the node. With the loop's DB calls + /// bounded by the batch deadline the call returns at ~the batch budget and the + /// permit comes back, even though the table is still locked. + /// + /// The endpoint is a LIVE mockito server, not the `""` the sibling test above + /// uses. `ipfs_pin::pin_new_objects` returns `vec![]` immediately on an empty + /// `ipfs_api`, so an empty-string copy would never reach `is_pinned`, never touch + /// the locked table, and pass identically with the bound deleted. The mock is at + /// `.expect(0)` because a stalled pinned-status check must not fall through to an + /// add. + /// + /// The budget is passed in (1500ms) rather than read from `PIN_BATCH_BUDGET`. + /// Before that seam existed this test cost 120s of wall clock, because the bound + /// it asserts IS the batch budget and the gate hardcoded the production constant. + /// 1500ms is deliberately above `PIN_READ_FLOOR` (1100ms): below the floor + /// `batch_budget_gate` breaks the loop as its first statement, so the run would + /// never reach a DB call and would pass with the bound deleted. + #[sqlx::test] + async fn pin_new_objects_gated_frees_permit_after_stalled_db(pool: sqlx::PgPool) { + use std::sync::Arc; + use tokio::sync::Semaphore; + + let state = crate::test_support::test_state(pool.clone()).await; + let db = state.db.clone(); + let tmp = tempfile::TempDir::new().unwrap(); + let pin_sem = Arc::new(Semaphore::new(1)); + + let mut server = mockito::Server::new_async().await; + let add = server + .mock("POST", mockito::Matcher::Any) + .with_status(200) + .with_body(r#"{"Hash":"QmShouldNotHappen"}"#) + .expect(0) + .create_async() + .await; + + // `ACCESS EXCLUSIVE` conflicts with the `ACCESS SHARE` every SELECT needs, so + // `is_pinned` blocks at lock acquisition regardless of row count. Held for the + // whole call: a lock released early would let the pre-fix bare await finish too, + // and the test would prove nothing. + let mut lock = pool.acquire().await.unwrap(); + sqlx::raw_sql("BEGIN; LOCK TABLE pinned_cids IN ACCESS EXCLUSIVE MODE;") + .execute(&mut *lock) + .await + .unwrap(); + + let objects = vec!["0123456789abcdef0123456789abcdef01234567".to_string()]; + let out = tokio::time::timeout( + std::time::Duration::from_secs(150), + pin_new_objects_gated( + &pin_sem, + &server.url(), + tmp.path(), + "git", + std::time::Duration::from_secs(30), + objects, + &db, + "repo-gated-stalled-db", + std::time::Duration::from_millis(1500), + ), + ) + .await + .expect( + "a stalled DB must cost the pin loop its batch budget, not the lock's \ + lifetime: an unbounded await inside the loop holds this permit past every \ + budget", + ); + + assert!(out.is_empty(), "a stalled pinned-status check pins nothing"); + assert_eq!( + pin_sem.available_permits(), + 1, + "the global pin permit must be back once the bounded loop returns" + ); + add.assert_async().await; + + sqlx::raw_sql("ROLLBACK").execute(&mut *lock).await.unwrap(); + } + /// #174 F2b: the pin permit bounds how many pin loops run concurrently, so a call /// with NOTHING to pin must not take one. It otherwise spends a global pin slot on no /// work, and the pool DEFERS rather than sheds, so those calls stall pins for every @@ -6217,7 +7023,17 @@ mod tests { let out = tokio::time::timeout( std::time::Duration::from_millis(500), - pin_new_objects_gated(&pin_sem, "", tmp.path(), vec![], &db), + pin_new_objects_gated( + &pin_sem, + "", + tmp.path(), + "git", + std::time::Duration::from_secs(30), + vec![], + &db, + "repo-gated-empty", + crate::ipfs_pin::PIN_BATCH_BUDGET, + ), ) .await .expect("an empty object list must not wait on pin admission (#174 F2b)"); diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index eee8fd8b..27b67786 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -490,6 +490,7 @@ mod tests { use clap::Parser; let keypair = Keypair::generate(); + let scan_token_key = crate::state::AppState::derive_scan_token_key(&keypair); let (ref_tx, _) = tokio::sync::broadcast::channel(1); let (task_tx, _) = tokio::sync::broadcast::channel(1); let pool = sqlx::postgres::PgPoolOptions::new() @@ -516,6 +517,16 @@ mod tests { rate_limiter: RateLimiter::new(100, Duration::from_secs(60)), create_ip_rate_limiter: RateLimiter::new(1000, Duration::from_secs(3600)), push_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + ipfs_work_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + ipfs_max_history_walks: crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, + ipfs_max_legacy_probes: crate::api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST, + ipfs_legacy_scan_page_rows: crate::api::ipfs::LEGACY_SCAN_PAGE_ROWS, + ipfs_max_legacy_scan_rows: crate::api::ipfs::MAX_LEGACY_SCAN_ROWS_PER_REQUEST, + ipfs_max_legacy_scan_rule_bytes: + crate::api::ipfs::MAX_LEGACY_SCAN_RULE_BYTES_PER_REQUEST, + ipfs_scan_token_key: Arc::new(scan_token_key), + ipfs_max_served_object_bytes: crate::api::ipfs::MAX_SERVED_OBJECT_BYTES, push_limiter_trust: crate::rate_limit::TrustedProxy::None, sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)), peer_write_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), @@ -534,7 +545,6 @@ mod tests { git_ipfs_walk_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_ipfs_walk_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), - ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), git_bin: "git".to_string(), } } diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index fefa063e..8cecd526 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -1,13 +1,13 @@ use clap::Parser; use std::path::PathBuf; -/// Upper bound on `git_service_timeout_secs` and `ipfs_request_budget_secs`, in seconds -/// (100 years). +/// Upper bound on `git_service_timeout_secs`, `ipfs_request_budget_secs`, and +/// `ipfs_resolve_budget_secs`, in seconds (100 years). /// -/// Two consumers now, so a future tightening moves both. `ipfs_request_budget_secs` -/// derives only the `Instant` addition in `get_by_cid`, not the lease-steal multiply -/// below, but it shares this ceiling because the defect class and the "set it very large -/// to disable" contract are the same. +/// Three consumers now, so a future tightening moves all of them. `ipfs_request_budget_secs` +/// and `ipfs_resolve_budget_secs` derive only the `Instant` addition in `get_by_cid`, not the +/// lease-steal multiply below, but they share this ceiling because the defect class and the +/// "set it very large to disable" contract are the same. /// /// The knob is not just stored, it is arithmetic input: the write path derives the /// per-repo lease steal bound from it (`* 2 + 60`), and #174 routed it into @@ -341,6 +341,17 @@ pub struct Config { /// Default: 32. Must be between 1 and 1_048_576 (the ceiling keeps the value /// under tokio's `Semaphore` permit limit so an oversized value is a clean CLI /// error rather than a boot-time panic). + /// + /// CONNECTION BUDGET. A push holds a Postgres connection from the node's separate + /// advisory-lock pool for the whole receive-pack, and that pool is sized from this + /// knob (this value + 8, clamped to 64 in `main.rs`). The node's total ceiling is + /// therefore `db_max_connections` (default 48) + the lock pool (default 40), i.e. + /// 88 by default, and at most `db_max_connections` + 64. Size BOTH against the + /// database server's `max_connections`: `db_max_connections`' own doc predates the + /// lock pool and no longer covers most of the node's connections. The +8 headroom + /// is shared with the three non-push `acquire_write` callers (`api/issues.rs` x2, + /// `api/pulls.rs`). Raising this knob past the clamp does NOT buy more lock-pool + /// connections; pushes beyond it wait briefly and then shed a 503 + Retry-After. #[arg( long, env = "GITLAWB_MAX_CONCURRENT_GIT_PUSHES", @@ -452,6 +463,54 @@ pub struct Config { )] pub ipfs_walk_per_source: usize, + /// Per-request ceiling on the number of legacy (NULL-provenance) repos the + /// `/ipfs/{cid}` resolver's scan fallback will PROBE (`acquire` + `git cat-file + /// -t`) before giving up. The provenance path targets its recorded sources; the + /// legacy scan, absent this bound, fans one anonymous request out to O(repos) + /// subprocess spawns and cold-cache fetches for a CID enumerable from the public + /// pins index. A truncated scan surfaces as a retryable 503, never a false 404. + /// Wired into `AppState::ipfs_max_legacy_probes` at construction. This knob does + /// not govern the history-walk ceiling; see `ipfs_max_repos_walked` for that. + /// Must be between 1 and 1_048_576. Default: 256. + #[arg( + long, + env = "GITLAWB_IPFS_MAX_LEGACY_PROBES", + default_value_t = crate::api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST as usize, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub ipfs_max_legacy_probes: usize, + + /// Per-request ceiling on how many repo ROWS the `/ipfs/{cid}` resolver's legacy + /// scan may fetch from the database. The probe ceiling above only starts counting + /// once a probe runs, and the two denial classes that dominate a hostile inventory + /// (quarantine, and a root-scope visibility deny) return before a probe or a visit + /// is spent, so without this an all-denying node paged its ENTIRE repo table for one + /// anonymous request while holding a scarce walk permit. + /// + /// Reach bound: a holder buried past the ceiling is servable in + /// `ceil(repos / ceiling) + 1` token-echoing retries. A truncated scan sheds a + /// retryable 503 carrying a sealed continuation token; the caller echoes it as + /// `?scan=` and the scan resumes where it stopped. No server-side scan state. + /// + /// Floor coupling: raising this knob raises every caller's per-window `/ipfs` work + /// allowance whenever the route limit sits below the derived floor, because the + /// floor must fit one full deep scan's page toll (see `AppState::ipfs_work_budget`). + /// + /// Tuning DOWN trade: token presence is a coarse inventory-size oracle. A ceiling + /// truncation emits a token and a wrapped scan does not, so laddering to the + /// `scan-wrapped` taint tells an anonymous caller the node's total repo count, + /// private and quarantined included, to within one ceiling. Tolled and coarse at + /// the 2048 default; it sharpens as the ceiling is lowered. + /// + /// Must be between 1 and 1_048_576. Default: 2048. + #[arg( + long, + env = "GITLAWB_IPFS_MAX_LEGACY_SCAN_ROWS", + default_value_t = crate::api::ipfs::MAX_LEGACY_SCAN_ROWS_PER_REQUEST, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub ipfs_max_legacy_scan_rows: usize, + /// Upper bound on the number of EXPENSIVE visibility walks /// (`allowed_blob_set_for_caller_bounded`, a full-history git walk in a /// blocking thread) a single `/ipfs/{cid}` request may run. Only a blob in a @@ -463,6 +522,22 @@ pub struct Config { /// sheds a retryable 503 + Retry-After rather than misreport existing content /// absent with a 404. The handler still short-circuits the moment it serves. /// Must be between 1 and 1_048_576. Default: 64. + /// + /// The effective ceiling is the TIGHTER of this knob and the node's internal + /// history-walk ceiling, `MAX_PIN_SOURCES + 1` = 17 (see + /// `api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST` and the `min()` that combines the + /// two in the resolver). Setting this above 17 changes nothing, because the + /// internal ceiling already binds. Setting it below 17 does lower the cap: the + /// constant side of the `min()` is what keeps a request from being truncated + /// before its whole bounded provenance source set has been tried, so an operator + /// who goes under it is choosing a tighter cap that can 503 a provenanced + /// request, which is allowed. + /// + /// That combined cap is charged PER PHASE, not per request: the provenance phase and + /// the legacy-scan fallback each get their own equal budget, so one request can run + /// up to twice it in total (see `MAX_HISTORY_WALKS_PER_REQUEST`, which explains why + /// the split is what keeps the fallback from inheriting a provenance phase's spent + /// remainder). #[arg( long, env = "GITLAWB_IPFS_MAX_REPOS_WALKED", @@ -531,14 +606,86 @@ pub struct Config { )] pub ipfs_request_budget_secs: u64, + /// Budget for the PRE-WALK CID resolve inside `get_by_cid`, in seconds: the + /// `oids_for_cid` lookup that maps the requested CID to its git oid(s), which runs + /// while the scarce walk admission (the global pool permit plus the per-source + /// sub-permit) is already held. + /// + /// It exists because that one await decides whether the request does any admitted + /// work at all. A syntactically valid CID with no `pinned_cids` row runs zero probes + /// and zero walks, so under a stalled or saturated pool it would otherwise occupy a + /// walk slot for the whole `ipfs_request_budget_secs` window (600s by default) while + /// nothing is walking, and enough distinct source keys doing that reject every real + /// `/ipfs` retrieval at admission. The other repair, resolving the CID before taking + /// admission, was rejected: admission stays FIRST so an anonymous flood sheds before + /// touching the database at all, and moving the read ahead of it would let arbitrarily + /// many unadmitted permissionless callers stack concurrent DB queries. + /// + /// The effective deadline is the lesser of this and the remaining request budget, so a + /// value larger than `ipfs_request_budget_secs` degrades to the request budget rather + /// than extending it. Only the resolve is on this clock; every later stage stays on the + /// full request budget, because from the second oid candidate on those run after real + /// probe and walk work and a short deadline anchored at admission would shed a + /// legitimately slow but progressing scan. + /// + /// Must be positive, and no larger than `GIT_SERVICE_TIMEOUT_SECS_MAX`, for the same + /// representability reason as the request budget above: `get_by_cid` derives the + /// resolve deadline as `Instant::now() + Duration::from_secs(this)`, and that addition + /// panics on overflow in release builds too. Default: 10s. + #[arg( + long, + env = "GITLAWB_IPFS_RESOLVE_BUDGET_SECS", + default_value_t = 10, + value_parser = clap::value_parser!(u64).range(1..=GIT_SERVICE_TIMEOUT_SECS_MAX) + )] + pub ipfs_resolve_budget_secs: u64, + /// Per-client-IP rate limit for `GET /ipfs/{cid}`, in requests per hour. The /// route is publicly reachable (`optional_signature`) and each request can drive /// a full-history git walk, so it carries a per-IP flood brake in addition to the /// concurrency cap above (a rate limit bounds request *rate*, the semaphore /// bounds concurrent slow holds — different axes). Keyed on the resolved client /// IP via `GITLAWB_TRUSTED_PROXY`. `0` disables. Default: 600. + /// + /// This is the pure once-per-request ROUTE brake. The resolver's internal + /// per-probe/per-walk WORK budget is a SEPARATE bucket whose capacity is DERIVED + /// from this value (`AppState::ipfs_work_budget`), not a knob of its own; `0` here + /// disables that derived bucket too. #[arg(long, env = "GITLAWB_IPFS_RATE_LIMIT", default_value_t = 600)] pub ipfs_rate_limit: usize, + + /// Rows the legacy provider-CID repair sweep reads per batch (U4, #173). + /// + /// The sweep walks every `pinned_cids` row on the node once, repairing rows that + /// releases before this branch keyed on a PROVIDER CID (Kubo dag-pb / Pinata CIDv0) + /// instead of the raw-content resolver key. This bounds one batch, so the sweep can + /// never turn into a single unbounded table scan competing with request traffic. + /// Conservative on purpose: paired with the inter-batch delay below the default is + /// ~64 rows per minute, which finishes a large pin set in hours of idle background + /// work rather than one expensive burst. Must be between 1 and 100_000. + #[arg( + long, + env = "GITLAWB_PIN_REPAIR_SWEEP_BATCH", + default_value_t = 64, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=100_000) + )] + pub pin_repair_sweep_batch: i64, + + /// Seconds the legacy provider-CID repair sweep sleeps between batches (U4, #173). + /// + /// Each batch costs an indexed range scan plus, for the legacy rows in it, a + /// `git cat-file` per row. The delay is what keeps that off the DB's and the disk's + /// critical path: the sweep is repairing rows that have been unresolvable since the + /// upgrade, so finishing slowly is fine and finishing fast at the cost of live + /// traffic is not. `0` disables the pause (test and one-off operational use only). + /// Must be between 0 and 86_400. + #[arg( + long, + env = "GITLAWB_PIN_REPAIR_SWEEP_DELAY_SECS", + default_value_t = 60, + value_parser = clap::builder::RangedU64ValueParser::::new().range(0..=86_400) + )] + pub pin_repair_sweep_delay_secs: u64, } impl Config { @@ -771,6 +918,35 @@ mod tests { ); } + /// U4 (#173): the repair sweep's bounds are conservative by default and a batch of + /// 0 (a sweep that walks nothing and never terminates) is a CLI error, not a + /// runtime hang. The delay does accept 0, for tests and one-off operational runs. + #[test] + fn pin_repair_sweep_knobs_default_conservatively() { + let c = Config::parse_from(["gitlawb-node"]); + assert_eq!(c.pin_repair_sweep_batch, 64); + assert_eq!(c.pin_repair_sweep_delay_secs, 60); + + assert!(Config::try_parse_from(["gitlawb-node", "--pin-repair-sweep-batch", "0"]).is_err()); + assert!( + Config::try_parse_from(["gitlawb-node", "--pin-repair-sweep-batch", "100001"]).is_err() + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--pin-repair-sweep-batch", "8"]) + .pin_repair_sweep_batch, + 8 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--pin-repair-sweep-delay-secs", "0"]) + .pin_repair_sweep_delay_secs, + 0 + ); + assert!( + Config::try_parse_from(["gitlawb-node", "--pin-repair-sweep-delay-secs", "86401"]) + .is_err() + ); + } + #[test] fn ipfs_walk_per_source_defaults_and_rejects_out_of_range() { assert_eq!(Config::parse_from(["gitlawb-node"]).ipfs_walk_per_source, 4); @@ -786,6 +962,195 @@ mod tests { ); } + /// The legacy-probe budget and the expensive-walk cap are SEPARATE knobs with + /// different defaults. They were one field until the probe budget and the walk cap + /// were split apart, so assert both defaults here: a future collapse back into one + /// field silently gives one of the two the other's default. + #[test] + fn ipfs_probe_and_walk_knobs_default_apart_and_reject_out_of_range() { + let default = Config::parse_from(["gitlawb-node"]); + assert_eq!(default.ipfs_max_legacy_probes, 256, "legacy-probe budget"); + assert_eq!(default.ipfs_max_repos_walked, 64, "expensive-walk cap"); + + assert_eq!( + Config::parse_from(["gitlawb-node", "--ipfs-max-legacy-probes", "8"]) + .ipfs_max_legacy_probes, + 8 + ); + // 0 would probe no repos (serve nothing); clap must reject it. + assert!(Config::try_parse_from(["gitlawb-node", "--ipfs-max-legacy-probes", "0"]).is_err()); + assert!( + Config::try_parse_from(["gitlawb-node", "--ipfs-max-legacy-probes", "1048577"]) + .is_err() + ); + assert!(Config::try_parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "0"]).is_err()); + } + + /// The `GITLAWB_IPFS_MAX_LEGACY_PROBES` knob must actually reach the legacy-probe + /// budget it advertises: production seeds `ipfs_max_legacy_probes` from this helper, + /// so the knob is a no-op unless the helper reflects it. RED while the helper returns + /// the hardcoded `MAX_LEGACY_PROBES_PER_REQUEST` (256 regardless of the knob), GREEN + /// once it reads the knob. + #[test] + fn ipfs_max_legacy_probes_wires_the_legacy_probe_budget() { + use crate::state::AppState; + // Knob set to 1 → a one-probe legacy budget. + let one = Config::parse_from(["gitlawb-node", "--ipfs-max-legacy-probes", "1"]); + assert_eq!( + AppState::ipfs_legacy_probe_budget(&one), + 1, + "the knob must control the legacy-probe budget, not be ignored" + ); + // Unset knob preserves the shipped 256-probe behaviour. + let default = Config::parse_from(["gitlawb-node"]); + assert_eq!( + AppState::ipfs_legacy_probe_budget(&default), + 256, + "the default knob keeps the shipped 256-probe budget" + ); + assert_eq!( + AppState::ipfs_legacy_probe_budget(&default), + crate::api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST, + "the default budget equals the constant it replaced" + ); + // Ceiling guard: the knob never governs the history-walk ceiling, which must + // stay at MAX_PIN_SOURCES + 1 or a provenanced full source set false-503s. + assert!( + crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST > crate::db::MAX_PIN_SOURCES as u32, + "the history-walk ceiling is independent of the repos-walked knob" + ); + } + + /// The `/ipfs` work-budget capacity is DERIVED from the route limit (R6, KTD6), with + /// a hard floor of one complete COMBINED resolution per window: the provenance + /// phase's walk term plus a full legacy search (the effective + /// `ipfs_max_legacy_probes` plus the row ceiling's page toll). This guards the + /// derived default so a single default-config deep search never self-throttles + /// mid-scan and recreates the F6 admit-then-429 for a legitimate caller. A + /// `RateLimiter` sized to the derived budget must admit the whole budget back to + /// back. + #[test] + fn ipfs_work_budget_derives_from_route_limit_and_clears_the_probe_floor() { + use crate::state::AppState; + + // Default config: derived work budget = max(route 600, probe budget 256) = 600, + // comfortably above the 256-probe floor. + let default = Config::parse_from(["gitlawb-node"]); + let budget = AppState::ipfs_work_budget(&default); + assert_eq!(budget, 600, "default derives max(route 600, probe 256)"); + assert!( + budget >= AppState::ipfs_legacy_probe_budget(&default) as usize, + "the work budget must clear one full legacy search per window" + ); + + // Tight route limit (1): the floor lifts the work budget to one complete + // COMBINED resolution, the 256-probe budget PLUS the page toll a 2048-row + // ceiling costs at 128 rows per page (16) PLUS the provenance phase's walk term + // min(17, 64) = 17, so 289, NOT down to 1. The provenance walks come off the + // same bucket before the fallback runs, so a floor without that term hands the + // legacy search a bucket the provenance phase already spent from. This case + // also carries the walk term's ABOVE-constant direction: the repos-walked knob + // is at its default 64, so `MAX_HISTORY_WALKS_PER_REQUEST` (17) is what binds. + let tight = Config::parse_from(["gitlawb-node", "--ipfs-rate-limit", "1"]); + assert_eq!( + AppState::ipfs_work_budget(&tight), + 289, + "a tight route limit is floored at probes + pages + walks \ + (256 + 16 + min(17, 64) = 17), not clamped to 1" + ); + + // The walk term's BELOW-constant direction: a repos-walked knob under the + // history-walk constant is what the resolver's own `walk_cap` min() selects, so + // it is what the floor must carry too. 256 + 16 + min(17, 3) = 275. + let narrow_walk = Config::parse_from([ + "gitlawb-node", + "--ipfs-rate-limit", + "1", + "--ipfs-max-repos-walked", + "3", + ]); + assert_eq!( + AppState::ipfs_work_budget(&narrow_walk), + 275, + "the walk term takes min(17, repos-walked 3) = 3, the resolver's own \ + walk_cap, so the floor is 256 + 16 + 3" + ); + + // Raised probe budget lifts the floor with it (the work budget tracks the + // effective probe budget, not the constant). The walk cap here is a SECOND + // below-constant proof at a different pair of values: min(17, 7) = 7, and the + // probe knob is raised at the same time so a floor that folded the two terms + // together (they were one field before the split) reads visibly wrong rather + // than plausibly right. + let raised = Config::parse_from([ + "gitlawb-node", + "--ipfs-rate-limit", + "10", + "--ipfs-max-legacy-probes", + "1000", + "--ipfs-max-repos-walked", + "7", + ]); + assert_eq!( + AppState::ipfs_work_budget(&raised), + 1023, + "the floor tracks the operator-raised legacy-probe budget (1000) plus the \ + default row ceiling's page toll (16) plus the walk term min(17, 7) = 7" + ); + + // The scan-rows knob is coupled to the floor too, and this EXECUTES the coupling + // rather than describing it: every page the ceiling permits is charged to the + // caller's work bucket, so a raised ceiling that did not lift the floor would + // 429 an honest caller part-way down their own token ladder. 4096 rows at 128 + // rows per page is 32 pages, so the floor is 256 + 32 + the default walk term + // of 17. + let wide_scan = Config::parse_from([ + "gitlawb-node", + "--ipfs-rate-limit", + "10", + "--ipfs-max-legacy-scan-rows", + "4096", + ]); + assert_eq!( + AppState::ipfs_work_budget(&wide_scan), + 305, + "raising the row ceiling must raise the work floor by the pages it buys \ + (256 probes + 4096/128 = 32 pages + min(17, 64) = 17 walks), or a full \ + deep scan self-throttles" + ); + + // 0 route limit disables the derived bucket too (a 0-capacity limiter admits all). + let disabled = Config::parse_from(["gitlawb-node", "--ipfs-rate-limit", "0"]); + assert_eq!( + AppState::ipfs_work_budget(&disabled), + 0, + "route limit 0 disables the derived work bucket alongside the route brake" + ); + + // Behavioral floor: a limiter sized to the derived (tight-route) budget admits + // a whole combined resolution's worth of charges back to back for one source, + // then sheds the next. + let budget = AppState::ipfs_work_budget(&tight); + let limiter = + crate::rate_limit::RateLimiter::new(budget, std::time::Duration::from_secs(3600)); + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + rt.block_on(async { + for i in 0..budget { + assert!( + limiter.check("1.2.3.4").await, + "charge {i} of one full combined resolution must be admitted (no mid-scan throttle)" + ); + } + assert!( + !limiter.check("1.2.3.4").await, + "the probe past the derived budget is shed" + ); + }); + } + #[test] fn ipfs_max_repos_walked_defaults_and_rejects_out_of_range() { assert_eq!( @@ -840,6 +1205,52 @@ mod tests { ); } + #[test] + fn ipfs_resolve_budget_secs_defaults_to_10_and_rejects_zero() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).ipfs_resolve_budget_secs, + 10 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--ipfs-resolve-budget-secs", "3"]) + .ipfs_resolve_budget_secs, + 3 + ); + // 0 would shed every /ipfs request at the pre-walk resolve (unconditional + // 503); clap must reject it. + assert!( + Config::try_parse_from(["gitlawb-node", "--ipfs-resolve-budget-secs", "0"]).is_err() + ); + // The ceiling is shared with the request budget: at the max it parses and the + // derived deadline is still representable, past it clap rejects. + let at_max = Config::try_parse_from([ + "gitlawb-node", + "--ipfs-resolve-budget-secs", + &GIT_SERVICE_TIMEOUT_SECS_MAX.to_string(), + ]) + .expect("the documented maximum must parse"); + assert_eq!( + at_max.ipfs_resolve_budget_secs, + GIT_SERVICE_TIMEOUT_SECS_MAX + ); + assert!(std::time::Instant::now() + .checked_add(std::time::Duration::from_secs( + at_max.ipfs_resolve_budget_secs + )) + .is_some()); + for over in [GIT_SERVICE_TIMEOUT_SECS_MAX + 1, u64::MAX] { + assert!( + Config::try_parse_from([ + "gitlawb-node", + "--ipfs-resolve-budget-secs", + &over.to_string(), + ]) + .is_err(), + "{over} is past the representable ceiling and must be rejected at parse time" + ); + } + } + /// #174 (RED-before/GREEN-after): the upper bound is what keeps the deadline derived /// from this knob in range. `get_by_cid` builds the request budget as /// `Instant::now() + Duration::from_secs(this)` (api/ipfs.rs), and that addition is an diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 50c3bdda..cc2cf0bd 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -23,6 +23,21 @@ pub struct RepoRecord { pub machine_id: Option, } +/// One row of a keyset page from [`Db::list_repos_page_for_scan`]. +/// +/// Carries the row's quarantine flag inline (so the IPFS scan needs no separate +/// whole-node quarantine query) and the RAW stored `created_at` text, which is +/// the first half of the keyset cursor. The raw text is kept because the keyset +/// comparison is a text comparison and re-serializing the parsed `DateTime` is +/// not guaranteed to reproduce the stored bytes — a cursor that differs from the +/// stored value by one character skips or repeats rows. +#[derive(Debug, Clone)] +pub struct ScanRepoRow { + pub repo: RepoRecord, + pub quarantined: bool, + pub created_at_key: String, +} + /// Per-rule replication mode for a visibility rule. /// `A` hides existence entirely (only valid at whole-repo scope `/`). /// `B` keeps object SHAs and the path visible but withholds content @@ -879,18 +894,17 @@ const MIGRATIONS: &[Migration] = &[ version: 11, name: "ref_update_owner_did", stmts: &[ - // Index deferred — the feed gate (#144) does not read owner_did yet. + // Index deferred: the feed gate (#144) does not read owner_did yet. "ALTER TABLE received_ref_updates ADD COLUMN IF NOT EXISTS owner_did TEXT", ], }, - // Reservation: v17, deliberately not main's current_max + 1 (which is 12). - // The runner keys the applied set on the integer alone, so a version another - // in-flight branch also claims is skipped in full on whichever side merges - // second — no error, no warning, and schema_migrations still reads healthy - // while the column is simply absent. Two open branches already claim into - // this range: #135/#173 holds through 14 (15 once it rebases past v11), and - // #253 took 16. 17 clears both. Gaps are harmless: the runner iterates the - // array and never requires contiguity. + // Reservation: v17 is deliberately not main's current_max + 1. The runner keys the + // applied set on the integer alone, so a version another in-flight branch also + // claims is skipped in full on whichever side merges second: no error, no warning, + // and schema_migrations still reads healthy while the column is simply absent. + // #253 took 16, and the pin-provenance work below took 18-23 when it merged, so 17 + // sits between them. Gaps are harmless: the runner iterates the array and never + // requires contiguity. Migration { version: 17, name: "sync_queue_attempted_at", @@ -901,8 +915,222 @@ const MIGRATIONS: &[Migration] = &[ "ALTER TABLE sync_queue ADD COLUMN IF NOT EXISTS attempted_at TEXT", ], }, + // The six pin-provenance migrations below were numbered 11-16 while this work was + // in flight and moved to 18-23 on merge, because 11 and 16 were claimed elsewhere. + // A database that ran an earlier commit of this branch therefore has schema_migrations + // rows for the old numbers. Those rows are orphans: the runner skips on `version` + // alone and never reads `name`, so nothing detects them, and the DDL below re-runs + // as a no-op against objects that already exist. Recreate any such database rather + // than upgrading it in place. + Migration { + version: 18, + name: "pinned_cids_cid_index", + stmts: &[ + // GET /ipfs/{cid} resolves an incoming CID -> git oid via pinned_cids.cid + // (#173); index it so the per-request lookup is not a table scan. This is + // a NEW versioned migration (not appended to the applied v1 bundle) so a + // node already past v1 actually gets the index. Non-unique on purpose: cid + // is a function of raw content, so a UNIQUE index could reject a legitimate + // record_pinned_cid insert, and colliding rows serve byte-identical content. + "CREATE INDEX IF NOT EXISTS idx_pinned_cids_cid ON pinned_cids(cid)", + ], + }, + Migration { + version: 19, + name: "pinned_cids_repo_provenance", + stmts: &[ + // Record the repository a pin came from so GET /ipfs/{cid} resolves a + // provenanced pin straight to its ONE source repo instead of scanning every + // repo (#173, jatmn round 2 — bounds the anonymous fan-out and removes the + // updated_at-ordering false-404). NEW versioned migration (never appended to + // the applied v1 pinned_cids table) so a node past v1 gets the column. + // Nullable: pins recorded before this migration have no provenance and fall + // back to the legacy repo scan; new pins carry repo_id and resolve to one + // repo. Indexed for the resolver's oid -> repo_id lookup. + "ALTER TABLE pinned_cids ADD COLUMN IF NOT EXISTS repo_id TEXT", + "CREATE INDEX IF NOT EXISTS idx_pinned_cids_repo_id ON pinned_cids(repo_id)", + ], + }, + Migration { + version: 20, + name: "pin_repo_sources", + stmts: &[ + // F1 (#173, jatmn round 8): a shared object (a blob/tree/commit common to + // forks and mirrors) can be pinned from more than one repo. `pinned_cids` + // keeps only the FIRST pinner's `repo_id`, so a shared object first pinned + // from a private/quarantined repo 404s by CID even when a later PUBLIC repo + // also pinned it. Record EVERY pin-path source so `GET /ipfs/{cid}` can try + // each. NEW versioned migration (never appended to an applied block, INV-7). + // Bounded per object at insert time (MAX_PIN_SOURCES) so an adversary pushing + // one object from N repos cannot make resolution O(repos) (R2, INV-10). + "CREATE TABLE IF NOT EXISTS pin_repo_sources ( + sha256_hex TEXT NOT NULL, + repo_id TEXT NOT NULL, + PRIMARY KEY (sha256_hex, repo_id) + )", + "CREATE INDEX IF NOT EXISTS idx_pin_repo_sources_sha ON pin_repo_sources(sha256_hex)", + ], + }, + Migration { + version: 21, + name: "pinned_cids_legacy_provider_cid", + stmts: &[ + // R8 (#173, jatmn round 10): the opportunistic legacy provider-CID repair + // rewrites `pinned_cids.cid` from a stored PROVIDER CID (Kubo dag-pb / + // Pinata CIDv0) to the raw-content resolver key and stashes the OLD value + // here, so the rewrite is auditable and the row's legacy origin survives. + // Distinct from `pinata_cid` on purpose: `has_pinata_cid` gates the Pinata + // pin-skip, so parking a Kubo-legacy CID there would make Pinata forever + // skip re-pinning that object. NEW versioned migration (never appended to an + // applied block, INV-7) so a node past v13 actually gets the column. + // Nullable: only a repaired row sets it. + "ALTER TABLE pinned_cids ADD COLUMN IF NOT EXISTS legacy_provider_cid TEXT", + ], + }, + Migration { + version: 22, + name: "pinned_cids_sources_incomplete", + stmts: &[ + // U3 (#173): `record_pin_source` is BEST EFFORT at every call site, so a + // non-empty, below-cap source set is not proof that every source was + // recorded. An object first pinned from a private repo and later pushed + // from a PUBLIC repo whose record failed keeps a set naming only the + // private source, and the resolver used to call that set complete and 404 + // an object the public repo would serve. Record the miss DURABLY here so + // `GET /ipfs/{cid}` keeps the bounded scan fallback for exactly those + // objects. Not inferable from row counts or timestamps: neither can tell + // "no other source exists" from "a source failed to record", which is the + // whole distinction. NEW versioned migration (never appended to an applied + // block, INV-7). NOT NULL DEFAULT FALSE so every pre-existing row reads as + // complete and ordinary denials stay off the O(repos) path (INV-10). + "ALTER TABLE pinned_cids ADD COLUMN IF NOT EXISTS pin_sources_incomplete BOOLEAN NOT NULL DEFAULT FALSE", + ], + }, + Migration { + version: 23, + name: "pin_repair_sweep_cursor", + stmts: &[ + // U4 (#173): the legacy provider-CID repair sweep walks `pinned_cids` in + // bounded batches over an ordered `sha256_hex` cursor. The cursor has to be + // DURABLE, or a restart rewinds the walk to the start of the table and an + // upgraded node with a large pin set never finishes repairing it. One row + // (`id = 1`, enforced by the CHECK) rather than a key-value table: there is + // exactly one sweep and no second consumer, and a real constraint beats a + // convention nobody can enforce. NEW versioned migration (never appended to + // an applied block, INV-7). No default row is inserted: an absent row is the + // "never swept" state, which the empty-string cursor start already means, so + // there is no first-run special case to get wrong. + "CREATE TABLE IF NOT EXISTS pin_repair_sweep ( + id INTEGER NOT NULL PRIMARY KEY CHECK (id = 1), + cursor TEXT NOT NULL + )", + ], + }, + Migration { + version: 24, + name: "pin_source_failures", + stmts: &[ + // #173 round 12 (jatmn): v22's `pin_sources_incomplete` is one boolean per + // OBJECT, so any successful source record cleared it, including one from a + // repo unrelated to the failure. The resolver then read the set as fully + // enumerated, dropped the scan fallback, and 404'd an anonymous caller whose + // only servable copy was the unrecorded public one. The missing source is a + // property of an (object, repo) PAIR, so it is stored as one. + // + // NEW versioned migration (never appended to an applied block, INV-7). A new + // table rather than a column on `pinned_cids`: the relation is many-per-object + // and `CREATE TABLE` takes no lock on the pin table a live node is reading. + "CREATE TABLE IF NOT EXISTS pin_source_failures ( + sha256_hex TEXT NOT NULL, + repo_id TEXT NOT NULL, + PRIMARY KEY (sha256_hex, repo_id) + )", + // Carry the pre-upgrade markers over. Which repo failed was never recorded, + // so they get the empty sentinel, which no real `repo_id` equals: those + // objects keep the scan fallback until something repairs them, rather than + // being cleared by the next unrelated record the way they would have been + // before. Strictly safer than the behavior being replaced, and bounded by how + // rare an exhausted record is. + "INSERT INTO pin_source_failures (sha256_hex, repo_id) + SELECT sha256_hex, '' FROM pinned_cids WHERE pin_sources_incomplete + ON CONFLICT DO NOTHING", + // `pinned_cids.pin_sources_incomplete` is deliberately NOT dropped. Nothing + // reads it after this migration, and leaving it costs one unused boolean, + // whereas dropping it makes a rollback to the previous release lose the + // markers it still reads. + ], + }, + Migration { + version: 25, + name: "repos_created_at_id_index", + stmts: &[ + // #173 (jatmn): backs the keyset order of the paged legacy CID scan + // (`list_repos_page_for_scan`, `ORDER BY created_at ASC, id ASC` with a + // `(created_at, id) > (...)` cursor). The scan replaced a whole-table + // preload precisely to stop one anonymous `GET /ipfs/{cid}` from costing + // work proportional to the node's repo inventory (INV-10), and without this + // index that bound is only half real: `repos` carries no index on + // `(created_at, id)`, so Postgres seq-scans the whole table and top-N sorts + // it to return EVERY page, while the scarce IPFS walk admission is held. + // Measured on a 50k-row fixture: 954 shared buffers and ~47ms per page + // without it, versus an Index Only Scan at 4-5 buffers, ~0.08ms, and + // `Heap Fetches: 0` with it — and the keyset predicate is pushed down as an + // `Index Cond` instead of filtering after a scan. + // + // Column order and direction are load-bearing and must match the query + // exactly; `idx_repos_updated_at` (the order the scan used to use) cannot + // serve this one. NOTHING NAMES THIS INDEX IN ANY QUERY TEXT, so a + // grep-driven "unused index" cleanup will not see its consumer: it is + // reachable from an unauthenticated route and dropping it reopens the + // amplification, so treat it as part of the resolver, not as tuning. + // + // NEW versioned migration (never appended to an applied block, INV-7). + "CREATE INDEX IF NOT EXISTS idx_repos_created_at_id ON repos (created_at ASC, id ASC)", + ], + }, + Migration { + version: 26, + name: "pin_repair_sweep_discovery_cursor", + stmts: &[ + // #173 round 13 (F5): discovery probes at most + // `MAX_LEGACY_DISCOVERY_PROBES` warm candidates per source-less row, taken + // from a list ordered `(created_at, id)`. That order is stable, so without a + // continuation every traversal probed the same oldest sixteen and a holder + // at position seventeen was never reached by anything, on any node, ever. + // These two columns are the boundary the next traversal's window starts + // after, so coverage becomes a bounded number of traversals rather than + // unreachable. + // + // STEERABILITY is why this is a keyset KEY and not an offset into the list. + // `repo_id` derives from a grindable owner DID, so the one thing an attacker + // must not be able to do is move the window off the true holder. Candidates + // enter and leave the warm list between traversals (a cold repo warming on a + // Tigris-backed node, a fresh registration, a deletion), and every such + // change silently renumbers an offset while leaving a key's boundary exactly + // where it was. Fresh registrations sort LAST under `created_at` and cannot + // be backdated, so they can only ever be appended behind the window. + // + // RESIDUAL, stated rather than implied: an operator who can insert repos + // with an arbitrary `created_at` can still place candidates between the + // continuation and the holder and delay it by a traversal per sixteen rows + // inserted. That is a privileged write, it costs a real repo row each, and + // it delays rather than prevents, since the window keeps advancing. + // + // NEW versioned migration (never appended to an applied block, INV-7). NOT + // NULL DEFAULT '' so an existing `pin_repair_sweep` row reads as "start at + // the head of the list", which is the same thing a never-swept node reads. + "ALTER TABLE pin_repair_sweep ADD COLUMN IF NOT EXISTS discovery_cursor_created_at TEXT NOT NULL DEFAULT ''", + "ALTER TABLE pin_repair_sweep ADD COLUMN IF NOT EXISTS discovery_cursor_id TEXT NOT NULL DEFAULT ''", + ], + }, ]; +/// Max distinct source repos recorded per pinned object (F1, #173 jatmn round 8). +/// Bounds both the resolver's per-OID source loop and the `pin_repo_sources` growth, +/// so an adversary re-pushing one object from many repos cannot make resolution +/// O(repos) (R2, INV-10). +pub const MAX_PIN_SOURCES: i64 = 16; + // ── Repos ───────────────────────────────────────────────────────────────────── pub(crate) fn normalize_owner_key(did: &str) -> &str { @@ -1079,6 +1307,22 @@ impl Db { Ok(row.map(row_to_repo)) } + /// Fetch a repo by its stable `id`. Used by the `/ipfs/{cid}` provenance path, + /// which resolves a pin straight to its ONE source repo (#173) instead of + /// paging the whole repo table. `id` is exact, so unlike `get_repo`'s fuzzy + /// owner/name match there is no mirror-vs-canonical disambiguation. + pub async fn get_repo_by_id(&self, id: &str) -> Result> { + let row = sqlx::query( + "SELECT id, name, owner_did, description, is_public, default_branch, + created_at, updated_at, disk_path, forked_from, machine_id + FROM repos WHERE id = $1 LIMIT 1", + ) + .bind(id) + .fetch_optional(&self.pool) + .await?; + Ok(row.map(row_to_repo)) + } + #[allow(dead_code)] pub async fn list_repos(&self, owner_did: &str) -> Result> { let rows = sqlx::query( @@ -1093,21 +1337,68 @@ impl Db { Ok(rows.into_iter().map(row_to_repo).collect()) } - /// Raw list of every repo row — NOT deduped (a mirror row and its canonical - /// row both appear) and without stars. For enumeration callers that must see - /// every physical row (e.g. the IPFS object scan in `api::ipfs`), not for - /// listing surfaces. Listing surfaces dedupe via `list_all_repos_deduped` or - /// `list_all_repos_with_stars` + `dedupe_canonical_repos`. - pub async fn list_all_repos(&self) -> Result> { + /// One keyset page of raw repo rows for the IPFS object scan (`api::ipfs`) — + /// NOT deduped (a mirror row and its canonical row both appear), since that + /// scan must see every physical row. Listing surfaces dedupe via + /// `list_all_repos_deduped` or `list_all_repos_with_stars` + + /// `dedupe_canonical_repos` and must not use this. + /// + /// Paged rather than whole-table because the scan runs on an anonymously + /// reachable route while holding scarce walk admission: materializing the + /// node's entire repo inventory (plus its rules) before the per-probe budget + /// has spent a single probe is an amplification sink (INV-10). The caller + /// stops asking for pages once its budgets are spent. + /// + /// Ordered on `(created_at, id)` ASC, both IMMUTABLE, so keyset paging is + /// exact: no row is visited twice and none is skipped. `updated_at` would be + /// wrong twice over — a repo touched mid-scan can cross a page boundary and go + /// unvisited (a servable public object misreported as a 404), and it is + /// attacker-bumpable, which would let a caller sort their own repos ahead of + /// the true holder and bury it past the probe budget. + /// + /// `after` is the raw `(created_at, id)` of the last row of the previous page, + /// `None` for the first page. It carries the STORED `created_at` text, not a + /// re-serialized `DateTime`: the comparison is a text comparison and a + /// round-trip through `to_rfc3339` is not guaranteed to reproduce the stored + /// bytes. + /// + /// Each row carries its own `quarantined` flag so the scan needs no separate + /// whole-node quarantine query (INV-11's hard drop stays per row). + pub async fn list_repos_page_for_scan( + &self, + after: Option<(&str, &str)>, + limit: i64, + ) -> Result> { + let (after_created, after_id) = match after { + Some((created_at, id)) => (Some(created_at), Some(id)), + None => (None, None), + }; let rows = sqlx::query( "SELECT id, name, owner_did, description, is_public, default_branch, - created_at, updated_at, disk_path, forked_from, machine_id - FROM repos ORDER BY updated_at DESC", + created_at, updated_at, disk_path, forked_from, machine_id, quarantined + FROM repos + WHERE $1::text IS NULL OR (created_at, id) > ($1::text, $2::text) + ORDER BY created_at ASC, id ASC + LIMIT $3", ) + .bind(after_created) + .bind(after_id) + .bind(limit) .fetch_all(&self.pool) .await?; - Ok(rows.into_iter().map(row_to_repo).collect()) + Ok(rows + .into_iter() + .map(|r| { + let quarantined: bool = r.get("quarantined"); + let created_at_key: String = r.get("created_at"); + ScanRepoRow { + quarantined, + created_at_key, + repo: row_to_repo(r), + } + }) + .collect()) } pub async fn list_all_repos_with_stars(&self) -> Result> { @@ -2483,20 +2774,532 @@ impl Db { Ok(row.get::("cnt") > 0) } - pub async fn record_pinned_cid(&self, sha256_hex: &str, cid: &str) -> Result<()> { + /// Every git oid a pinned CID maps to (`pinned_cids.cid` -> `sha256_hex`). + /// `GET /ipfs/{cid}` resolves the content-addressed CID a client sends back to + /// the object's git oid this way: a real pin CID digests the raw object + /// content, not the git oid, so the digest cannot be `git cat-file`d directly + /// (#173). The index is unique on the git oid but NON-unique on cid, so two + /// distinct oids can share one content-CID (a tree and a blob whose raw bytes + /// collide, or byte-identical content pinned under two oids). Returning every + /// candidate lets the handler try each rather than pick one arbitrarily and + /// false-404 when the chosen one is withheld or absent while another is + /// readable (#173). Empty when the CID was never pinned on this node. + /// + /// ORDERED, for the same reason `pin_sources_for_oid` orders its union: the handler + /// walks these candidates under ONE shared probe budget, visit budget and pager, so + /// whichever comes back first is the one that spends the request's budget. Left + /// unordered this is a bare sequential scan returning heap order, which an unpin and + /// re-pin of any one object rewrites, so two nodes holding identical data could + /// resolve the same CID differently and one could 503 where the other serves. + pub async fn oids_for_cid(&self, cid: &str) -> Result> { + let rows = + sqlx::query("SELECT sha256_hex FROM pinned_cids WHERE cid = $1 ORDER BY sha256_hex") + .bind(cid) + .fetch_all(&self.pool) + .await?; + Ok(rows + .into_iter() + .map(|r| r.get::("sha256_hex")) + .collect()) + } + + /// Record a pinned object's CID and the repository it was pinned from + /// (`repo_id`, #173). On conflict the `COALESCE` backfills a NULL provenance + /// from a known source while keeping first-pinner-owns: an existing non-NULL + /// `repo_id` is never rewritten by a later push of the same oid, but a legacy + /// pin (or a pin recorded before provenance existed) whose `repo_id` is NULL + /// gets it filled the next time the object is re-pinned with a known source. + /// `cid`/`pinned_at` are left untouched on conflict. `repo_id` is `None` only + /// for a legacy pin with no known source; those fall back to the resolver's scan. + /// + /// The production first-pin path now goes through [`Self::record_pinned_cid_with_source`] + /// (U3, #173) so the pin and its source land atomically; this remains the seam for + /// seeding legacy, source-less rows in tests. + #[cfg_attr(not(test), allow(dead_code))] + pub async fn record_pinned_cid( + &self, + sha256_hex: &str, + cid: &str, + repo_id: Option<&str>, + ) -> Result<()> { sqlx::query( - "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at) - VALUES ($1, $2, $3) - ON CONFLICT(sha256_hex) DO NOTHING", + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) + VALUES ($1, $2, $3, $4) + ON CONFLICT(sha256_hex) DO UPDATE SET + repo_id = COALESCE(pinned_cids.repo_id, EXCLUDED.repo_id)", ) .bind(sha256_hex) .bind(cid) .bind(Utc::now().to_rfc3339()) + .bind(repo_id) + .execute(&self.pool) + .await?; + Ok(()) + } + + /// The resolver key currently stored for a pinned object (`pinned_cids.cid`), + /// or `None` for an unpinned oid. The opportunistic legacy-repair path reads + /// it to decide candidacy from the codec of the string alone (no object bytes) + /// before it recomputes anything. + pub async fn cid_for_oid(&self, sha256_hex: &str) -> Result> { + let row = sqlx::query("SELECT cid FROM pinned_cids WHERE sha256_hex = $1") + .bind(sha256_hex) + .fetch_optional(&self.pool) + .await?; + Ok(row.map(|r| r.get::("cid"))) + } + + /// Rewrite a legacy provider-CID row to the raw-content resolver key, stashing + /// the old provider value in `legacy_provider_cid` (#173 R8, KTD8). Before this + /// branch the pin path stored the PROVIDER CID (Kubo dag-pb / Pinata CIDv0) in + /// `cid`; the `/ipfs` resolver recomputes the raw CID and 404s a mismatched key + /// even though `list_pinned_cids` still advertises it. The `WHERE cid = + /// $old_provider_cid` guard makes a concurrent double-repair a no-op (the second + /// writer sees the already-rewritten key and matches nothing) and never touches + /// a row keyed on a different value. Stashed in `legacy_provider_cid`, NOT + /// `pinata_cid`: the latter gates the Pinata pin-skip (`has_pinata_cid`), so a + /// Kubo-legacy CID parked there would make Pinata permanently skip the object. + pub async fn repair_legacy_provider_cid( + &self, + sha256_hex: &str, + raw_cid: &str, + old_provider_cid: &str, + ) -> Result<()> { + sqlx::query( + "UPDATE pinned_cids + SET cid = $2, legacy_provider_cid = $3 + WHERE sha256_hex = $1 AND cid = $3", + ) + .bind(sha256_hex) + .bind(raw_cid) + .bind(old_provider_cid) .execute(&self.pool) .await?; Ok(()) } + /// One ordered batch of `pinned_cids` rows strictly after `cursor`, for the U4 + /// legacy provider-CID repair sweep. Returns `(sha256_hex, cid)` ordered by + /// `sha256_hex` (the table's primary key, so the walk rides the PK index) and + /// capped at `limit` rows, which is what BOUNDS the sweep: one pass can never read + /// more than a batch, however large the pin set is. + /// + /// Deliberately NOT filtered to legacy rows in SQL. "Is this a raw CIDv1" is a + /// multibase+codec decode (`is_raw_cidv1`), which Postgres cannot express, and a + /// prefix-match approximation would silently mis-classify keys under a different + /// multihash. The caller applies the real predicate, so `limit` bounds rows READ + /// (the DB cost), not rows repaired. + pub async fn pinned_cids_after( + &self, + cursor: &str, + limit: i64, + ) -> Result> { + let rows = sqlx::query( + "SELECT sha256_hex, cid FROM pinned_cids + WHERE sha256_hex > $1 + ORDER BY sha256_hex + LIMIT $2", + ) + .bind(cursor) + .bind(limit) + .fetch_all(&self.pool) + .await?; + Ok(rows + .into_iter() + .map(|r| (r.get::("sha256_hex"), r.get::("cid"))) + .collect()) + } + + /// Where the U4 repair sweep's walk left off, or `""` before it has ever run. + /// Empty string sorts below every hex oid, so a first run and a rewound run are + /// the same code path (`sha256_hex > ''` is the whole table). + pub async fn pin_repair_cursor(&self) -> Result { + let row = sqlx::query("SELECT cursor FROM pin_repair_sweep WHERE id = 1") + .fetch_optional(&self.pool) + .await?; + Ok(row + .map(|r| r.get::("cursor")) + .unwrap_or_default()) + } + + /// Persist the sweep's walk position. Written after every batch, so a restart + /// resumes rather than re-walking the table from the beginning. A rewrite is a + /// plain upsert: the sweep is the single writer, and re-repairing an + /// already-repaired row is a no-op anyway (the codec cost gate spares it). + pub async fn set_pin_repair_cursor(&self, cursor: &str) -> Result<()> { + sqlx::query( + "INSERT INTO pin_repair_sweep (id, cursor) VALUES (1, $1) + ON CONFLICT (id) DO UPDATE SET cursor = EXCLUDED.cursor", + ) + .bind(cursor) + .execute(&self.pool) + .await?; + Ok(()) + } + + /// Where the sweep's DISCOVERY window left off, as a `(created_at, id)` keyset + /// key, or `("", "")` before any traversal has completed one. + /// + /// A second, independent position from [`Db::pin_repair_cursor`]: that one walks + /// `pinned_cids` rows, this one walks the warm CANDIDATE list a source-less row is + /// probed against. Both are per-TRAVERSAL, and the candidate one only ever moves + /// at the end of a completed traversal, so every pass of one traversal reads the + /// same value and every source-less row in it shares one window. + /// + /// A key rather than an offset. Repos enter and leave the warm candidate list + /// between traversals (a cold repo warming on a Tigris-backed node, a fresh + /// registration, a deletion), and an offset silently means a different candidate + /// once anything below it moves, which slides the window off the row it was about + /// to reach. A key names the boundary itself, so an insert below it is invisible. + /// The key is the RAW stored `created_at` text (`ScanRepoRow::created_at_key`), + /// never a re-serialized `DateTime`, for the reason that struct's own doc gives. + pub async fn discovery_continuation(&self) -> Result<(String, String)> { + let row = sqlx::query( + "SELECT discovery_cursor_created_at, discovery_cursor_id + FROM pin_repair_sweep WHERE id = 1", + ) + .fetch_optional(&self.pool) + .await?; + Ok(row + .map(|r| { + ( + r.get::("discovery_cursor_created_at"), + r.get::("discovery_cursor_id"), + ) + }) + .unwrap_or_default()) + } + + /// Persist the discovery window's continuation at the end of a completed traversal. + /// + /// The INSERT arm names `cursor` explicitly with `''`. v23 declares that column + /// `NOT NULL` and seeds NO row, so a never-swept node has nothing to update and an + /// upsert naming only the continuation columns would fail its NOT NULL check. + /// Every caller treats a failed persist as warn-only, so that failure would be + /// SILENT and the window would never rotate on exactly the nodes this sweep exists + /// for. `''` is the same value `pin_repair_cursor` reads as "never swept", so + /// seeding it here starts no walk anywhere but the top of the table. + /// + /// The UPDATE arm touches ONLY the two continuation columns. Writing `cursor` there + /// too would clobber a live row-walk position with `''` every time the window + /// rotated, rewinding the `pinned_cids` walk to the start of the table. + pub async fn set_discovery_continuation(&self, created_at_key: &str, id: &str) -> Result<()> { + sqlx::query( + "INSERT INTO pin_repair_sweep (id, cursor, discovery_cursor_created_at, discovery_cursor_id) + VALUES (1, '', $1, $2) + ON CONFLICT (id) DO UPDATE SET + discovery_cursor_created_at = EXCLUDED.discovery_cursor_created_at, + discovery_cursor_id = EXCLUDED.discovery_cursor_id", + ) + .bind(created_at_key) + .bind(id) + .execute(&self.pool) + .await?; + Ok(()) + } + + /// The repository a pinned object was recorded from (`pinned_cids.repo_id`), + /// or `None` for a legacy pin (recorded before provenance existed) or an + /// unpinned oid. `GET /ipfs/{cid}` uses this to gate+serve the ONE source + /// repo instead of scanning every repo (#173). + pub async fn provenance_for_oid(&self, sha256_hex: &str) -> Result> { + let row = sqlx::query("SELECT repo_id FROM pinned_cids WHERE sha256_hex = $1") + .bind(sha256_hex) + .fetch_optional(&self.pool) + .await?; + Ok(row.and_then(|r| r.get::, _>("repo_id"))) + } + + /// Backfill the source repo on an already-pinned object whose provenance is + /// NULL (a legacy pin recorded before provenance existed, #173, jatmn). The + /// `AND repo_id IS NULL` guard keeps first-pinner-owns: an existing non-NULL + /// provenance is left untouched. Touches only `repo_id` and never re-pins the + /// object's bytes, so it is safe to call on the already-pinned skip path. + pub async fn backfill_pin_provenance(&self, sha256_hex: &str, repo_id: &str) -> Result<()> { + sqlx::query( + "UPDATE pinned_cids SET repo_id = $2 WHERE sha256_hex = $1 AND repo_id IS NULL", + ) + .bind(sha256_hex) + .bind(repo_id) + .execute(&self.pool) + .await?; + Ok(()) + } + + /// Record a repository as a source for a pinned object (F1, #173 jatmn round 8), + /// bounded to about `MAX_PIN_SOURCES` distinct repos per object. The count guard + /// lives inside the INSERT (a single statement), which suppresses a re-push of the + /// SAME `(oid, repo)` via `ON CONFLICT DO NOTHING`. It does NOT hard-serialize + /// concurrent inserts of DIFFERENT repos for the same object: under Postgres READ + /// COMMITTED each concurrent writer's count subquery reads a snapshot that omits the + /// others' uncommitted rows, so N concurrent pushers can each see `count < cap` and + /// overshoot by up to N-1 rows. The overshoot is a small constant (bounded by + /// concurrent-pusher count, never O(repos)), and the RESOLVER read side + /// (`pin_sources_for_oid`) caps the ADDITIONAL sources at `MAX_PIN_SOURCES` (always + /// keeping the first-pinner), so the INV-10 bound on serve-time work holds at + /// `O(MAX_PIN_SOURCES + 1)` regardless of a table overshoot. + /// + /// A record that ACTUALLY ADDS a source row also CLEARS the + /// `pin_sources_incomplete` marker for the object, in the SAME transaction as the + /// insert (U3, #173), so the clear cannot drift across the four call sites or land + /// without the row it describes. + /// + /// The clear is gated on `rows_affected() > 0` because the INSERT is a no-op in two + /// ordinary cases: the `(oid, repo)` pair already exists (`ON CONFLICT DO NOTHING`) + /// and the source set is at cap (the count guard). The skip path calls this for + /// EVERY already-pinned object, and on a requeue pass that list is the whole-repo + /// enumeration, so an unconditional clear meant the next coalesced push from a repo + /// already in the set wiped the marker for every object in the repo without + /// recording anything (round 11 regression). The residual, which the gate does not + /// close: the marker is per-object, not per-(object, repo), so a GENUINE record from + /// a third repo C still clears a marker that repo A's failed record set. That is the + /// deliberate cost of a single boolean; closing it needs a per-(oid, repo) marker + /// table, and it fails in the safe direction (the marker only ever ADDS the scan + /// fallback, never removes a source the resolver already tries). + pub async fn record_pin_source(&self, sha256_hex: &str, repo_id: &str) -> Result<()> { + let mut tx = self.pool.begin().await?; + let inserted = sqlx::query( + "INSERT INTO pin_repo_sources (sha256_hex, repo_id) + SELECT $1, $2 + WHERE (SELECT count(*) FROM pin_repo_sources WHERE sha256_hex = $1) < $3 + ON CONFLICT DO NOTHING", + ) + .bind(sha256_hex) + .bind(repo_id) + .bind(MAX_PIN_SOURCES) + .execute(&mut *tx) + .await? + .rows_affected(); + if inserted > 0 { + // Clears THIS repo's failure only (#173 round 12). A boolean per object meant + // repo C's genuine record wiped the marker repo B's failure set, and the + // resolver then dropped the scan fallback while B's copy was still unrecorded. + sqlx::query("DELETE FROM pin_source_failures WHERE sha256_hex = $1 AND repo_id = $2") + .bind(sha256_hex) + .bind(repo_id) + .execute(&mut *tx) + .await?; + } + tx.commit().await?; + Ok(()) + } + + /// Record a first pin and its source ATOMICALLY (U3, #173). The first-pin path + /// used to run `record_pinned_cid` and `record_pin_source` as two independent + /// best-effort calls, so the pin could land while its source did not, leaving a + /// source set that is silently missing its own first pinner. One transaction + /// removes that window entirely: either both rows land or neither does, and a + /// total failure leaves the object unpinned so the next push retries it. + /// + /// The marker clear carries the same `rows_affected` gate as `record_pin_source`. + /// It is not load-bearing here: this path runs only when `is_pinned` said no row + /// exists, and `mark_pin_sources_incomplete` is a no-op without a `pinned_cids` row, + /// so there is no marker to wrongly clear. The gate is kept for the one window that + /// is not covered by that argument, a concurrent pinner landing the row between the + /// `is_pinned` check and this upsert, and so the two clears cannot drift apart. + pub async fn record_pinned_cid_with_source( + &self, + sha256_hex: &str, + cid: &str, + repo_id: &str, + ) -> Result<()> { + let mut tx = self.pool.begin().await?; + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) + VALUES ($1, $2, $3, $4) + ON CONFLICT(sha256_hex) DO UPDATE SET + repo_id = COALESCE(pinned_cids.repo_id, EXCLUDED.repo_id)", + ) + .bind(sha256_hex) + .bind(cid) + .bind(Utc::now().to_rfc3339()) + .bind(repo_id) + .execute(&mut *tx) + .await?; + let inserted = sqlx::query( + "INSERT INTO pin_repo_sources (sha256_hex, repo_id) + SELECT $1, $2 + WHERE (SELECT count(*) FROM pin_repo_sources WHERE sha256_hex = $1) < $3 + ON CONFLICT DO NOTHING", + ) + .bind(sha256_hex) + .bind(repo_id) + .bind(MAX_PIN_SOURCES) + .execute(&mut *tx) + .await? + .rows_affected(); + if inserted > 0 { + // Clears THIS repo's failure only (#173 round 12). A boolean per object meant + // repo C's genuine record wiped the marker repo B's failure set, and the + // resolver then dropped the scan fallback while B's copy was still unrecorded. + sqlx::query("DELETE FROM pin_source_failures WHERE sha256_hex = $1 AND repo_id = $2") + .bind(sha256_hex) + .bind(repo_id) + .execute(&mut *tx) + .await?; + } + tx.commit().await?; + Ok(()) + } + + /// Record a DISCOVERED holder and arm the resolver's fallback ATOMICALLY (U5, #173). + /// The sweep's discovery arm used to call `record_pin_source` and then, separately, + /// `mark_pin_sources_incomplete`. Two best-effort writes, so a transient failure of + /// the second one left the row with a nonempty, below-cap, UNMARKED source set: the + /// resolver's `needs_scan` is `sources.is_empty() || at_cap || incomplete`, so all + /// three signals were off, the bounded legacy scan was dropped, and an unrecorded + /// public duplicate stayed 404'd for good once the DB error cleared (no later sweep + /// revisits a raw-CIDv1 row). One transaction removes that state entirely: either the + /// source row and the sentinel both land or neither does, and neither-lands is the + /// benign end (an empty set is itself a `needs_scan` signal). + /// + /// The sentinel insert is UNCONDITIONAL, unlike the marker clear's `rows_affected` + /// gate: discovery probes a bounded, warm-only candidate set and stops at the first + /// holder, so finding one holder is never evidence the set is complete, whether or + /// not this particular call added a row. It names the empty-string UNKNOWN-repo + /// sentinel (the same one the v24 migration carries pre-upgrade markers under), so no + /// real per-repo record can clear it, and it carries the same + /// `WHERE EXISTS (pinned_cids row)` guard as [`Self::mark_pin_sources_incomplete`] + /// so a marker never sits in the table for an object this node never pinned. + /// + /// Commit-terminated, like [`Self::record_pin_source`], so a caller that wraps this + /// in `db_bounded` may read `BoundedDbError::Elapsed` as "definitely did not land": + /// the cancelled future never reaches `tx.commit()`, no COMMIT is sent, and Postgres + /// discards the transaction when the connection resets. + pub async fn record_discovered_pin_source( + &self, + sha256_hex: &str, + repo_id: &str, + ) -> Result<()> { + let mut tx = self.pool.begin().await?; + let inserted = sqlx::query( + "INSERT INTO pin_repo_sources (sha256_hex, repo_id) + SELECT $1, $2 + WHERE (SELECT count(*) FROM pin_repo_sources WHERE sha256_hex = $1) < $3 + ON CONFLICT DO NOTHING", + ) + .bind(sha256_hex) + .bind(repo_id) + .bind(MAX_PIN_SOURCES) + .execute(&mut *tx) + .await? + .rows_affected(); + if inserted > 0 { + // Clears THIS repo's failure only, the same gate and reason as + // `record_pin_source`: a per-object clear let one repo's genuine record wipe + // a marker another repo's failure set. + sqlx::query("DELETE FROM pin_source_failures WHERE sha256_hex = $1 AND repo_id = $2") + .bind(sha256_hex) + .bind(repo_id) + .execute(&mut *tx) + .await?; + } + sqlx::query( + "INSERT INTO pin_source_failures (sha256_hex, repo_id) + SELECT $1, '' WHERE EXISTS (SELECT 1 FROM pinned_cids WHERE sha256_hex = $1) + ON CONFLICT DO NOTHING", + ) + .bind(sha256_hex) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(()) + } + + /// Mark this object's pin-source set as KNOWN INCOMPLETE for `repo_id` (U3, #173). + /// Called when a `record_pin_source` exhausts its retries, which is the only moment + /// the node knows a source it meant to record is missing. `GET /ipfs/{cid}` reads it + /// to keep the bounded scan fallback for that object, so a public copy that would + /// serve is no longer 404'd. + /// + /// The marker names the PAIR, so only a later successful record from the same repo + /// clears it (#173 round 12). A no-op when no `pinned_cids` row exists: the first-pin + /// path is transactional, so there is no half-recorded pin to describe, and without + /// the guard a marker for an object this node never pinned would sit in the table + /// arming a fallback for nothing. + pub async fn mark_pin_sources_incomplete(&self, sha256_hex: &str, repo_id: &str) -> Result<()> { + sqlx::query( + "INSERT INTO pin_source_failures (sha256_hex, repo_id) + SELECT $1, $2 WHERE EXISTS (SELECT 1 FROM pinned_cids WHERE sha256_hex = $1) + ON CONFLICT DO NOTHING", + ) + .bind(sha256_hex) + .bind(repo_id) + .execute(&self.pool) + .await?; + Ok(()) + } + + /// Whether this object's pin-source set is KNOWN INCOMPLETE (U3, #173): a + /// `record_pin_source` for it failed outright and no later record from the same repo + /// has repaired it. `false` for an unpinned oid and for every object with no recorded + /// failure, so the common path is unchanged and an ordinary denial never fans out + /// (INV-10). + pub async fn pin_sources_incomplete(&self, sha256_hex: &str) -> Result { + let found: Option = + sqlx::query_scalar("SELECT 1 FROM pin_source_failures WHERE sha256_hex = $1 LIMIT 1") + .bind(sha256_hex) + .fetch_optional(&self.pool) + .await?; + Ok(found.is_some()) + } + + /// Every source repository recorded for a pinned object (F1, #173 jatmn round 8): + /// the union of the first-pinner `pinned_cids.repo_id` and the `pin_repo_sources` + /// rows, deduped and ordered for a deterministic resolver walk. + /// + /// The first-pinner (a single row by `pinned_cids`' PK on `sha256_hex`) is ALWAYS + /// included; the `LIMIT MAX_PIN_SOURCES` caps only the ADDITIONAL `pin_repo_sources` + /// rows. This keeps the resolver's per-source work a bounded `O(MAX_PIN_SOURCES + 1)` + /// ceiling (INV-10) while never letting the cap evict the original source. A prior + /// version applied the `LIMIT` to the whole UNION with a lexicographic `ORDER BY`, + /// which let an attacker 404 a legacy public CID (first-pinner in `pinned_cids` but + /// not yet in `pin_repo_sources`) by pushing the same object from `MAX_PIN_SOURCES` + /// repos whose grindable ids sort before it, evicting the public source from the + /// window. Empty for a legacy pin with no known source (it falls back to the repo + /// scan) or an unpinned oid. + pub async fn pin_sources_for_oid(&self, sha256_hex: &str) -> Result> { + let rows = sqlx::query( + "SELECT repo_id FROM pinned_cids + WHERE sha256_hex = $1 AND repo_id IS NOT NULL + UNION + SELECT repo_id FROM ( + SELECT repo_id FROM pin_repo_sources + WHERE sha256_hex = $1 + ORDER BY repo_id + LIMIT $2 + ) capped + ORDER BY repo_id", + ) + .bind(sha256_hex) + .bind(MAX_PIN_SOURCES) + .fetch_all(&self.pool) + .await?; + Ok(rows + .into_iter() + .map(|r| r.get::("repo_id")) + .collect()) + } + + /// Whether `pin_repo_sources` is at the `MAX_PIN_SOURCES` cap for this oid, i.e. + /// the provenance source set returned by [`Self::pin_sources_for_oid`] may be + /// INCOMPLETE. `record_pin_source` stops inserting at exactly `MAX_PIN_SOURCES` + /// rows and drops later sources silently, so a full table is the only observable + /// signal that a servable source (e.g. a later public pinner) may have been + /// dropped. `get_by_cid` uses this to decide whether a provenance miss should fall + /// back to the bounded legacy scan (which gates every repo through the real + /// visibility gate and so finds a dropped public source) rather than 404 — closing + /// the pin-source griefing hole where 16 attacker sources bury a public one. `>=` + /// (not `==`) is defensive against any future overshoot. + pub async fn pin_sources_at_cap(&self, sha256_hex: &str) -> Result { + let count: i64 = + sqlx::query_scalar("SELECT count(*) FROM pin_repo_sources WHERE sha256_hex = $1") + .bind(sha256_hex) + .fetch_one(&self.pool) + .await?; + Ok(count >= MAX_PIN_SOURCES) + } + pub async fn record_encrypted_blob( &self, repo_id: &str, @@ -2567,6 +3370,17 @@ impl Db { Ok(row.map(|r| r.get("recipients_tag"))) } + /// Every pinned object this node ADVERTISES (`GET /api/v1/ipfs/pins`). + /// + /// U4 (#173): rows still keyed on a legacy PROVIDER CID (Kubo dag-pb / Pinata + /// CIDv0, written by releases before this branch) are withheld from the listing. + /// The `/ipfs/{cid}` resolver recomputes the raw-content CID from the object bytes + /// and refuses any row whose stored key does not match, so advertising the legacy + /// key hands a client a CID this node deliberately will not serve. The background + /// repair sweep rewrites those rows to the raw key, and each one reappears here the + /// moment it is repaired. Filtering is done in Rust because the raw-CIDv1 test is a + /// multibase+codec decode (`is_raw_cidv1`), not something SQL can express; it is the + /// SAME predicate the repair path uses as its cost gate, so the two cannot drift. pub async fn list_pinned_cids(&self) -> Result> { let rows = sqlx::query( "SELECT sha256_hex, cid, pinned_at, pinata_cid FROM pinned_cids ORDER BY pinned_at DESC", @@ -2575,6 +3389,7 @@ impl Db { .await?; Ok(rows .into_iter() + .filter(|r| gitlawb_core::cid::is_raw_cidv1(r.get::<&str, _>("cid"))) .map(|r| PinnedCidRecord { sha256_hex: r.get("sha256_hex"), cid: r.get("cid"), @@ -2596,18 +3411,34 @@ impl Db { } /// Record the Pinata CID for a git object. - /// Inserts the row if it doesn't exist (objects pinned directly to Pinata - /// without a prior local IPFS pin get cid = pinata_cid). - pub async fn record_pinata_cid(&self, sha256_hex: &str, pinata_cid: &str) -> Result<()> { + /// + /// `raw_cid` is the locally-computed raw-content CID (`Cid::from_git_object_bytes`, + /// CIDv1/raw/sha2-256), the resolver key `GET /ipfs/{cid}` looks up; `pinata_cid` + /// is the provider CID Pinata returned (a dag-pb/UnixFS CID for gateway retrieval). + /// Inserts the row if it doesn't exist (an object pinned directly to Pinata with + /// no prior local IPFS pin gets `cid = raw_cid`, never the provider CID — a dag-pb + /// provider CID must never become an alias that serves raw bytes that do not hash + /// to it, #173). On conflict `cid` is left untouched: a prior local pin already + /// stored the correct raw CID, and the COALESCE backfills a NULL provenance from a + /// known source while keeping first-pinner-owns. + pub async fn record_pinata_cid( + &self, + sha256_hex: &str, + raw_cid: &str, + pinata_cid: &str, + repo_id: Option<&str>, + ) -> Result<()> { sqlx::query( - "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, pinata_cid) - VALUES ($1, $2, $3, $4) - ON CONFLICT(sha256_hex) DO UPDATE SET pinata_cid = EXCLUDED.pinata_cid", + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, pinata_cid, repo_id) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT(sha256_hex) DO UPDATE SET pinata_cid = EXCLUDED.pinata_cid, + repo_id = COALESCE(pinned_cids.repo_id, EXCLUDED.repo_id)", ) .bind(sha256_hex) - .bind(pinata_cid) // fallback local cid if row is new + .bind(raw_cid) // resolver-key cid: locally-computed raw CID, never the provider CID .bind(Utc::now().to_rfc3339()) .bind(pinata_cid) + .bind(repo_id) .execute(&self.pool) .await?; Ok(()) @@ -3321,6 +4152,113 @@ impl Db { } Ok(out) } + + /// Visibility rules for one scan page, bounded IN THE QUERY by a byte budget. + /// + /// The unbounded sibling above is right for the listing surfaces: they read a page + /// the caller is already authorized for. It is wrong for the resolver's legacy scan, + /// which runs on an anonymously reachable route while holding scarce walk admission. + /// A repo owner controls both how many rules their repos carry and how long each + /// `reader_dids` list is, so summing the bytes AFTER the rows arrive truncates the + /// request without bounding the work: the oversized page has already been transferred + /// and allocated by the time the sum is taken (INV-10 bounds work done, never results + /// measured afterwards). + /// + /// The cut lands on a REPO boundary, never inside one. A partially loaded rule set is + /// indistinguishable at the gate from a repo with no rules at all, so a mid-repo cut + /// would FAIL OPEN and serve a path-scoped object the missing rules would have + /// denied. Every repo this returns is therefore complete, and the caller drops the + /// page's tail from the cut onward rather than gating it against rules it does not + /// have. + /// + /// `repo_ids` must be in the page's `(created_at, id)` order; the returned cut is the + /// 0-based index into that slice of the first repo whose rules did NOT fit, or `None` + /// when the whole page fit. Repos carrying no rules never cut. + /// + /// The FIRST rule-carrying repo of a page is admitted whatever its size, so a page + /// always makes progress. Without that a repo whose rules alone exceed the remaining + /// budget would put the cut at the cursor, the caller's next request would reproduce + /// it exactly, and the ladder would be wedged on a permanent 503. One repo's rule set + /// is the residual bound this leaves; the whole page's was the bound before. + pub async fn list_visibility_rules_for_repos_bounded( + &self, + repo_ids: &[String], + byte_budget: usize, + ) -> Result<( + std::collections::HashMap>, + Option, + )> { + use std::collections::HashMap; + if repo_ids.is_empty() { + return Ok((HashMap::new(), None)); + } + // `running` is a sum of non-negative per-repo sizes over the page order, so it is + // monotonic: once it passes the budget every later repo is excluded too, which is + // what makes "the kept set is a prefix" true and the single cut index meaningful. + // `rn = 1` is the always-admit escape for the first rule-carrying repo. + let rows = sqlx::query( + "WITH sized AS ( + SELECT v.id, v.repo_id, v.path_glob, v.mode, v.reader_dids, v.created_by, + v.created_at, + octet_length(v.id) + octet_length(v.repo_id) + + octet_length(v.path_glob) + octet_length(v.created_by) + + octet_length(v.reader_dids) AS b, + array_position($1::text[], v.repo_id) AS pos + FROM visibility_rules v + WHERE v.repo_id = ANY($1::text[]) + ), + per_repo AS ( + SELECT repo_id, pos, SUM(b) AS repo_bytes FROM sized GROUP BY repo_id, pos + ), + cum AS ( + SELECT repo_id, pos, + SUM(repo_bytes) OVER (ORDER BY pos ROWS UNBOUNDED PRECEDING) AS running, + ROW_NUMBER() OVER (ORDER BY pos) AS rn + FROM per_repo + ), + kept AS ( + SELECT repo_id, pos FROM cum WHERE running <= $2::bigint OR rn = 1 + ), + cut AS ( + SELECT MIN(pos) AS cut_pos FROM cum WHERE running > $2::bigint AND rn > 1 + ) + SELECT s.id, s.repo_id, s.path_glob, s.mode, s.reader_dids, s.created_by, + s.created_at, cut.cut_pos + FROM sized s + JOIN kept k ON k.repo_id = s.repo_id + CROSS JOIN cut + ORDER BY k.pos, s.path_glob", + ) + .bind(repo_ids) + .bind(byte_budget.min(i64::MAX as usize) as i64) + .fetch_all(&self.pool) + .await?; + + // `array_position` is 1-based and the caller indexes a slice. No rows means no + // rules matched the page at all, which is also no cut. + let cut_at = rows + .first() + .and_then(|r| r.get::, _>("cut_pos")) + .map(|pos| (pos as usize).saturating_sub(1)); + let mut out: HashMap> = HashMap::new(); + for r in rows { + let readers: String = r.get("reader_dids"); + let created_at: String = r.get("created_at"); + let rule = VisibilityRule { + id: r.get("id"), + repo_id: r.get("repo_id"), + path_glob: r.get("path_glob"), + mode: VisibilityMode::from_db(&r.get::("mode")), + reader_dids: serde_json::from_str(&readers).unwrap_or_default(), + created_by: r.get("created_by"), + created_at: created_at + .parse::>() + .unwrap_or_else(|_| Utc::now()), + }; + out.entry(rule.repo_id.clone()).or_default().push(rule); + } + Ok((out, cut_at)) + } } // ── Repo Stars ──────────────────────────────────────────────────────────────── @@ -6083,6 +7021,232 @@ mod ref_certificate_tests { ); } + /// INV-7: upgrade-path test — an existing node already past v1 must still get + /// the `pinned_cids.cid` index. It ships as its OWN v11 migration (not appended + /// to the applied v1 bundle), so dropping the index + its `schema_migrations` + /// row and re-running migrations must recreate it, exercising the real code + /// path rather than hand-copying the SQL. + #[sqlx::test] + async fn v18_pinned_cids_cid_index_applies_on_upgrade(pool: PgPool) { + async fn index_exists(pool: &PgPool) -> bool { + sqlx::query_scalar::<_, bool>( + "SELECT EXISTS(SELECT 1 FROM pg_indexes WHERE indexname = 'idx_pinned_cids_cid')", + ) + .fetch_one(pool) + .await + .unwrap() + } + + let db = Db::for_testing(pool.clone()); + db.run_migrations().await.unwrap(); + assert!( + index_exists(&pool).await, + "fresh migration chain creates the index" + ); + + // Simulate a node at pre-v18: drop the index and its migration record. + sqlx::query("DROP INDEX IF EXISTS idx_pinned_cids_cid") + .execute(&pool) + .await + .unwrap(); + sqlx::query("DELETE FROM schema_migrations WHERE version = 18") + .execute(&pool) + .await + .unwrap(); + assert!( + !index_exists(&pool).await, + "precondition: index and its migration record removed" + ); + + // Re-run migrations: v11 re-applies and recreates the index on the upgrade. + db.run_migrations().await.unwrap(); + assert!( + index_exists(&pool).await, + "v11 must recreate idx_pinned_cids_cid on an upgrading node" + ); + } + + /// #173 (jatmn), INV-7 + INV-10: the paged legacy CID scan orders on + /// `(created_at, id)` ASC, and `repos` had no index in that order — only + /// `idx_repos_updated_at`, which backed the order the paging REPLACED. Without a + /// matching index Postgres seq-scans `repos` and top-N sorts it to return every + /// page (measured: 954 shared buffers, ~47ms per page on 50k rows) while the + /// scarce IPFS walk admission is held, so the application-side bound the paging + /// buys is cancelled by an O(rows) database cost on an anonymously reachable + /// route. With the index each page is an Index Only Scan at 4-5 buffers with the + /// keyset predicate pushed down as an `Index Cond`. + /// + /// PRESENCE is the whole property, so this asserts it structurally rather than by + /// name: some index on `repos` must lead with `created_at` then `id`, in that + /// order and ascending. A rename is fine; a reorder, a direction flip, or a drop + /// is not. Nothing names this index in any query text, so nothing else would + /// notice its removal. + /// + /// Also the INV-7 upgrade path, in the shape of the v18 test above: an existing + /// node past v1 gets the index from its OWN v25 entry, proven by dropping the + /// index plus its `schema_migrations` row and re-running the real migration code. + /// MUTATION (RED): delete the v25 entry from `MIGRATIONS` and the fresh-chain + /// assertion fails. + #[sqlx::test] + async fn v25_repos_created_at_id_index_applies_on_upgrade(pool: PgPool) { + // Structural, not by name: the leading two columns must be `created_at` then + // `id`, ascending (ASC is the default, so it renders with no DESC). + async fn keyset_index_exists(pool: &PgPool) -> bool { + sqlx::query_scalar::<_, bool>( + "SELECT EXISTS( + SELECT 1 + FROM pg_index i + JOIN pg_class t ON t.oid = i.indrelid + WHERE t.relname = 'repos' + AND i.indnatts >= 2 + AND (SELECT a.attname FROM pg_attribute a + WHERE a.attrelid = t.oid AND a.attnum = i.indkey[0]) = 'created_at' + AND (SELECT a.attname FROM pg_attribute a + WHERE a.attrelid = t.oid AND a.attnum = i.indkey[1]) = 'id' + AND pg_get_indexdef(i.indexrelid) NOT LIKE '%DESC%' + )", + ) + .fetch_one(pool) + .await + .unwrap() + } + + let db = Db::for_testing(pool.clone()); + db.run_migrations().await.unwrap(); + assert!( + keyset_index_exists(&pool).await, + "the paged legacy CID scan's ORDER BY created_at ASC, id ASC must be \ + index-backed, or every page seq-scans and sorts the whole repos table \ + while the IPFS walk admission is held (INV-10)" + ); + + // Simulate a node at pre-v25: drop the index and its migration record. + sqlx::query("DROP INDEX IF EXISTS idx_repos_created_at_id") + .execute(&pool) + .await + .unwrap(); + sqlx::query("DELETE FROM schema_migrations WHERE version = 25") + .execute(&pool) + .await + .unwrap(); + assert!( + !keyset_index_exists(&pool).await, + "precondition: index and its migration record removed" + ); + + db.run_migrations().await.unwrap(); + assert!( + keyset_index_exists(&pool).await, + "v25 must recreate the keyset index on an upgrading node" + ); + } + + /// U4 (#173 round 13, F5, INV-7 upgrade path): an existing node past v1 gets the + /// discovery-continuation columns from its OWN v26 entry, proven by dropping the + /// columns plus their `schema_migrations` row and re-running the real migration + /// code. + /// + /// The round-trip runs on a NEVER-SWEPT database, with no `pin_repair_sweep` row at + /// all, because that is the state the setter's insert arm is written for. v23 + /// declares `cursor` NOT NULL and seeds no row, so an upsert naming only the two new + /// columns fails its NOT NULL check on exactly the nodes this sweep exists for, and + /// every caller of the setter treats a failure as warn-only, so the window would + /// simply never rotate and nothing would say so. Asserting the read-back is what + /// makes that failure visible here. + /// + /// MUTATION (RED): delete the v26 entry from `MIGRATIONS` and the fresh-chain + /// round-trip fails on the missing columns. + #[sqlx::test] + async fn v26_discovery_continuation_applies_on_upgrade(pool: PgPool) { + async fn continuation_columns_exist(pool: &PgPool) -> bool { + sqlx::query_scalar::<_, i64>( + "SELECT count(*) FROM information_schema.columns + WHERE table_name = 'pin_repair_sweep' + AND column_name IN ('discovery_cursor_created_at', 'discovery_cursor_id')", + ) + .fetch_one(pool) + .await + .unwrap() + == 2 + } + + let db = Db::for_testing(pool.clone()); + db.run_migrations().await.unwrap(); + assert!( + continuation_columns_exist(&pool).await, + "the fresh migration chain must carry the discovery continuation columns" + ); + + // Simulate a node at pre-v26: drop the columns and their migration record. + sqlx::query( + "ALTER TABLE pin_repair_sweep + DROP COLUMN IF EXISTS discovery_cursor_created_at, + DROP COLUMN IF EXISTS discovery_cursor_id", + ) + .execute(&pool) + .await + .unwrap(); + sqlx::query("DELETE FROM schema_migrations WHERE version = 26") + .execute(&pool) + .await + .unwrap(); + assert!( + !continuation_columns_exist(&pool).await, + "precondition: columns and their migration record removed" + ); + + db.run_migrations().await.unwrap(); + assert!( + continuation_columns_exist(&pool).await, + "v26 must add the continuation columns on an upgrading node" + ); + + // NEVER SWEPT: no `pin_repair_sweep` row exists, so the setter has to INSERT and + // its insert arm has to satisfy v23's NOT NULL `cursor`. + assert_eq!( + sqlx::query_scalar::<_, i64>("SELECT count(*) FROM pin_repair_sweep") + .fetch_one(&pool) + .await + .unwrap(), + 0, + "precondition: the sweep has never run on this node" + ); + assert_eq!( + db.discovery_continuation().await.unwrap(), + (String::new(), String::new()), + "an unswept node reads the empty continuation, which means the head of the list" + ); + db.set_discovery_continuation("2020-01-01T00:00:00+00:00", "repo-42") + .await + .expect("the continuation persists on a never-swept node"); + assert_eq!( + db.discovery_continuation().await.unwrap(), + ( + "2020-01-01T00:00:00+00:00".to_string(), + "repo-42".to_string() + ), + "the continuation round-trips" + ); + assert_eq!( + db.pin_repair_cursor().await.unwrap(), + "", + "the insert arm seeds the row-walk cursor at the head of the table" + ); + + // A rotation must never move the row walk. Park the row cursor, rotate again, + // and read it back. + db.set_pin_repair_cursor("ff00").await.unwrap(); + db.set_discovery_continuation("2021-06-01T00:00:00+00:00", "repo-99") + .await + .unwrap(); + assert_eq!( + db.pin_repair_cursor().await.unwrap(), + "ff00", + "the update arm touches only the continuation columns, so an in-progress \ + table walk is never rewound by a window rotation" + ); + } + /// INV-7: upgrade-path test — seed a database at v9 with duplicate /// ref_certificates, then let the real v10 migration fire via /// run_migrations(). This exercises the migration code path rather than @@ -7114,8 +8278,8 @@ mod peer_authority_tests { /// | `prune_non_public_peers` (db/mod.rs) | a delete keyed on a computed bad-DID array; cannot repoint; boot-only caller in main.rs | /// | `seed_local_peer` (sync.rs) | excluded by test-module location: a deliberate `upsert_peer` bypass for `file://` fixtures, which the public-URL gate rejects | /// | `a_legacy_row_can_still_refresh_its_liveness` (db/mod.rs) | test-only. Seeds a PRE-GATE row by raw SQL on purpose: `upsert_peer` cannot create one, since the gate it is testing refuses exactly that DID. The fixture models what a deployed table already holds | -/// | `gossip_ping_round_requires_two_failures_before_persisting_unreachable` (main.rs) | test-only. Seeds a peer row by raw SQL so the gossip ping round can probe readiness hysteresis without going through `upsert_peer` | -/// | `manual_ping_uses_readiness_without_mutating_federation_gate` (api/peers.rs) | test-only. Seeds a peer row by raw SQL so the manual ping route can assert readiness probing without mutating federation gate state | +/// | `gossip_ping_round_requires_two_failures_before_persisting_unreachable` (main.rs) | test-only fixture seed. Raw SQL because the test drives the readiness HYSTERESIS, which needs a row already at `last_ping_ok = TRUE` before the round runs; it never exercises the announce gate | +/// | `manual_ping_uses_readiness_without_mutating_federation_gate` (api/peers.rs) | test-only fixture seed, same shape and same reason: the row under test must pre-exist so the assertion is about what the ping does NOT rewrite | /// /// And the `upsert_peer` CALL-SITE authority table, which the ledger above /// structurally cannot hold, because the bootstrap site issues no SQL of its own @@ -7327,3 +8491,64 @@ mod peers_table_writer_guard { ); } } + +#[cfg(test)] +mod cid_candidate_order_tests { + use super::Db; + use sqlx::PgPool; + + /// The candidate order `oids_for_cid` returns must not depend on the physical + /// row order in `pinned_cids`. + /// + /// `get_by_cid` walks the candidates under ONE shared probe budget, visit budget + /// and pager, so whichever candidate comes back first is the one that spends the + /// request's budget. Without an `ORDER BY` the query is a bare sequential scan and + /// Postgres is free to return heap order, which any UPDATE to any row rewrites: two + /// nodes holding identical data, or one node before and after an unrelated write, + /// resolve the same CID by trying candidates in a different order, so one serves the + /// object and the other sheds a 503. + /// + /// The sibling `pin_sources_for_oid` already orders its union for exactly this + /// reason, and the handler's own comment leans on that determinism. + /// + /// MUTATION (RED): drop the `ORDER BY` and the post-UPDATE read comes back rotated. + #[sqlx::test] + async fn oids_for_cid_is_ordered_independently_of_physical_row_order(pool: PgPool) { + let db = Db::for_testing(pool.clone()); + db.run_migrations().await.unwrap(); + + let cid = "bafkreiorderingfixtureaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + let oids = ["aa".repeat(32), "bb".repeat(32), "cc".repeat(32)]; + for oid in &oids { + db.record_pinned_cid(oid, cid, None).await.unwrap(); + } + + let before = db.oids_for_cid(cid).await.unwrap(); + assert_eq!(before.len(), 3, "fixture must seed three candidates"); + + // Move the first candidate to the end of the heap the way production does it: + // an unpin followed by a re-pin of the same object. An in-place UPDATE is not + // enough, since a HOT update leaves the row reachable from its original item + // pointer and a sequential scan still returns it in its old position. + sqlx::query("DELETE FROM pinned_cids WHERE sha256_hex = $1") + .bind(&oids[0]) + .execute(&pool) + .await + .expect("unpin one candidate"); + db.record_pinned_cid(&oids[0], cid, None).await.unwrap(); + + let after = db.oids_for_cid(cid).await.unwrap(); + assert_eq!( + before, after, + "an unrelated write to one candidate must not reorder the candidate list; \ + the order decides which oid spends the request's shared budget" + ); + + let mut sorted = after.clone(); + sorted.sort(); + assert_eq!( + after, sorted, + "the order must be a stated one (ascending oid), not whatever the heap holds" + ); + } +} diff --git a/crates/gitlawb-node/src/error.rs b/crates/gitlawb-node/src/error.rs index f5e14df1..474408e5 100644 --- a/crates/gitlawb-node/src/error.rs +++ b/crates/gitlawb-node/src/error.rs @@ -50,6 +50,16 @@ pub enum AppError { #[error("incomplete: {0}")] Incomplete(String), + /// A bounded search that could not complete. `continuation`, when present, is the + /// sealed scan position the caller echoes as `?scan=` to resume where the search + /// stopped (#173 round 13, F2). It is AEAD-sealed at the mint site, never plaintext: + /// the row it names is by construction one the caller was denied (INV-13). + #[error("search incomplete: {message}")] + SearchIncomplete { + message: String, + continuation: Option, + }, + #[error("git error: {0}")] Git(String), @@ -161,6 +171,15 @@ impl IntoResponse for AppError { AppError::Incomplete(msg) => { (StatusCode::UNPROCESSABLE_ENTITY, "incomplete", msg.clone()) } + // A bounded search that could not complete (the CID resolver hit its + // legacy-probe or walk ceiling), distinct from the 404 that asserts a + // definitive not-found: absence was NOT proven, so the caller should + // retry rather than treat it as gone (#173, F2). 503, retryable. + AppError::SearchIncomplete { message, .. } => ( + StatusCode::SERVICE_UNAVAILABLE, + "search_incomplete", + message.clone(), + ), AppError::Git(msg) => (StatusCode::INTERNAL_SERVER_ERROR, "git_error", msg.clone()), // 504, distinct from the 500 git_error and from the read-gate's 404 / // the auth 401, so the client can tell a deadline from a failure. @@ -201,16 +220,32 @@ impl IntoResponse for AppError { } }; - let body = Json(json!({ + let mut body = json!({ "error": code, "message": message, - })); + }); + // A truncated CID search may carry the sealed position the caller echoes as + // `?scan=` to resume. Rendered as a third body field, present only when the + // shed actually left something to resume: a wrapped scan and a throttled + // request both omit it, and its ABSENCE is what tells a caller the ladder is + // over. It is opaque ciphertext; see `gitlawb_core::scan_token`. + if let AppError::SearchIncomplete { + continuation: Some(token), + .. + } = &self + { + body["continuation"] = json!(token); + } - let mut resp = (status, body).into_response(); - // Overloaded advertises when to retry. It rides the shared tail above for - // its body/status, so the header is attached here rather than in a bespoke - // early return — keeping the variant handled in exactly one place. - if matches!(self, AppError::Overloaded(_)) { + let mut resp = (status, Json(body)).into_response(); + // Both retryable 503s advertise when to retry: Overloaded (capacity shed) and + // SearchIncomplete (a bounded CID search cut short by a cap — retry may complete + // it). They ride the shared tail above for body/status, so the header is attached + // here rather than in bespoke early returns, keeping each variant handled once. + if matches!( + self, + AppError::Overloaded(_) | AppError::SearchIncomplete { .. } + ) { resp.headers_mut().insert( axum::http::header::RETRY_AFTER, axum::http::HeaderValue::from_static("1"), diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 2aef6ff0..45820746 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -11,9 +11,11 @@ use std::collections::HashSet; use std::path::{Path, PathBuf}; use std::sync::Arc; +use std::time::Duration; use anyhow::{Context, Result}; use sqlx::pool::PoolConnection; +use sqlx::postgres::PgPoolOptions; use sqlx::{PgPool, Postgres}; use tokio::sync::Mutex; use tracing::{debug, info, warn}; @@ -26,11 +28,37 @@ use super::tigris::TigrisClient; pub struct RepoStore { repos_dir: PathBuf, tigris: Option, - /// Shared Postgres pool for advisory locks. - pool: PgPool, + /// Dedicated Postgres pool for repo write advisory locks, built by + /// `build_lock_pool` (see there for why it is separate and why it carries an + /// `after_release` hook). Never use this for ordinary queries. + lock_pool: PgPool, /// Tracks repos already confirmed to exist in Tigris — avoids redundant /// HEAD checks and background uploads for repos we've already migrated. migrated: Arc>>, + /// Test-only stall injected at the head of `acquire_write`'s Tigris phase, + /// i.e. AFTER the advisory lock is taken and BEFORE the guard exists. That + /// window is exactly where the outer `tokio::time::timeout` in + /// `api/repos.rs` can drop the future (#173). `TigrisClient` takes its + /// endpoint from process-wide AWS env vars and has no injectable seam, so + /// this flag is the smallest way to hold a real `acquire_write` open in that + /// window and cancel it there. + #[cfg(test)] + tigris_stall: Option, + /// Test-only counter of how many times a write guard from this store REACHED the + /// Tigris upload site in `release` (the point past the `success` check, where a + /// configured client would be uploaded to). It counts the decision, not a network + /// call: `TigrisClient` takes its endpoint from process-wide AWS env vars and has no + /// injectable seam, so every test runs with `tigris: None` and a counter inside the + /// `Some` arm could never move. Reaching the site is the property under test anyway: + /// an interrupted push must not publish a half-applied repo, and the disconnect path + /// must therefore never get here (#173 F2). + /// + /// Per store rather than a process global, so cases running in parallel do not see + /// each other's uploads, and an `Arc` rather than a `thread_local` because the guard + /// is released from a detached task on another worker thread. Same test-only counter + /// idiom as `ipfs_pin::note_legacy_repair_read`. + #[cfg(test)] + upload_site_reached: Arc, /// Test-only seam: armed here, copied into every `RepoWriteGuard` this store /// hands out, so a test that only holds the `AppState` (not the guard) can /// still park `release` at its pre-unlock point. See @@ -40,15 +68,16 @@ pub struct RepoStore { } impl RepoStore { + /// Derives its own lock pool from `pool`, so callers that only have the main + /// pool (tests, `for_testing` sites in other modules) still get the + /// `after_release` semantics `acquire_write` depends on. #[cfg(test)] pub fn for_testing(repos_dir: PathBuf, pool: PgPool) -> Self { - Self { + Self::new( repos_dir, - tigris: None, - pool, - migrated: Arc::new(tokio::sync::Mutex::new(std::collections::HashSet::new())), - pre_unlock_gate: None, - } + None, + build_lock_pool(&pool, 8, Duration::from_secs(5)), + ) } /// Test-only: every guard from this store parks in `release` right before the @@ -60,13 +89,43 @@ impl RepoStore { self } - pub fn new(repos_dir: PathBuf, tigris: Option, pool: PgPool) -> Self { + /// Test-only: the dedicated advisory-lock pool this store runs its write locks + /// on. `for_testing` DERIVES it from the pool it is handed (see `build_lock_pool`), + /// so a test that wants to observe what happened to a guard's connection has to + /// look here, not at the pool it passed in. + #[cfg(test)] + pub(crate) fn lock_pool(&self) -> &PgPool { + &self.lock_pool + } + + /// Test-only: see `tigris_stall`. + #[cfg(test)] + pub fn with_tigris_stall(mut self, stall: Duration) -> Self { + self.tigris_stall = Some(stall); + self + } + + /// Test-only: how many write guards from this store have reached the Tigris upload + /// site. See [`RepoStore::upload_site_reached`]. + #[cfg(test)] + pub fn tigris_upload_site_reached(&self) -> usize { + self.upload_site_reached + .load(std::sync::atomic::Ordering::SeqCst) + } + + /// `lock_pool` must come from `build_lock_pool`; a plain pool leaks advisory + /// locks on cancellation. + pub fn new(repos_dir: PathBuf, tigris: Option, lock_pool: PgPool) -> Self { Self { repos_dir, tigris, - pool, + lock_pool, migrated: Arc::new(Mutex::new(HashSet::new())), #[cfg(test)] + tigris_stall: None, + #[cfg(test)] + upload_site_reached: Arc::new(std::sync::atomic::AtomicUsize::new(0)), + #[cfg(test)] pre_unlock_gate: None, } } @@ -203,62 +262,70 @@ impl RepoStore { let (owner_slug, local_path) = self.local_path(owner_did, repo_name)?; let lock_key = advisory_lock_key(&owner_slug, repo_name); - // Pin a dedicated pooled connection and build the guard holding it BEFORE - // issuing the lock query. Session-level pg advisory locks are - // connection-affine (they can only be released on the session that took - // them), so the guard must own the locking connection; and building the - // guard first means any cancellation after the lock is taken — a - // `tokio::time::timeout` firing during the Tigris download below — drops a - // guard that CAN release, closing the leak the outer timeout otherwise - // opened (#174 F1). - let conn = self - .pool - .acquire() - .await - .context("acquiring db connection for the write advisory lock")?; - let mut guard = RepoWriteGuard { - owner_slug: owner_slug.clone(), - repo_name: repo_name.to_string(), - local_path: local_path.clone(), - lock_key, - conn: Some(conn), - locked: false, - released: false, - tigris: self.tigris.clone(), - #[cfg(test)] - test_pre_unlock_gate: self.pre_unlock_gate.clone(), - }; - - // Acquire the advisory lock with retry, through the guard's OWN connection, - // so the matching unlock (in release, or the Drop backstop) runs on the same - // session — pg_advisory_unlock on a different pooled connection is a no-op. - let mut acquired = false; + // Acquire the Postgres advisory lock with retry, using pg_try_advisory_lock so a + // stale lock from a crashed connection can't block us indefinitely. + // + // The connection is checked out INSIDE the loop and RETURNED before each sleep. + // Only the connection that actually took the lock is retained. Two constraints + // pull in opposite directions here, and this is what satisfies both: + // + // * Session ownership. A session-level advisory lock belongs to the CONNECTION + // that took it, so the lock and its `pg_advisory_unlock` must run on the same + // one. Running them through the pool (`fetch_one(&self.pool)`) lets them land + // on different connections: the unlock silently returns false and the lock + // leaks, while a competing acquire that happens to draw the holding + // connection re-enters the lock and two pushes to one repo run concurrently. + // Hence: keep the connection that WON. + // * Occupancy. Holding a connection across the ~60 one-second sleeps would let + // one spinning acquire park a lock-pool connection for a minute. That is not + // just a push-path concern: `api/issues.rs` and `api/pulls.rs` reach + // acquire_write holding no concurrency permit at all, so a caller could park + // the whole pool and starve authenticated pushes on every repo (#173 F1). + // Hence: return the connection when we LOSE, before sleeping. + // + // Returning a losing connection is safe with respect to the cancellation design: + // `after_release` runs `pg_advisory_unlock_all()`, a no-op on a connection that + // took nothing, so it cannot disturb a lock held by any other connection + // (proven by `returning_an_unlocked_connection_does_not_clear_another_connections_lock`). + // + // Cancellation safety is unchanged: the future can only be dropped while a + // connection is checked out, and dropping it runs the same `after_release` hook, + // which clears whatever lock it had just taken (#173 U1). + let mut lock_conn = None; for attempt in 0..60 { - let c = guard - .conn - .as_deref_mut() - .expect("write guard holds its connection during acquisition"); + let mut conn = self.lock_pool.acquire().await.map_err(|e| { + anyhow::Error::new(LockPoolBusy) + .context(format!("checking out a lock-pool connection: {e}")) + })?; let row: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") .bind(lock_key) - .fetch_one(&mut *c) + .fetch_one(&mut *conn) .await .context("trying advisory lock")?; if row.0 { - acquired = true; + lock_conn = Some(conn); break; } + // Lost the race: give the connection back so a spinning acquire occupies + // nothing while it waits. + drop(conn); if attempt < 59 { tokio::time::sleep(std::time::Duration::from_secs(1)).await; } } - if !acquired { + let Some(lock_conn) = lock_conn else { anyhow::bail!("could not acquire advisory lock after 60s — possible stale lock for {owner_slug}/{repo_name}"); + }; + + #[cfg(test)] + if let Some(stall) = self.tigris_stall { + tokio::time::sleep(stall).await; } - guard.locked = true; // Always download the latest from Tigris before writing. Local disk may be - // stale if another machine pushed since our last access. The guard already - // owns the lock + its connection, so a cancellation here drops through Drop. + // stale if another machine pushed since our last access. The lock connection + // is already held, so a cancellation here returns it through `after_release`, + // which clears the lock. if let Some(ref tigris) = self.tigris { if tigris.exists(&owner_slug, repo_name).await.unwrap_or(false) { debug!(repo = %repo_name, "write acquire: downloading latest from tigris"); @@ -276,7 +343,19 @@ impl RepoStore { } } - Ok(guard) + Ok(RepoWriteGuard { + owner_slug, + repo_name: repo_name.to_string(), + local_path, + lock_key, + lock_conn: Some(lock_conn), + released: false, + tigris: self.tigris.clone(), + #[cfg(test)] + upload_site_reached: Arc::clone(&self.upload_site_reached), + #[cfg(test)] + test_pre_unlock_gate: self.pre_unlock_gate.clone(), + }) } /// Initialize a new bare repo on local disk and upload to Tigris. @@ -329,40 +408,61 @@ impl RepoStore { /// segment is rejected. This is the CodeQL-recognised barrier /// pattern for `rust/path-injection`. fn local_path(&self, owner_did: &str, repo_name: &str) -> Result<(String, PathBuf)> { - validate_path_components(owner_did, repo_name)?; - let owner_slug = owner_did.replace([':', '/'], "_"); - let local_path = self - .repos_dir - .join(&owner_slug) - .join(format!("{repo_name}.git")); - - if !local_path.starts_with(&self.repos_dir) { - anyhow::bail!( - "computed repo path escaped repos_dir: {}", - local_path.display() - ); - } + let local_path = validated_repo_disk_path(&self.repos_dir, owner_did, repo_name)?; + Ok((owner_slug, local_path)) + } +} - // Explicit component walk — sanitisation barrier that static analysers - // (CodeQL `rust/path-injection`) recognise. The path must be composed - // entirely of Normal segments after the root prefix; any ParentDir or - // CurDir component is a traversal attempt. - for component in local_path.components() { - use std::path::Component; - match component { - Component::Prefix(_) | Component::RootDir | Component::Normal(_) => {} - Component::ParentDir => { - anyhow::bail!("path contains parent-directory component"); - } - Component::CurDir => { - anyhow::bail!("path contains current-directory component"); - } +/// The three-layer validated form of `store::repo_disk_path`, with NO Tigris fetch and +/// no `RepoStore` (#173 round 11, F3). Extracted from `RepoStore::local_path` so a +/// second caller that must not pull a cold repo, the U4 legacy provider-CID sweep, gets +/// the same barrier instead of the raw join. `local_path` is now a thin wrapper over +/// this, so the two cannot drift. +/// +/// Three-layer defence against path traversal: +/// 1. Strict allowlist on `owner_did` and `repo_name` (no `..`, slashes, +/// null bytes, leading dots; length-bounded). +/// 2. The joined path must remain rooted at `repos_dir`. +/// 3. Every component of the joined path must be `Component::Normal` +/// (or the prefix/root from `repos_dir`); any `ParentDir`/`CurDir` +/// segment is rejected. This is the CodeQL-recognised barrier +/// pattern for `rust/path-injection`. +pub(crate) fn validated_repo_disk_path( + repos_dir: &Path, + owner_did: &str, + repo_name: &str, +) -> Result { + validate_path_components(owner_did, repo_name)?; + + let owner_slug = owner_did.replace([':', '/'], "_"); + let local_path = repos_dir.join(&owner_slug).join(format!("{repo_name}.git")); + + if !local_path.starts_with(repos_dir) { + anyhow::bail!( + "computed repo path escaped repos_dir: {}", + local_path.display() + ); + } + + // Explicit component walk — sanitisation barrier that static analysers + // (CodeQL `rust/path-injection`) recognise. The path must be composed + // entirely of Normal segments after the root prefix; any ParentDir or + // CurDir component is a traversal attempt. + for component in local_path.components() { + use std::path::Component; + match component { + Component::Prefix(_) | Component::RootDir | Component::Normal(_) => {} + Component::ParentDir => { + anyhow::bail!("path contains parent-directory component"); + } + Component::CurDir => { + anyhow::bail!("path contains current-directory component"); } } - - Ok((owner_slug, local_path)) } + + Ok(local_path) } /// Strict allowlist validator for `owner_did` and `repo_name`. @@ -544,6 +644,19 @@ fn validate_repo_name(repo_name: &str) -> Result<()> { Ok(()) } +/// Error marker for "no lock-pool connection was available in time". +/// +/// Carried through the `anyhow` chain (like [`smart_http::GitServiceTimeout`]) so the +/// HTTP handler can `downcast_ref` it and shed a 503 + Retry-After instead of the +/// generic 500 a git error maps to: an exhausted lock pool is a CAPACITY signal, and +/// telling the client to retry shortly is the same shed semantics the surrounding +/// admission code already uses (#173 F1). +/// +/// [`smart_http::GitServiceTimeout`]: crate::git::smart_http::GitServiceTimeout +#[derive(Debug, thiserror::Error)] +#[error("no lock-pool connection available")] +pub struct LockPoolBusy; + /// Guard returned by `acquire_write()`. Holds the Postgres advisory lock and /// uploads to Tigris + releases the lock on `release()`. pub struct RepoWriteGuard { @@ -551,24 +664,36 @@ pub struct RepoWriteGuard { repo_name: String, pub local_path: PathBuf, lock_key: i64, - /// The pooled connection that took the advisory lock. Session-level pg - /// advisory locks are connection-affine, so the guard pins that connection - /// for its whole lifetime and unlocks on it (in `release`, or the `Drop` - /// backstop). `None` only after the connection has been taken, either to run - /// the detached unlock in `Drop` or to be closed when `release`'s unlock - /// errored (#174 F3b). - conn: Option>, - /// Set once the advisory lock has actually been taken. A guard dropped - /// before the lock is held (or after `release`) performs no unlock. - locked: bool, - /// Set once `release` has run its unlock, making the `Drop` backstop inert. + /// The lock-pool connection that TOOK the advisory lock. It must be the one + /// that releases it (session locks are owned by their connection), and + /// holding it here is also what makes a guard dropped without `release` + /// safe: the drop returns the connection through the pool's `after_release` + /// hook, which runs `pg_advisory_unlock_all()`. + /// + /// `Option` because that hook is not a complete answer. When the unlock ERRORS + /// on a live session (a statement timeout, an admin cancel, an aborted + /// transaction), `after_release` issues its `pg_advisory_unlock_all()` on the + /// SAME broken session and it fails too, so the connection goes back to the pool + /// still holding the lock and nothing ever clears it (measured: never freed in + /// 15s, #174 F3b). Those paths `take()` the connection and close it instead; + /// ending the session is what actually frees the lock. `None` only after such a + /// disposal, or after `Drop` has moved it into the detached unlock. + lock_conn: Option>, + /// Set once `release` has run its unlock, making the `Drop` backstop inert. A + /// guard is only ever constructed with the lock already held, so there is no + /// "never locked" state to track alongside it. released: bool, tigris: Option, - /// Test-only seam: when set, `release` parks on this gate at the exact point - /// it is about to await `pg_advisory_unlock` (connection still owned, not yet - /// released). Dropping the `release` future while it is parked reproduces a - /// mid-unlock cancellation, so a test can assert the `Drop` backstop still - /// frees the session lock. Never set outside tests. + /// Shared with the store that handed this guard out; see + /// [`RepoStore::upload_site_reached`]. + #[cfg(test)] + upload_site_reached: Arc, + /// Test-only seam: when set, `release` parks on this gate at the exact point it + /// is about to await `pg_advisory_unlock` (connection still owned, not yet + /// returned to the lock pool). Dropping the `release` future while it is parked + /// reproduces a mid-unlock cancellation, so a test can assert the lock is still + /// freed: the drop returns the connection through the pool's `after_release` + /// hook, which runs `pg_advisory_unlock_all()`. Never set outside tests. #[cfg(test)] test_pre_unlock_gate: Option>, } @@ -623,6 +748,13 @@ impl RepoWriteGuard { pub async fn release(mut self, success: bool) { // Upload to Tigris only on success. if success { + // The upload site, recorded for tests before the client is consulted: with + // no injectable seam on `TigrisClient` a counter inside the arm below could + // never move, and it is reaching this point at all that an interrupted push + // must not do (#173 F2). + #[cfg(test)] + self.upload_site_reached + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); if let Some(ref tigris) = self.tigris { if let Err(e) = tigris .upload(&self.owner_slug, &self.repo_name, &self.local_path) @@ -635,82 +767,64 @@ impl RepoWriteGuard { warn!(repo = %self.repo_name, "write failed — skipping tigris upload to avoid propagating an inconsistent repo"); } - // Release the advisory lock on the SAME connection that took it (session - // advisory locks are connection-affine). Unlock through the connection - // while it is STILL owned by `self` — do not `take()` it first. If this - // future is cancelled during the unlock await, `self` is dropped with - // `conn == Some(..)` and `released == false`, so the `Drop` backstop still - // runs the detached unlock. `released` is set only AFTER the await - // resolves, so a cancellation cannot make the backstop inert (#174 F4). - if self.locked { - #[cfg(test)] - let pre_unlock_gate = self.test_pre_unlock_gate.clone(); - let unlock = if let Some(conn) = self.conn.as_deref_mut() { - // Test-only: park right before the unlock await so a test can drop - // this future mid-unlock (connection owned, not yet released). - #[cfg(test)] - if let Some(gate) = pre_unlock_gate { - gate.notified().await; - } - Some( - sqlx::query("SELECT pg_advisory_unlock($1)") - .bind(self.lock_key) - .execute(&mut *conn) - .await, - ) - } else { - None - }; - // An unlock that ERRORS is a different failure from a cancellation: the - // await resolved, so `Drop` is about to be made inert by `released` - // below, but the session is still alive and still holds the lock - // (statement timeout, admin cancel, aborted transaction). Returning that - // `PoolConnection` to the pool would hand the next caller a connection - // holding a lock nobody tracks (#174 F3b). Connection disposal is the - // single mechanism here, and it is why we do not instead try to keep the - // `Drop` backstop armed: disposal needs `conn.take()`, and `Drop` - // early-returns on `conn == None`. Ending the session is what frees the - // lock, so `released = true` still holds. - if let Some(Err(e)) = unlock { - warn!(repo = %self.repo_name, err = %e, - "advisory unlock failed, closing the connection so the session ends and postgres drops the lock"); - if let Some(conn) = self.conn.take() { - // `close()` over `detach()`: both consume the `PoolConnection` by - // value in sqlx 0.8.6, but we are in an async fn, so `close()` - // sends Terminate and waits for the socket to go down before - // `release` returns. `detach()` would only end the session - // whenever the returned `PgConnection` is dropped and its - // background close completes. If this future is cancelled during - // `close()`, the connection is dropped mid-close, which still - // tears the session down. That last point is also why the await is - // safe to bound: see `close_conn_bounded`, which gives it the - // deadline sqlx does not. - close_conn_bounded(&self.repo_name, conn.close()).await; - } + // Test-only: park right before the unlock await so a test can drop this + // future mid-unlock, with the connection still owned. + #[cfg(test)] + if let Some(gate) = self.test_pre_unlock_gate.clone() { + gate.notified().await; + } + // Release the advisory lock on the connection that took it. Anything else + // (a fresh `&pool` checkout) is a no-op that returns false: Postgres + // scopes a session lock to its owning connection. + // + // Unlock through the connection while it is STILL owned by `self`; do not + // `take()` it first. A cancellation during this await then drops `self` with + // the connection still in place, so it returns to the lock pool and + // `after_release` clears the lock (#174 F4). + let unlock = match self.lock_conn.as_deref_mut() { + Some(conn) => Some( + sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(self.lock_key) + .execute(&mut *conn) + .await, + ), + None => None, + }; + // An unlock that ERRORS is a different failure from a cancellation: the await + // resolved, so the session is alive and still holds the lock. Returning that + // connection to the pool does NOT recover it, because `after_release` runs its + // `pg_advisory_unlock_all()` on the same broken session and fails identically + // (#174 F3b). Close it: ending the session is what frees the lock. + if let Some(Err(e)) = unlock { + warn!(repo = %self.repo_name, err = %e, + "advisory unlock failed, closing the connection so the session ends and postgres drops the lock"); + if let Some(conn) = self.lock_conn.take() { + close_conn_bounded(&self.repo_name, conn.close()).await; } } + // On the clean path, dropping `self` returns the connection to the lock pool, + // where `after_release` sweeps anything the unlock above missed. self.released = true; } } impl Drop for RepoWriteGuard { - /// Cancellation-safe backstop: if the guard is dropped while still holding the - /// advisory lock (a `tokio::time::timeout` cancelled `acquire_write`, or a - /// handler future was dropped before `release`), unlock on the pinned - /// connection. This is NOT the backstop for an unlock that ran and returned an - /// error: that case is closed inside `release` by disposing of the connection, - /// because `Drop` early-returns on `conn == None` and the two mechanisms cannot - /// both apply (#174 F3b). `Drop` cannot await, so spawn a detached unlock — it runs on the - /// same session (connection-affine). An off-runtime drop has nothing to spawn onto, - /// so it disposes of the connection instead. On runtime - /// SHUTDOWN the spawned unlock task may be dropped before it polls, so the unlock - /// may not run — but shutdown tears down the pool, and closing the connection - /// releases the session-level advisory lock server-side, so this too is bounded. + /// Backstop for a guard dropped WITHOUT `release` (a cancelled `acquire_write`, a + /// handler future dropped before the release call). The pool's `after_release` + /// hook covers the ordinary case on its own, but not one: if the detached unlock + /// ERRORS on a live session, the hook's `pg_advisory_unlock_all()` fails the same + /// way and the connection returns to the pool still holding the lock (#174 F3b). + /// So the unlock runs here and disposes of the connection when it errors. + /// + /// `Drop` cannot await, so the unlock is spawned; it runs on the same session, + /// which is what makes it effective. With no runtime to spawn onto there is + /// nothing that can unlock, so the connection is detached and dropped instead: + /// closing the socket ends the session, and that frees the lock server-side. fn drop(&mut self) { - if self.released || !self.locked { + if self.released { return; } - let Some(mut conn) = self.conn.take() else { + let Some(mut conn) = self.lock_conn.take() else { return; }; let lock_key = self.lock_key; @@ -722,12 +836,10 @@ impl Drop for RepoWriteGuard { .bind(lock_key) .execute(&mut *conn) .await; - // Same failure as `release`'s (#174 F3b), one level down: the await - // RESOLVED with an error, so the session is alive and still holds - // the lock. Letting this async block end here would drop `conn` and - // RETURN it to the pool, handing the next caller a connection - // holding a lock nobody tracks. Close it instead, which both keeps - // it out of the pool and ends the session that holds the lock. + // Same failure as `release`'s, one level down: the await RESOLVED + // with an error, so the session is alive and still holds the lock. + // Ending this block would drop `conn` and RETURN it to the pool, + // where `after_release` fails identically. Close it instead. if let Err(e) = unlock { warn!(repo = %repo_name, err = %e, "detached advisory-unlock on write-guard drop failed, closing the connection so the session ends and postgres drops the lock"); close_conn_bounded(&repo_name, conn.close()).await; @@ -735,15 +847,10 @@ impl Drop for RepoWriteGuard { }); } Err(_) => { - // No runtime to spawn the unlock onto, and the connection is already - // out of the guard, so there is no path that unlocks on this session. - // Returning it to the pool would hand the next caller a connection - // still holding the lock. `PoolConnection`'s own drop also spawns its - // return-to-pool task, which panics with no runtime. `detach` gives up - // the pool slot and yields a plain `PgConnection`; dropping that closes - // the socket, which ends the session and is what frees the lock - // server-side. `Drop` cannot await, so this is the whole disposal: - // `close_conn_bounded` is not available here. + // `PoolConnection`'s own drop spawns its return-to-pool task, which + // panics with no runtime. `detach` gives up the pool slot and yields a + // plain `PgConnection`; dropping that closes the socket, which ends the + // session and is what frees the lock. drop(conn.detach()); warn!( repo = %repo_name, @@ -756,6 +863,47 @@ impl Drop for RepoWriteGuard { } } +/// Build the dedicated advisory-lock pool a `RepoStore` runs its write locks on. +/// Connect options are cloned off an existing pool so callers need not re-parse +/// the database URL; the pool is lazy, so no connection is opened here. +/// +/// Two properties, both load-bearing: +/// +/// * The `after_release` hook runs `pg_advisory_unlock_all()` before a +/// connection goes back into the pool. sqlx's `PoolConnection::drop` spawns +/// `return_to_pool()`, which invokes this hook, so a connection dropped by +/// CANCELLATION still clears its locks. That is what keeps an `acquire_write` +/// killed mid-Tigris by the caller's `tokio::time::timeout` from leaking a +/// lock and wedging every later push to that repo (#173). Note the hook runs +/// from that spawned task, so the unlock is asynchronous with respect to the +/// drop: the lock clears shortly after the connection goes away, not +/// synchronously with it. +/// * It is a SEPARATE pool from the main query pool, not a slice of it. A push +/// holds its lock connection for the whole receive-pack, so drawing these +/// from the main pool would let a burst of `max_concurrent_git_pushes` +/// pushes park that many query connections for the length of their +/// receive-packs and starve every other query. That is true at any pool +/// size, so the separation does not rest on how the two knobs are set; +/// `Config::validate` separately requires `db_max_connections` to clear +/// `max_concurrent_git_pushes` by `DB_POOL_APP_HEADROOM`. +/// +/// `acquire_timeout` bounds the wait when every lock-pool connection is busy, so +/// exhaustion surfaces as a clean error rather than an unbounded hang. +pub fn build_lock_pool(source: &PgPool, max_connections: u32, acquire_timeout: Duration) -> PgPool { + PgPoolOptions::new() + .max_connections(max_connections) + .acquire_timeout(acquire_timeout) + .after_release(|conn, _meta| { + Box::pin(async move { + sqlx::query("SELECT pg_advisory_unlock_all()") + .execute(&mut *conn) + .await?; + Ok(true) + }) + }) + .connect_lazy_with((*source.connect_options()).clone()) +} + /// Compute a stable i64 hash for a Postgres advisory lock key. /// /// Uses SHA-256 (not `DefaultHasher`) so the same `(owner_slug, repo_name)` @@ -783,6 +931,503 @@ pub(crate) fn advisory_lock_key(owner_slug: &str, repo_name: &str) -> i64 { #[cfg(test)] mod tests { use super::*; + use std::time::Duration; + + // ── advisory-lock test helpers (#173 U1) ─────────────────────────────── + + /// Postgres advisory locks live in a CLUSTER-wide space, not a per-database + /// one, so two `#[sqlx::test]` cases running against their own temporary + /// databases still share the key space. Every lock test therefore mints its + /// own key instead of reusing a fixed constant. + fn unique_lock_key() -> i64 { + use std::sync::atomic::{AtomicI64, Ordering}; + static NEXT: AtomicI64 = AtomicI64::new(0); + let n = NEXT.fetch_add(1, Ordering::Relaxed); + ((std::process::id() as i64) << 24) | (n & 0xff_ffff) + } + + /// A plain pool (no `after_release` hook, no idle timeout) at the same + /// database as the `#[sqlx::test]` pool. Two separate reasons these tests + /// cannot just use the pool the harness hands them: + /// + /// 1. Observing lock state has to happen from a session that is definitely + /// not the one under test. Session advisory locks are re-entrant, so + /// `pg_try_advisory_lock` on the very connection that already holds the key + /// returns true, and a same-pool probe silently reports a leaked lock free. + /// 2. The harness pool sets `idle_timeout(1s)`, so a connection returned to it + /// is closed about a second later and Postgres drops every lock that + /// session held. That would mask exactly the leak these tests exist to + /// catch, so the store under test runs on one of these too. + fn sibling_pool(pool: &PgPool, max_connections: u32) -> PgPool { + sqlx::postgres::PgPoolOptions::new() + .max_connections(max_connections) + .connect_lazy_with((*pool.connect_options()).clone()) + } + + /// Probe the lock from a connection that is NOT the one under test. Session + /// advisory locks are re-entrant within their own session, so a check from the + /// holding connection would pass vacuously and prove nothing. + async fn lock_is_free_elsewhere(pool: &PgPool, key: i64) -> bool { + let mut probe = pool.acquire().await.expect("probe connection"); + let taken: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut *probe) + .await + .expect("probe try-lock"); + if taken.0 { + sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *probe) + .await + .expect("probe unlock"); + } + taken.0 + } + + /// `after_release` runs from the task sqlx spawns in `PoolConnection::drop`, + /// so the unlock is ASYNCHRONOUS with respect to the drop. Callers must poll + /// rather than assume the lock is gone the instant the connection goes away. + async fn wait_until_free(pool: &PgPool, key: i64, within: Duration) -> bool { + let deadline = std::time::Instant::now() + within; + loop { + if lock_is_free_elsewhere(pool, key).await { + return true; + } + if std::time::Instant::now() >= deadline { + return false; + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + } + + // ── DESIGN GATE ──────────────────────────────────────────────────────── + // The whole cancellation-safety design rests on one sqlx behaviour: + // `PoolConnection::drop` spawns `return_to_pool()`, which invokes the pool's + // `after_release` hook before the connection is reused. If that holds, a + // connection dropped by cancellation still runs `pg_advisory_unlock_all()` + // and the lock cannot leak. This test proves it by execution, through the + // production `build_lock_pool` so that stripping the hook there turns it red. + + #[sqlx::test] + async fn dropped_pool_connection_runs_after_release_and_clears_locks(pool: PgPool) { + let key = unique_lock_key(); + let lock_pool = build_lock_pool(&pool, 4, Duration::from_secs(5)); + + { + let mut conn = lock_pool.acquire().await.expect("lock-pool connection"); + let taken: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut *conn) + .await + .expect("try-lock"); + assert!(taken.0, "first try-lock must succeed"); + assert!( + !lock_is_free_elsewhere(&pool, key).await, + "lock must be observably HELD from another session while the connection lives" + ); + // Drop WITHOUT calling pg_advisory_unlock: this models cancellation. + } + + assert!( + wait_until_free(&pool, key, Duration::from_secs(5)).await, + "after_release must clear the advisory lock of a dropped connection" + ); + } + + // ── acquire_write cancellation safety (#173 U1) ──────────────────────── + + /// The reviewer's named regression. `api/repos.rs` wraps `acquire_write` in a + /// `tokio::time::timeout`; when that fires during the Tigris phase the future + /// is dropped after the advisory lock was taken and before `RepoWriteGuard` + /// (the only thing that unlocks) exists. The lock then leaks and every later + /// push to the same repo spins the 60-attempt / 60s ceiling and fails. + #[sqlx::test] + async fn cancelled_acquire_write_mid_tigris_does_not_leak_the_lock(pool: PgPool) { + let repos_dir = PathBuf::from("/tmp/gitlawb-test-repos"); + let owner = "did:key:z6MkCancelMidTigris"; + let repo = "cancel-mid-tigris"; + + let store_pool = sibling_pool(&pool, 8); + let stalling = RepoStore::for_testing(repos_dir.clone(), store_pool.clone()) + .with_tigris_stall(Duration::from_secs(30)); + let cancelled = tokio::time::timeout( + Duration::from_millis(500), + stalling.acquire_write(owner, repo), + ) + .await; + assert!( + cancelled.is_err(), + "the acquire must still be inside the Tigris phase when the timeout fires" + ); + + // Observed from an independent session, so the check cannot be satisfied + // by re-entrancy on whichever pooled connection happens to be handed back. + let probe = sibling_pool(&pool, 2); + let key = advisory_lock_key(&owner.replace([':', '/'], "_"), repo); + assert!( + wait_until_free(&probe, key, Duration::from_secs(5)).await, + "a cancelled acquire_write must leave no advisory lock held" + ); + + // A subsequent acquire for the SAME repo must succeed promptly. Before the + // fix it blocks on the leaked lock until the 60-attempt ceiling. + let store = RepoStore::for_testing(repos_dir, store_pool); + let guard = tokio::time::timeout(Duration::from_secs(5), store.acquire_write(owner, repo)) + .await + .expect("second acquire_write must not block on a leaked lock") + .expect("second acquire_write must succeed"); + guard.release(false).await; + } + + /// Cancellation BEFORE the lock is taken must leave nothing behind: no lock, + /// and no lock-pool connection stranded. The lock pool here holds exactly one + /// connection, so a stranded one would make the follow-up acquire time out + /// waiting for a checkout. + #[sqlx::test] + async fn cancelled_acquire_write_before_the_lock_leaves_nothing_held(pool: PgPool) { + let probe = sibling_pool(&pool, 2); + let owner = "did:key:z6MkCancelEarly"; + let repo = "cancel-early"; + let key = advisory_lock_key(&owner.replace([':', '/'], "_"), repo); + + let store = RepoStore::new( + PathBuf::from("/tmp/gitlawb-test-repos"), + None, + build_lock_pool(&pool, 1, Duration::from_secs(3)), + ); + + // A zero deadline polls the future once, which gets it no further than the + // first await (the pool checkout / the first try-lock round trip), so it is + // cancelled before any lock can be taken. + let cancelled = + tokio::time::timeout(Duration::ZERO, store.acquire_write(owner, repo)).await; + assert!(cancelled.is_err(), "the acquire must be cancelled"); + + assert!( + lock_is_free_elsewhere(&probe, key).await, + "no lock may be held when the acquire never got that far" + ); + + // The single lock-pool connection must be back: if cancellation stranded + // it, this checkout blocks until the 3s acquire timeout and fails. + let guard = tokio::time::timeout(Duration::from_secs(2), store.acquire_write(owner, repo)) + .await + .expect("the lock-pool connection must have been returned") + .expect("acquire after cancellation"); + guard.release(false).await; + } + + /// Lock-pool exhaustion is a bounded wait and a clean error, never a panic and + /// never an unbounded hang. + #[sqlx::test] + async fn lock_pool_exhaustion_is_a_bounded_error(pool: PgPool) { + let owner = "did:key:z6MkExhaustion"; + let store = RepoStore::new( + PathBuf::from("/tmp/gitlawb-test-repos"), + None, + build_lock_pool(&pool, 1, Duration::from_secs(2)), + ); + + let held = store + .acquire_write(owner, "exhaust-a") + .await + .expect("first acquire"); + + // Different repo, so this is not the advisory lock queueing: the only + // connection in the lock pool is checked out by `held`. + let started = std::time::Instant::now(); + let err = tokio::time::timeout( + Duration::from_secs(10), + store.acquire_write(owner, "exhaust-b"), + ) + .await + .expect("the wait must be bounded by the pool acquire timeout"); + let err = match err { + Ok(_) => panic!("an exhausted lock pool must surface an error, not a guard"), + Err(e) => e, + }; + assert!( + started.elapsed() < Duration::from_secs(6), + "the error must arrive on the acquire timeout, not after a long hang" + ); + assert!( + err.to_string().contains("lock-pool connection"), + "the error must name the lock-pool checkout, got: {err}" + ); + + held.release(false).await; + } + + /// #173 F1 (RED-before/GREEN-after). A contended `acquire_write` spins for up to + /// 60 one-second attempts. It must not OCCUPY a lock-pool connection for that whole + /// spin: `acquire_write` has non-push callers (`api/issues.rs`, `api/pulls.rs`) that + /// hold no concurrency permit, so any self-minted did:key could otherwise park a + /// connection per call and starve authenticated pushes on EVERY repo. + /// + /// Lock pool of exactly 2, two spinners. Pre-fix (checkout hoisted above the retry + /// loop) they pin both connections for the full spin and an UNCONTENDED acquire on a + /// third repo dies on the pool acquire timeout. Post-fix each spinner returns its + /// connection before sleeping, so it occupies ~0 and the uncontended acquire sails + /// through. + #[sqlx::test] + async fn a_spinning_acquire_write_does_not_occupy_a_lock_pool_connection(pool: PgPool) { + let owner = "did:key:z6MkSpinOccupancy"; + let owner_slug = owner.replace([':', '/'], "_"); + let store = RepoStore::new( + PathBuf::from("/tmp/gitlawb-test-repos"), + None, + build_lock_pool(&pool, 2, Duration::from_secs(2)), + ); + + // An independent session holds both contended keys, so the spinners' try-locks + // return false on every iteration and they stay in the retry loop. + let holder = sibling_pool(&pool, 2); + let mut held_conn = holder.acquire().await.expect("holder connection"); + for repo in ["spin-a", "spin-b"] { + let taken: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(advisory_lock_key(&owner_slug, repo)) + .fetch_one(&mut *held_conn) + .await + .expect("holder try-lock"); + assert!(taken.0, "the holder must own {repo}'s key"); + } + + let mut spinners = Vec::new(); + for repo in ["spin-a", "spin-b"] { + let store = store.clone(); + spinners.push(tokio::spawn(async move { + store.acquire_write(owner, repo).await + })); + } + // Let both reach the spin (each has done at least one failed try-lock by now). + tokio::time::sleep(Duration::from_millis(500)).await; + + let started = std::time::Instant::now(); + let uncontended = tokio::time::timeout( + Duration::from_secs(10), + store.acquire_write(owner, "spin-free"), + ) + .await + .expect("the uncontended acquire must return, not hang"); + let elapsed = started.elapsed(); + let free_guard = uncontended.unwrap_or_else(|e| { + panic!( + "an UNCONTENDED acquire_write on a DIFFERENT repo must not be starved by \ + spinners holding the lock pool; got: {e}" + ) + }); + assert!( + elapsed < Duration::from_secs(2), + "the uncontended acquire must not queue behind the spinners for the pool \ + acquire timeout; took {elapsed:?}" + ); + free_guard.release(false).await; + + // The drop-and-retake cycle must still END in a real, exclusive lock: free + // spin-a's key and the spinner that was cycling connections must take it. + sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(advisory_lock_key(&owner_slug, "spin-a")) + .execute(&mut *held_conn) + .await + .expect("release spin-a"); + let winner = tokio::time::timeout(Duration::from_secs(15), spinners.remove(0)) + .await + .expect("the spinner must finish once its key frees") + .expect("spinner task") + .expect("the spinner must acquire once the key frees"); + let probe = sibling_pool(&pool, 2); + assert!( + !lock_is_free_elsewhere(&probe, advisory_lock_key(&owner_slug, "spin-a")).await, + "the lock a spinner finally took must be observably held from another session" + ); + winner.release(false).await; + + for s in spinners { + s.abort(); + } + sqlx::query("SELECT pg_advisory_unlock_all()") + .execute(&mut *held_conn) + .await + .expect("release the remaining holder lock"); + } + + /// #173 F1, the property the fix rests on: returning a lock-pool connection that + /// holds NOTHING runs `after_release`'s `pg_advisory_unlock_all()`, which is a no-op + /// and must not disturb a lock held on a DIFFERENT connection of the same pool. + /// Session advisory locks are per connection, so this is by construction, but the + /// spin fix depends on it, so it is proven by execution rather than assumed. + #[sqlx::test] + async fn returning_an_unlocked_connection_does_not_clear_another_connections_lock( + pool: PgPool, + ) { + let owner = "did:key:z6MkNoOpUnlockAll"; + let repo = "noop-unlock"; + let key = advisory_lock_key(&owner.replace([':', '/'], "_"), repo); + let probe = sibling_pool(&pool, 2); + let lock_pool = build_lock_pool(&pool, 4, Duration::from_secs(5)); + let store = RepoStore::new( + PathBuf::from("/tmp/gitlawb-test-repos"), + None, + lock_pool.clone(), + ); + + let guard = store.acquire_write(owner, repo).await.expect("acquire"); + + // Churn the pool: check out and drop connections that hold no lock, exactly what + // a spinning acquire now does between attempts. Each return fires + // pg_advisory_unlock_all() on that connection. + for _ in 0..10 { + let mut conn = lock_pool.acquire().await.expect("churn checkout"); + let _: (i32,) = sqlx::query_as("SELECT 1") + .fetch_one(&mut *conn) + .await + .expect("churn query"); + drop(conn); + tokio::time::sleep(Duration::from_millis(10)).await; + } + + assert!( + !lock_is_free_elsewhere(&probe, key).await, + "a held write lock must survive other lock-pool connections being returned" + ); + guard.release(true).await; + assert!( + lock_is_free_elsewhere(&probe, key).await, + "release must still free the lock after the churn" + ); + } + + /// #173 F1: lock-pool exhaustion is a DISTINCT error the handler can shed as a 503, + /// not a generic git 500. Both directions: an exhausted pool downcasts to + /// [`LockPoolBusy`], and an unrelated failure (a rejected repo name) does not. + #[sqlx::test] + async fn lock_pool_exhaustion_is_a_distinct_downcastable_error(pool: PgPool) { + let owner = "did:key:z6MkBusyDowncast"; + let store = RepoStore::new( + PathBuf::from("/tmp/gitlawb-test-repos"), + None, + build_lock_pool(&pool, 1, Duration::from_secs(1)), + ); + let held = store + .acquire_write(owner, "busy-a") + .await + .expect("first acquire"); + + let err = match store.acquire_write(owner, "busy-b").await { + Ok(_) => panic!("an exhausted lock pool must error, not hand back a guard"), + Err(e) => e, + }; + assert!( + err.downcast_ref::().is_some(), + "lock-pool exhaustion must be downcastable so the handler sheds 503, got: {err}" + ); + + // MUST-NOT: an ordinary rejection is not a capacity signal. + let other = match store.acquire_write(owner, "../escape").await { + Ok(_) => panic!("a traversal repo name must be rejected"), + Err(e) => e, + }; + assert!( + other.downcast_ref::().is_none(), + "a validation failure must not masquerade as lock-pool capacity, got: {other}" + ); + + held.release(false).await; + } + + /// Round trip: the lock is observably HELD between acquire and release, and + /// observably FREE after. Both checks run from an independent session; from + /// the holding session they would pass vacuously (session locks are + /// re-entrant) and would not notice an unlock that landed on the wrong + /// connection. + #[sqlx::test] + async fn acquire_write_holds_the_lock_until_release(pool: PgPool) { + let probe = sibling_pool(&pool, 2); + let owner = "did:key:z6MkRoundTrip"; + let repo = "round-trip"; + let key = advisory_lock_key(&owner.replace([':', '/'], "_"), repo); + + let store = RepoStore::for_testing(PathBuf::from("/tmp/gitlawb-test-repos"), pool.clone()); + let guard = store.acquire_write(owner, repo).await.expect("acquire"); + assert!( + !lock_is_free_elsewhere(&probe, key).await, + "the lock must be held while the guard is alive" + ); + + // No polling here, deliberately. `release` must free the lock SYNCHRONOUSLY, + // which it can only do by unlocking on the connection that took it; a + // `pg_advisory_unlock` sent through the pool would land on some other + // session and return false. The `after_release` hook is a net for the + // cancellation path and fires from a spawned task well after this point, so + // it must not be what makes this assertion pass. + guard.release(true).await; + assert!( + lock_is_free_elsewhere(&probe, key).await, + "release must free the lock as seen from another session" + ); + } + + /// `release(false)` skips the Tigris upload but must still free the lock; a + /// failed write that kept the lock would wedge the repo. + #[sqlx::test] + async fn release_after_failed_write_still_frees_the_lock(pool: PgPool) { + let probe = sibling_pool(&pool, 2); + let owner = "did:key:z6MkFailedWrite"; + let repo = "failed-write"; + let key = advisory_lock_key(&owner.replace([':', '/'], "_"), repo); + + let store = RepoStore::for_testing(PathBuf::from("/tmp/gitlawb-test-repos"), pool.clone()); + let guard = store.acquire_write(owner, repo).await.expect("acquire"); + guard.release(false).await; + + assert!( + wait_until_free(&probe, key, Duration::from_secs(5)).await, + "release(success = false) must still free the lock" + ); + } + + /// The lock is per repo: a second acquire for the SAME repo waits for the + /// first to release, while a different repo proceeds straight through. + #[sqlx::test] + async fn same_repo_acquires_serialize_and_different_repos_do_not(pool: PgPool) { + let repos_dir = PathBuf::from("/tmp/gitlawb-test-repos"); + let owner = "did:key:z6MkSerialize"; + let store = RepoStore::for_testing(repos_dir, pool.clone()); + + let first = store + .acquire_write(owner, "serialize-a") + .await + .expect("first acquire"); + + // Different repo: unaffected by the held lock. + let other = tokio::time::timeout( + Duration::from_secs(2), + store.acquire_write(owner, "serialize-b"), + ) + .await + .expect("a different repo must not wait on this lock") + .expect("acquire other repo"); + other.release(false).await; + + // Same repo: must not acquire while `first` is alive. + let contender = tokio::spawn({ + let store = store.clone(); + async move { store.acquire_write(owner, "serialize-a").await } + }); + tokio::time::sleep(Duration::from_millis(1500)).await; + assert!( + !contender.is_finished(), + "a second acquire for the same repo must block while the first guard lives" + ); + + first.release(false).await; + let second = tokio::time::timeout(Duration::from_secs(10), contender) + .await + .expect("contender must finish once the lock is free") + .expect("contender task") + .expect("contender acquire"); + second.release(false).await; + } // ── sync slug validation (#272) ──────────────────────────────────────── @@ -1309,16 +1954,17 @@ mod tests { // ── cancellation-safe unlock (#174 F4, RED-before/GREEN-after) ────────── /// F4 (P1): a cancellation DURING the unlock await must still free the session - /// advisory lock. `release` unlocks through the connection while `self` still - /// owns it, so if the future is dropped mid-unlock, `Drop` sees `conn == Some` - /// + `locked && !released` and runs its detached-unlock backstop. A test-only - /// gate parks `release` at the exact pre-unlock point; dropping the future - /// there reproduces the cancellation. + /// advisory lock. The guard owns the lock-pool connection that took the lock, so + /// dropping the parked `release` future returns that connection to the pool, + /// where the `after_release` hook runs `pg_advisory_unlock_all()` and clears + /// whatever the interrupted unlock did not. A test-only gate parks `release` at + /// the exact pre-unlock point; dropping the future there reproduces the + /// cancellation. /// - /// Load-bearing: RED on the original ordering (`self.conn.take()` before the - /// await → at cancellation `self.conn == None` → `Drop` skips → the local - /// connection returns to the pool with the session lock still held → the - /// checker's `pg_try_advisory_lock` returns false). GREEN after the reorder. + /// Load-bearing: build the store's lock pool WITHOUT the `after_release` hook + /// and this goes RED, since the connection then returns to the pool still + /// holding the session lock and the checker's `pg_try_advisory_lock` returns + /// false. #[sqlx::test] async fn write_guard_release_cancelled_mid_unlock_frees_the_lock(pool: sqlx::PgPool) { let dir = tempfile::TempDir::new().unwrap(); @@ -1404,7 +2050,7 @@ mod tests { /// in this module and can reach `conn` directly. async fn poison_guard_connection(guard: &mut RepoWriteGuard) { let conn = guard - .conn + .lock_conn .as_deref_mut() .expect("guard holds its connection before release"); sqlx::query("BEGIN") @@ -1460,27 +2106,6 @@ mod tests { } } - /// A second pool over the same test database with the idle reaper DISABLED. - /// - /// `#[sqlx::test]`'s own pool sets `idle_timeout(1s)`, so a connection returned to - /// it is closed by the reaper about a second later, which ends the session and - /// frees the advisory lock all on its own. The old fixed 400ms sleep landed inside - /// that window by luck; polling to a deadline long enough to be flake-proof would - /// land outside it and go green whether or not `release` disposed of the - /// connection, so the tests below would stop testing anything (measured: with the - /// reaper in play, the disposal shows up at ~2s even with the fix reverted). With - /// no reaper, `release` is the only thing that can end that session, so the poll - /// measures exactly the property these two tests exist for. - async fn pool_without_idle_reaper(pool: &sqlx::PgPool) -> sqlx::PgPool { - sqlx::postgres::PgPoolOptions::new() - .max_connections(5) - .idle_timeout(None) - .max_lifetime(None) - .connect_with(pool.connect_options().as_ref().clone()) - .await - .expect("a second pool over the test database") - } - /// F3c (P2): the connection teardown on the failing-unlock path must be BOUNDED. /// `release` awaits it inline while the global write permit, the per-source permit /// and the write lease are all still held, and sqlx's `close()` carries no deadline @@ -1520,6 +2145,83 @@ mod tests { ); } + /// U8, the off-runtime arm: with no Tokio runtime there is nothing to spawn the + /// unlock onto, and the connection has already been taken out of the guard, so + /// dropping it with no unlock attempted returns it to the pool with the session + /// lock still held. Dropping a `PoolConnection` off a runtime is worse than that: + /// sqlx's return-to-pool path spawns, and its no-runtime fallback panics, so that + /// arm also panics in a destructor. + /// + /// Reached by dropping the guard on a plain `std::thread`, where + /// `Handle::try_current()` fails. + /// + /// Load-bearing: replace the `detach` arm with a plain `drop(conn)` and the join + /// sees sqlx's "requires a Tokio context" panic; `detach` gives up the pool slot, + /// so nothing is spawned and dropping the detached connection closes the socket, + /// which ends the session and frees the lock. + #[sqlx::test] + async fn write_guard_dropped_off_runtime_disposes_the_connection(pool: sqlx::PgPool) { + let dir = tempfile::TempDir::new().unwrap(); + let store_pool = pool_without_idle_reaper(&pool).await; + let store = RepoStore::for_testing(dir.path().to_path_buf(), store_pool.clone()); + let owner = "did:key:z6MkDropOffRuntimeProofKKKKKKKKKKKKKKKKKK"; + let name = "dropoffruntimetest"; + let slug = owner.replace([':', '/'], "_"); + let key = advisory_lock_key(&slug, name); + + let mut checker = pool.acquire().await.expect("checker connection"); + let guard = store.acquire_write(owner, name).await.expect("acquire"); + // The guard's connection lives in the store's DERIVED lock pool, not the pool + // handed to `for_testing`; see `RepoStore::lock_pool`. + let lock_pool = store.lock_pool().clone(); + let size_before = lock_pool.size(); + assert!(size_before > 0, "the lock pool owns the guard's connection"); + + let dropped = std::thread::spawn(move || drop(guard)).join(); + assert!( + dropped.is_ok(), + "dropping a write guard off a Tokio runtime must not panic" + ); + + wait_until( + || lock_pool.size() == size_before - 1, + "the connection of a guard dropped off a runtime to be disposed of rather \ + than returned to the pool with no unlock attempted", + ) + .await; + wait_until_lock_free( + &mut checker, + key, + "a guard dropped off a runtime to end its session so postgres drops the lock", + ) + .await; + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *checker) + .await; + } + + /// A second pool over the same test database with the idle reaper DISABLED. + /// + /// `#[sqlx::test]`'s own pool sets `idle_timeout(1s)`, so a connection returned to + /// it is closed by the reaper about a second later, which ends the session and + /// frees the advisory lock all on its own. The old fixed 400ms sleep landed inside + /// that window by luck; polling to a deadline long enough to be flake-proof would + /// land outside it and go green whether or not `release` disposed of the + /// connection, so the tests below would stop testing anything (measured: with the + /// reaper in play, the disposal shows up at ~2s even with the fix reverted). With + /// no reaper, `release` is the only thing that can end that session, so the poll + /// measures exactly the property these two tests exist for. + async fn pool_without_idle_reaper(pool: &sqlx::PgPool) -> sqlx::PgPool { + sqlx::postgres::PgPoolOptions::new() + .max_connections(5) + .idle_timeout(None) + .max_lifetime(None) + .connect_with(pool.connect_options().as_ref().clone()) + .await + .expect("a second pool over the test database") + } + /// F3b (P1): when `pg_advisory_unlock` ERRORS while the session is still alive /// (statement timeout, admin cancel, aborted transaction), the lock must not /// survive `release`. The old code discarded the error with `let _ =` and set @@ -1597,15 +2299,16 @@ mod tests { let mut guard = store.acquire_write(owner, name).await.expect("acquire"); poison_guard_connection(&mut guard).await; - let size_before = store_pool.size(); - assert!(size_before > 0, "the pool owns the guard's connection"); + let lock_pool = store.lock_pool().clone(); + let size_before = lock_pool.size(); + assert!(size_before > 0, "the lock pool owns the guard's connection"); guard.release(false).await; // The pool's size drops when the closed connection's slot is given up, which // is not synchronous with `release` returning: poll rather than sleep. wait_until( - || store_pool.size() == size_before - 1, + || lock_pool.size() == size_before - 1, "the connection that saw the unlock error to be closed rather than returned \ to the pool still holding the session lock", ) @@ -1683,14 +2386,15 @@ mod tests { let mut guard = store.acquire_write(owner, name).await.expect("acquire"); poison_guard_connection(&mut guard).await; - let size_before = store_pool.size(); - assert!(size_before > 0, "the pool owns the guard's connection"); + let lock_pool = store.lock_pool().clone(); + let size_before = lock_pool.size(); + assert!(size_before > 0, "the lock pool owns the guard's connection"); // The backstop shape: dropped without release(), with an unlock that errors. drop(guard); wait_until( - || store_pool.size() == size_before - 1, + || lock_pool.size() == size_before - 1, "the connection whose detached unlock errored to be closed rather than \ returned to the pool still holding the session lock", ) @@ -1723,19 +2427,20 @@ mod tests { let mut checker = pool.acquire().await.expect("checker connection"); let guard = store.acquire_write(owner, name).await.expect("acquire"); - let size_before = store_pool.size(); - assert!(size_before > 0, "the pool owns the guard's connection"); + let lock_pool = store.lock_pool().clone(); + let size_before = lock_pool.size(); + assert!(size_before > 0, "the lock pool owns the guard's connection"); drop(guard); // The connection goes back only once the detached unlock task has finished. wait_until( - || store_pool.num_idle() > 0, + || lock_pool.num_idle() > 0, "the detached unlock to finish and hand the connection back", ) .await; assert_eq!( - store_pool.size(), + lock_pool.size(), size_before, "a successful detached unlock must leave the connection in the pool" ); @@ -1750,94 +2455,4 @@ mod tests { .execute(&mut *checker) .await; } - - /// U8, the off-runtime arm: with no Tokio runtime there is nothing to spawn the - /// unlock onto, and the connection has already been taken out of the guard, so the - /// old code dropped it with no unlock attempted at all, back to the pool, session - /// lock still held. Dropping a `PoolConnection` off a runtime is worse than that: - /// sqlx's return-to-pool path spawns, and its no-runtime fallback panics, so the - /// old arm also panicked in a destructor. - /// - /// Reached by dropping the guard on a plain `std::thread`, where - /// `Handle::try_current()` fails. - /// - /// Load-bearing: RED before the fix (the join sees the "requires a Tokio context" - /// panic from sqlx's return-to-pool spawn), GREEN after (`detach` gives up the - /// pool slot, so nothing is spawned and dropping the detached connection closes - /// the socket, which ends the session and frees the lock). - #[sqlx::test] - async fn write_guard_dropped_off_runtime_disposes_the_connection(pool: sqlx::PgPool) { - let dir = tempfile::TempDir::new().unwrap(); - let store_pool = pool_without_idle_reaper(&pool).await; - let store = RepoStore::for_testing(dir.path().to_path_buf(), store_pool.clone()); - let owner = "did:key:z6MkDropOffRuntimeProofKKKKKKKKKKKKKKKKKK"; - let name = "dropoffruntimetest"; - let slug = owner.replace([':', '/'], "_"); - let key = advisory_lock_key(&slug, name); - - let mut checker = pool.acquire().await.expect("checker connection"); - let guard = store.acquire_write(owner, name).await.expect("acquire"); - let size_before = store_pool.size(); - assert!(size_before > 0, "the pool owns the guard's connection"); - - let dropped = std::thread::spawn(move || drop(guard)).join(); - assert!( - dropped.is_ok(), - "dropping a write guard off a Tokio runtime must not panic" - ); - - wait_until( - || store_pool.size() == size_before - 1, - "the connection of a guard dropped off a runtime to be disposed of rather \ - than returned to the pool with no unlock attempted", - ) - .await; - wait_until_lock_free( - &mut checker, - key, - "a guard dropped off a runtime to end its session so postgres drops the lock", - ) - .await; - let _ = sqlx::query("SELECT pg_advisory_unlock($1)") - .bind(key) - .execute(&mut *checker) - .await; - } - - /// F4: releasing a guard that never took the lock (`locked == false`, the state - /// acquire_write leaves after a failed acquisition) must not unlock or panic. - #[sqlx::test] - async fn write_guard_release_when_not_locked_does_not_unlock_or_panic(pool: sqlx::PgPool) { - let dir = tempfile::TempDir::new().unwrap(); - let owner = "did:key:z6MkNotLockedProofEEEEEEEEEEEEEEEEEEEEEE"; - let name = "notlockedtest"; - let slug = owner.replace([':', '/'], "_"); - let key = advisory_lock_key(&slug, name); - - let guard = RepoWriteGuard { - owner_slug: slug, - repo_name: name.to_string(), - local_path: dir.path().to_path_buf(), - lock_key: key, - conn: Some(pool.acquire().await.expect("conn")), - locked: false, - released: false, - tigris: None, - test_pre_unlock_gate: None, - }; - // Must complete without panic and issue no unlock. - guard.release(false).await; - - let mut checker = pool.acquire().await.expect("checker"); - let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") - .bind(key) - .fetch_one(&mut *checker) - .await - .unwrap(); - assert!(free, "release on an unlocked guard must not touch the key"); - let _ = sqlx::query("SELECT pg_advisory_unlock($1)") - .bind(key) - .execute(&mut *checker) - .await; - } } diff --git a/crates/gitlawb-node/src/git/smart_http.rs b/crates/gitlawb-node/src/git/smart_http.rs index 5667b576..67a85d69 100644 --- a/crates/gitlawb-node/src/git/smart_http.rs +++ b/crates/gitlawb-node/src/git/smart_http.rs @@ -14,13 +14,13 @@ use tokio::process::Command; /// the work they admitted, so admission is released only when that work is truly /// done — not the instant the handler future drops on a client disconnect. /// -/// A move-only wrapper: no methods beyond construction and `Drop`. The handler -/// MOVEs its permits in and keeps no copy (a retained copy would drop early and -/// release admission the moment the future is dropped, defeating the guard). It is -/// threaded into `drive_git_child`, whose [`KillGroupOnDrop`] moves it into the -/// detached reaper on disconnect, so both permits drop only after the process group -/// is confirmed reaped (`kill(-pgid,0)==ESRCH`) rather than while the group is still -/// alive holding PIDs past the concurrency cap (#174 P1-a, plain-spawn residual). +/// A drop-only wrapper: nothing here inspects what it holds. The handler MOVEs its +/// permits in and keeps no copy (a retained copy would drop early and release admission +/// the moment the future is dropped, defeating the guard). It is threaded into +/// `drive_git_child`, whose [`KillGroupOnDrop`] moves it into the detached reaper on +/// disconnect, so both permits drop only after the process group is confirmed reaped +/// (`kill(-pgid,0)==ESRCH`) rather than while the group is still alive holding PIDs past +/// the concurrency cap (#174 P1-a, plain-spawn residual). /// /// The `be0cdd6` path-scoped upload-pack walk already applies this discipline by /// moving its permits into the `spawn_blocking`; this generalizes it to the plain @@ -33,6 +33,8 @@ pub struct AdmissionGuard { // 'static` so the guard can move into the detached reaper task. _global: Option>, _caller: Option>, + // Any further work-scoped hold that must outlive the process group; see `with_hold`. + _hold: Option>, // Per-repo write lease (#174 U2/F3), `Some` ONLY on the receive-pack write path // (via [`with_lease`](Self::with_lease)); `None` on every read path and every // non-receive-pack write path. It rides this guard into `KillGroupOnDrop`'s detached @@ -54,10 +56,26 @@ impl AdmissionGuard { Self { _global: Some(Box::new(global)), _caller: caller.map(|c| Box::new(c) as Box), + _hold: None, _lease: None, } } + /// Attach a further hold that must not be released until the process group is + /// reaped, and ride it through the same seam as the permits. + /// + /// The push handler uses this for the repo WRITE LOCK (#173 F2). Its + /// `guard.release(..)` line is only reached if `receive_pack` returns, so on a client + /// disconnect the lock used to be freed by the dropped future while the detached + /// reaper was still giving the group its SIGTERM grace, admitting a second + /// `receive-pack` on the same repo. Carrying the lock here holds it until the group + /// is ESRCH-confirmed gone, which is the same invariant the timeout path already + /// keeps ("a caller releasing a write lock can't race them", `reap_group_on_timeout`). + pub fn with_hold(mut self, hold: impl Send + 'static) -> Self { + self._hold = Some(Box::new(hold)); + self + } + /// Attach the per-repo write lease (#174 U2/F3). Called ONLY on the receive-pack /// write path, so the lease rides the disconnect reaper; read paths never call this. pub fn with_lease(mut self, lease: crate::state::RepoWriteLease) -> Self { @@ -92,7 +110,7 @@ pub async fn info_refs( .arg("--stateless-rpc") .arg("--advertise-refs") .arg(repo_path); - // No request body — advertise-refs does not read stdin. + // No request body: advertise-refs does not read stdin. let (stdout, admission) = drive_git_child(command, Bytes::new(), timeout, "advertise-refs", admission).await?; // Single-stage op: the advertisement's group is reaped by now; release admission @@ -659,14 +677,15 @@ pub async fn build_filtered_pack( command .args(["pack-objects", "--stdout"]) .current_dir(repo_path); - drive_git_child( + let (out, admission) = drive_git_child( command, Bytes::from(data), deadline.saturating_duration_since(Instant::now()), "pack-objects", admission, ) - .await + .await?; + Ok((out, admission)) } /// Serve a clone/fetch with the withheld blobs removed from the response pack. @@ -1880,6 +1899,143 @@ mod tests { ); } + // #174 U1 (R2, KTD3, RED-before/GREEN-after): the path-scoped filtered-pack serve + // must hold read + per-caller admission until its pack-objects process group is + // reaped on a client disconnect, exactly as the plain upload_pack path does. Before + // the fix build_filtered_pack took no AdmissionGuard and the handler's `_hold` + // permits dropped the instant the request future was dropped, so disconnect-spam on + // a path-scoped repo could hold PIDs past the concurrency cap while the permits were + // already free (#174 P1-a, on the filtered path the plain path had already closed). + // + // A real AdmissionGuard built from two owned semaphore permits rides + // rev-list -> pack-objects. We drive the future until pack-objects has forked its + // grandchild (the streaming pack-writer stand-in), assert the permits are still held + // mid-serve, then DROP the future (client disconnect) and assert the permits are + // released only AFTER the group is ESRCH-confirmed gone — never while it is alive. + // Goes RED if the guard is not threaded into the pack-objects stage (it would then + // drop after rev-list, freeing the permits mid-serve or on the bare future drop). + #[cfg(unix)] + #[tokio::test] + async fn filtered_pack_holds_admission_until_group_reaped_on_disconnect() { + use std::sync::Arc; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + let pidfile = tmp.path().join("pids"); + // rev-list returns one oid fast; pack-objects forks a grandchild (the streaming + // writer stand-in), records leader+grandchild pids, then hangs so the future + // parks mid-serve with the guard owned by the pack-objects KillGroupOnDrop. The + // grandchild inherits (holds open) the stdout pipe, so drive_git_child's + // read_to_end blocks and the future stays pending until we drop it. + let body = format!( + "#!/bin/sh\ncase \"$1\" in\n rev-list) echo deadbeefdeadbeefdeadbeefdeadbeefdeadbeef ;;\n pack-objects) sleep 300 &\nprintf '%s\\n%s\\n' \"$$\" \"$!\" > \"{}\"\nwait ;;\n *) exit 1 ;;\nesac\n", + pidfile.display() + ); + let git_bin = write_fake_git(tmp.path(), &body); + let withheld = HashSet::new(); + + // Retry the fake-git spawn race like the sibling disconnect test; each attempt + // gets a FRESH semaphore so a dropped losing attempt can't skew the winning + // attempt's permit accounting. Keep the winning attempt's future PENDING so the + // drop below exercises the client-disconnect teardown. + let (fut, sem, leader, grandchild) = { + let mut attempt = 0u64; + loop { + attempt += 1; + let _ = std::fs::remove_file(&pidfile); + // Semaphore(4): two owned permits model the handler's global-read + + // per-caller admission, leaving 2 available while the op is in flight. + let sem = Arc::new(Semaphore::new(4)); + let g = sem.clone().try_acquire_owned().unwrap(); + let c = sem.clone().try_acquire_owned().unwrap(); + let admission = AdmissionGuard::new(g, Some(c)); + let mut fut = Box::pin(build_filtered_pack( + git_bin.to_str().unwrap(), + tmp.path(), + &withheld, + Duration::from_secs(60), + Some(admission), + )); + // Advance the future a slice at a time until the fake records its pids + // (i.e. pack-objects is running). `Ok(_)` means the future returned + // before the pidfile appeared (spawn error / early exit); stop polling + // then, since re-polling a completed future panics. + let mut pids = None; + for _ in 0..500 { + let finished = tokio::time::timeout(Duration::from_millis(10), &mut fut) + .await + .is_ok(); + if let Some(p) = read_two_pids(&pidfile) { + pids = Some(p); + break; + } + if finished { + break; + } + } + match pids { + Some((l, gch)) => break (fut, sem, l, gch), + None => { + // Transient spawn miss: drop the still-armed future so its guard + // reaps anything that spawned (and returns the permits), then + // back off before retrying. + drop(fut); + assert!( + attempt < FAKE_GIT_RETRY_ATTEMPTS, + "fake git failed to reach the pack-objects stage after \ + {FAKE_GIT_RETRY_ATTEMPTS} attempts (persistent failure, \ + not a transient parallel-runner miss)" + ); + tokio::time::sleep(Duration::from_millis( + FAKE_GIT_BACKOFF_STEP_MS * attempt, + )) + .await; + } + } + } + }; + let _cleanup = ReapOnPanic(vec![leader, grandchild]); + assert!(alive(grandchild), "grandchild must be running mid-serve"); + + // Mid-serve: the pack-objects stage owns the guard, so the two permits are still + // held. A build that dropped the guard after rev-list (unthreaded pack-objects + // stage) would have freed them here. + assert_eq!( + sem.available_permits(), + 2, + "admission permits must be held while the filtered serve is in flight" + ); + + // Client disconnect: drop the request future. The pack-objects KillGroupOnDrop + // must tear the group down AND hold the permits until the group is + // ESRCH-confirmed gone, releasing them only then. + drop(fut); + + let mut released_while_alive = false; + let mut released_after_reap = false; + for _ in 0..500 { + let released = sem.available_permits() == 4; + let group_alive = alive(grandchild); + if released && group_alive { + released_while_alive = true; + } + if released && !group_alive { + released_after_reap = true; + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert!( + !released_while_alive, + "admission permits were released while the git group was still alive — the \ + path-scoped concurrency-cap bypass (#174 P1-a) is open" + ); + assert!( + released_after_reap, + "admission permits must be released once the group is reaped on disconnect" + ); + } + // ── F1: filtered-serve admission threaded through both pack stages ────── // // The handler-level disconnect regression lives in api/repos.rs diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index 80b63230..5617b419 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -431,7 +431,9 @@ fn batch_check_probe( /// never on any English `fatal:` wording (KTD-4): a genuinely-absent object is the only /// `Ok(None)` (404) path. A probe that could not honestly examine the store is a /// [`ProbeError`], split by object-store readability into `Transient` (retryable 503) -/// and `Deterministic` (terminal 500) so the serve path can shed the right status. +/// and `Deterministic` (terminal 500) so the serve path can shed the right status. The +/// deadline itself arrives as a `Transient` fault, so the handler marks the search +/// truncated rather than reporting a false not-found (#173 round-10 R1/KTD2). pub fn object_type_bounded( git_bin: &str, repo_path: &Path, @@ -599,6 +601,63 @@ pub(crate) fn object_store_readable_store_wide(repo_path: &Path) -> bool { object_store_readable(repo_path, "") } +/// Bounded `git cat-file -s` size read for the `GET /ipfs/{cid}` serve path (#173 +/// round-10, R1/KTD2): reads the object size WITHOUT its content (so an oversized object +/// is rejected before it is buffered, #173 F6), under +/// [`run_bounded_git`](crate::git::visibility_pack::run_bounded_git) so a wedged size +/// read is reaped at `deadline` instead of pinning the held /ipfs walk admission. +/// `Ok(n)` on success, and otherwise a [`ProbeError`] in the same vocabulary the type and +/// content stages use: a reaped child is `Transient` (retryable), and every other failure +/// goes through [`classify_store_fault`], so it is `Transient` when the object store is +/// not readable and `Deterministic` when it is. +/// +/// There is deliberately NO absence value (#173 round 12). This returned +/// `Ok(None)` for every non-timeout failure, which `gate_and_serve` reads as a verified +/// absence and does not taint the search for, so a corrupt object, an unreadable pack, or +/// a failed spawn handed an authorized caller a definitive 404 instead of the retryable +/// 503 tail. Absence is also not this stage's question: the caller has already had a +/// `Present` verdict from [`object_type_bounded`], so an object that cannot be sized here +/// is a fault, not a not-found, and the one honest exception (a concurrent gc between the +/// two stages) is classified by store readability like any other. Making the absence +/// unrepresentable is what keeps the next caller from reintroducing the swallow. +/// +/// Takes the shared `deadline` rather than its own timeout so a caller that pairs this +/// size check with a later read spends ONE budget across the pair, not one per stage. +pub fn object_size_bounded( + git_bin: &str, + repo_path: &Path, + sha256_hex: &str, + deadline: std::time::Instant, +) -> std::result::Result { + match crate::git::visibility_pack::run_bounded_git( + git_bin, + &["cat-file", "-s", sha256_hex], + repo_path, + b"", + deadline, + ) { + Ok(out) => { + let text = String::from_utf8_lossy(&out); + text.trim().parse::().map_err(|e| { + // git exited 0 but did not print a size: the store answered something + // this code cannot read, which is a fault and never an absence. + classify_store_fault( + repo_path, + sha256_hex, + anyhow::anyhow!("unparseable `cat-file -s` output {:?}: {e}", text.trim()), + ) + }) + } + // The watchdog reaped the child at `deadline`; retryable whatever the store looks + // like, so it is routed before readability gets a say (same rule as the content + // stage in `read_object_bounded`). + Err(e) if e.is::() => { + Err(ProbeError::Transient(e)) + } + Err(e) => Err(classify_store_fault(repo_path, sha256_hex, e)), + } +} + /// Bounded, reaped variant of [`read_object_content`] for the async `/ipfs` serve /// path (#174 F3). Same teardown guarantees as [`object_type_bounded`]. pub fn read_object_content_bounded( @@ -866,6 +925,92 @@ mod tests { ); } + /// #173 round-10 (KTD2): `object_type_bounded` reaps a wedged `cat-file` child at its + /// deadline instead of blocking on it to natural exit, so a hung probe cannot pin the + /// /ipfs walk admission the owning task holds. A fake `git` records its pid and sleeps + /// far past the 1s deadline; the `run_bounded_git` watchdog (SIGTERM -> grace -> + /// SIGKILL of the process group) must kill it well before that natural exit, and the + /// call must surface `GitServiceTimeout`. REVERT PROOF (RED): swap the twin's + /// `run_bounded_git` for the bare `Command::output()` and the wedged child stays alive + /// past the deadline — the mid-flight liveness poll below reads it still running. + #[cfg(unix)] + #[test] + fn object_type_bounded_reaps_wedged_child_at_deadline() { + use std::time::Duration; + let tmp = tempfile::TempDir::new().unwrap(); + let pidfile = tmp.path().join("catfile.pid"); + // `cat-file` records its own pid then sleeps 8s (>> the 1s deadline) so the probe + // is genuinely wedged; the watchdog is what must end it. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + cat-file) echo $$ > \"{}\"; sleep 8 ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + pidfile.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + let repo = tmp.path().to_path_buf(); + let git = git_path.to_str().unwrap().to_string(); + + let alive = |pid: i32| unsafe { libc::kill(pid, 0) == 0 }; + + // The bounded probe blocks until the watchdog tears the child down, so run it on + // a worker thread and poll for the reap from here. + let handle = std::thread::spawn(move || { + let deadline = std::time::Instant::now() + Duration::from_secs(1); + super::object_type_bounded(&git, &repo, "deadbeef", deadline) + }); + + let mut pid = None; + for _ in 0..500 { + if let Some(p) = std::fs::read_to_string(&pidfile) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + pid = Some(p); + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + let pid = pid.expect("the fake cat-file must have spawned and recorded its pid"); + + // Past the 1s deadline + SIGTERM grace but well before the 8s natural exit: the + // watchdog must already have reaped the wedged group. A bare, unbounded read would + // leave it running here — the load-bearing RED. + std::thread::sleep(Duration::from_secs(3)); + let reaped = !alive(pid); + // Defensive reap so a RED run leaks no orphan. + unsafe { + libc::kill(pid, libc::SIGKILL); + } + assert!( + reaped, + "object_type_bounded must reap the wedged cat-file child at the deadline, \ + not leave it running to its natural exit" + ); + + let res = handle.join().expect("probe thread joins"); + let err = res.expect_err("a deadline overrun must be an error, not a value"); + // A reaped deadline is a retryable fault, not a verdict about the object, so it + // must arrive as Transient (-> 503) carrying GitServiceTimeout. + let super::ProbeError::Transient(inner) = &err else { + panic!("a deadline overrun must be a Transient probe fault, got: {err:?}"); + }; + assert!( + inner.is::(), + "a deadline overrun must surface GitServiceTimeout, got: {inner:?}" + ); + } + /// #174 F5 (RED-before/GREEN-after): a packed object whose pack/idx is unreadable /// makes `git cat-file -t` emit "could not get object info" — byte-identical to a /// genuine miss. `object_type_bounded` must report absence ONLY when the object @@ -1371,6 +1516,134 @@ mod tests { ); } + /// #173 round 12 (jatmn): the SIZE probe must use the same absence-versus-fault + /// vocabulary the type probe does. It mapped every non-timeout failure to `Ok(None)`, + /// which `gate_and_serve` reads as a verified absence and does not taint, so a + /// corrupt object, an unreadable pack, or a failed spawn ended the search cleanly and + /// handed an authorized caller a definitive 404 instead of the retryable 503 tail. + /// + /// A corrupt loose object is the same fixture the type stage uses for this, and it is + /// the honest case: the object EXISTS, the type probe says so, and only the size read + /// fails. RED before the change (`Ok(None)`). + #[cfg(unix)] + #[test] + fn object_size_bounded_corrupt_loose_object_is_fault_not_absence() { + use std::os::unix::fs::PermissionsExt; + let td = tempfile::TempDir::new().unwrap(); + let work = td.path().join("sizecorrupt"); + std::fs::create_dir_all(&work).unwrap(); + let g = |args: &[&str]| { + assert!( + Command::new("git") + .args(args) + .current_dir(&work) + .status() + .unwrap() + .success(), + "git {args:?}" + ); + }; + g(&["init", "-q", "--object-format=sha256", "."]); + g(&["config", "user.email", "t@t"]); + g(&["config", "user.name", "t"]); + std::fs::write(work.join("f.txt"), b"loose object content\n").unwrap(); + g(&["add", "f.txt"]); + g(&["commit", "-qm", "c1"]); + let blob = String::from_utf8( + Command::new("git") + .args(["rev-parse", "HEAD:f.txt"]) + .current_dir(&work) + .output() + .unwrap() + .stdout, + ) + .unwrap() + .trim() + .to_string(); + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + // Healthy first: the probe reads the real size, so the fault case below is not + // passing for the trivial reason that this fixture never worked. + let healthy = super::object_size_bounded("git", &work, &blob, deadline); + assert!( + matches!(healthy, Ok(n) if n > 0), + "a readable object reports its size; got {healthy:?}" + ); + + let obj = work.join(".git/objects").join(&blob[0..2]).join(&blob[2..]); + let mut perms = std::fs::metadata(&obj).unwrap().permissions(); + perms.set_mode(0o644); + std::fs::set_permissions(&obj, perms).unwrap(); + std::fs::write(&obj, b"garbage not a zlib stream").unwrap(); + + let res = super::object_size_bounded("git", &work, &blob, deadline); + assert!( + res.is_err(), + "a corrupt object must surface as a probe fault, never as an absence the \ + resolver renders as a clean 404; got {res:?}" + ); + } + + /// #173 round 12 (jatmn), the transient arm: a store this process cannot read is the + /// retryable case, so the size probe classifies it `Transient` exactly as the type + /// probe does. Distinguishing the two arms is the whole point of routing through + /// `classify_store_fault` rather than returning one undifferentiated error. + #[cfg(unix)] + #[test] + fn object_size_bounded_unreadable_store_is_transient() { + use std::os::unix::fs::PermissionsExt; + let td = tempfile::TempDir::new().unwrap(); + let work = td.path().join("sizeunreadable"); + std::fs::create_dir_all(&work).unwrap(); + let g = |args: &[&str]| { + assert!( + Command::new("git") + .args(args) + .current_dir(&work) + .status() + .unwrap() + .success(), + "git {args:?}" + ); + }; + g(&["init", "-q", "--object-format=sha256", "."]); + g(&["config", "user.email", "t@t"]); + g(&["config", "user.name", "t"]); + std::fs::write(work.join("f.txt"), b"loose object content\n").unwrap(); + g(&["add", "f.txt"]); + g(&["commit", "-qm", "c1"]); + let blob = String::from_utf8( + Command::new("git") + .args(["rev-parse", "HEAD:f.txt"]) + .current_dir(&work) + .output() + .unwrap() + .stdout, + ) + .unwrap() + .trim() + .to_string(); + + // Make this oid's loose fan-out unreadable: git fails, and the store cannot + // certify absence, so the fault is retryable rather than terminal. + let fanout = work.join(".git/objects").join(&blob[0..2]); + let mut perms = std::fs::metadata(&fanout).unwrap().permissions(); + perms.set_mode(0o000); + std::fs::set_permissions(&fanout, perms).unwrap(); + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + let res = super::object_size_bounded("git", &work, &blob, deadline); + + let mut restore = std::fs::metadata(&fanout).unwrap().permissions(); + restore.set_mode(0o755); + std::fs::set_permissions(&fanout, restore).unwrap(); + + assert!( + matches!(res, Err(super::ProbeError::Transient(_))), + "an unreadable object store is the retryable arm; got {res:?}" + ); + } + /// #174 F5/U4: a corrupt LOOSE object makes `git cat-file --batch-check` print /// ` missing` on stdout (exit 0) yet emit `error:` diagnostics on stderr. The /// clean-`missing` absence path must NOT fire here — the `error:` line disqualifies diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index f26caf18..08666994 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -679,6 +679,477 @@ pub fn allowed_blob_set_for_caller_bounded( Ok(allowed) } +/// The reachable-commit enumeration for the LENIENT walks (the `/ipfs/{cid}` tree +/// gate and the commit/tag reachability set): bounded `git rev-list --all [HEAD]` +/// under the caller's shared `deadline`, deliberately WITHOUT +/// `assert_all_refs_are_commits`. That guard fail-closes a repo's whole walk when +/// any ref peels to a non-commit (an annotated tag of a tree is pushable through +/// receive-pack), which would 404 every reachable tree/commit/tag CID here for a +/// legitimate reader. `rev-list --all` skips such refs cleanly, so the commit set +/// stays complete; an object reachable only via such a ref is simply excluded — +/// correctly fail-closed. Fails closed on a rev-list error. +/// +/// Safe ONLY for a caller whose output feeds a fail-closed allow-list where absence +/// = withhold: a tolerant walk there over-withholds, never leaks. NOT safe for a +/// serve/replication filter, where a missed reachable object under-withholds — +/// those go through `blob_paths`, which runs the guard first. +fn reachable_commit_oids( + repo_path: &Path, + git_bin: &str, + deadline: Instant, +) -> Result> { + // The HEAD probe is a bounded `git rev-parse --verify HEAD` (a clean exit means + // HEAD resolves), matching `blob_paths`. When HEAD does not resolve (unborn + // branch on an empty repo) `--all` alone yields nothing, which is correct. + let head_resolves = run_bounded_git( + git_bin, + &["rev-parse", "--verify", "HEAD"], + repo_path, + b"", + deadline, + ) + .is_ok(); + let mut rev_args = vec!["rev-list", "--all"]; + if head_resolves { + rev_args.push("HEAD"); + } + let out = run_bounded_git(git_bin, &rev_args, repo_path, b"", deadline)?; + Ok(String::from_utf8_lossy(&out) + .lines() + .map(|l| l.trim().to_string()) + .filter(|l| !l.is_empty()) + .collect()) +} + +/// Every `(oid, "/repo/relative/path", kind)` triple reachable from the given +/// `commits` — the shared ls-tree seam the tree walk filters (`kind == "tree"`). +/// One bounded `git ls-tree -rzt` per commit under the caller's shared `deadline`: +/// `-rzt` is byte-identical to `-rz` for blob records and additionally emits the +/// tree object for each directory at its own path. `kind` is git's object-type +/// string ("blob", "tree", or "commit" for a gitlink). The commit's ROOT tree is +/// not emitted by `ls-tree` (it lists entries *under* a tree); `tree_paths` adds +/// it. Triples are de-duplicated across commits and paths carry a leading "/" to +/// match the glob form of visibility rules ("/secret/**"). +/// +/// Fails closed: if any tree walk fails — or a path is not valid UTF-8 — it +/// returns an error so the caller aborts rather than producing a partial +/// (under-withheld) set. +fn object_paths( + repo_path: &Path, + git_bin: &str, + commits: &[String], + deadline: Instant, +) -> Result> { + let mut out: HashSet<(String, String, String)> = HashSet::new(); + for commit in commits { + let listing_out = run_bounded_git( + git_bin, + &["ls-tree", "-rzt", commit], + repo_path, + b"", + deadline, + )?; + // `-z` NUL-delimits records and emits paths raw; plain `git ls-tree -r` + // C-quotes any path with non-ASCII or special bytes (e.g. café.txt becomes + // "secret/caf\303\251.txt"), and that quoted literal would not match a + // visibility rule like "/secret/**", under-withholding the object. The TAB + // field separator survives `-z`, so the per-record parse is unchanged. + // + // Parse strictly: a lossy decode would replace an invalid byte in a denied + // path (e.g. a non-UTF-8 directory name) with U+FFFD, and the mangled string + // would no longer match its deny rule — the same under-withholding class, one + // layer down. Fail closed instead so the caller aborts rather than leaks. + let Ok(listing_stdout) = std::str::from_utf8(&listing_out) else { + anyhow::bail!( + "git ls-tree -rzt {commit} returned a non-UTF-8 path; \ + refusing to produce a partial (under-withheld) set" + ); + }; + for record in listing_stdout.split('\0') { + // " \t" + let Some((meta, path)) = record.split_once('\t') else { + continue; + }; + let mut parts = meta.split_whitespace(); + let _mode = parts.next(); + let kind = parts.next(); + let oid = parts.next(); + if let (Some(kind), Some(oid)) = (kind, oid) { + out.insert((oid.to_string(), format!("/{path}"), kind.to_string())); + } + } + } + Ok(out) +} + +/// Root tree oid of every reachable commit, at "/". `ls-tree` never emits a commit's +/// own root tree (it lists entries *under* a tree), so it is added explicitly here. +/// Resolved in ONE bounded `git log --no-walk --format=%T --stdin` pass over the +/// shared commit set — not a per-commit `rev-parse` — so a tree-set walk costs the +/// same subprocess order as the blob walk. The commit oids go on STDIN, not argv: a +/// long history has tens of thousands of reachable commits, and passing them all as +/// arguments overflows ARG_MAX so `git log` fails to spawn — which the caller treats +/// as a walk error and fail-closed 404s an authorized reader of a reachable/root +/// tree (#173 P2). `run_bounded_git` drains stdout concurrently with the stdin +/// write, so a large history cannot deadlock the pipes. A commit whose root tree git +/// cannot resolve fails the pass (bail), failing closed. +fn root_tree_pairs( + repo_path: &Path, + git_bin: &str, + commits: &[String], + deadline: Instant, +) -> Result> { + if commits.is_empty() { + return Ok(HashSet::new()); + } + let mut buf = String::with_capacity(commits.len() * 65); + for c in commits { + buf.push_str(c); + buf.push('\n'); + } + let out = run_bounded_git( + git_bin, + &["log", "--no-walk=unsorted", "--format=%T", "--stdin"], + repo_path, + buf.as_bytes(), + deadline, + )?; + let mut set = HashSet::new(); + for line in String::from_utf8_lossy(&out).lines() { + let oid = line.trim(); + if !oid.is_empty() { + set.insert((oid.to_string(), "/".to_string())); + } + } + Ok(set) +} + +/// Every `(tree_oid, "/path")` pair reachable in `repo_path`: the `kind == "tree"` +/// slice of [`object_paths`] (subtree trees at their directory paths) PLUS every +/// reachable commit's root tree at "/" (see [`root_tree_pairs`]). Computes the +/// reachable-commit set ONCE (leniently — see [`reachable_commit_oids`]; the tree +/// allowed-set feeds ONLY the `/ipfs/{cid}` tree gate, where absence = fail-closed +/// 404) and drives both the ls-tree walk and the root-tree pass from it, so the two +/// cannot diverge and neither re-enumerates. The tree analog of [`blob_paths`], +/// bounded by the same shared `deadline`. +fn tree_paths( + repo_path: &Path, + git_bin: &str, + deadline: Instant, +) -> Result> { + let commits = reachable_commit_oids(repo_path, git_bin, deadline)?; + let mut out: HashSet<(String, String)> = object_paths(repo_path, git_bin, &commits, deadline)? + .into_iter() + .filter(|(_, _, kind)| kind == "tree") + .map(|(oid, path, _)| (oid, path)) + .collect(); + out.extend(root_tree_pairs(repo_path, git_bin, &commits, deadline)?); + Ok(out) +} + +/// The OIDs from a `(oid, "/path")` listing that visibility ALLOWS `caller` at some +/// path — the shared inner loop of the blob and tree allowed-sets. An oid reachable +/// at an allowed path is kept even when also reachable at a denied one. +fn allowed_set_from_pairs<'a>( + pairs: impl IntoIterator, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, + caller: Option<&str>, +) -> HashSet { + pairs + .into_iter() + .filter(|(_, path)| { + visibility_check(rules, is_public, owner_did, caller, path) == Decision::Allow + }) + .map(|(oid, _)| oid.clone()) + .collect() +} + +/// Reachable tree OIDs that visibility ALLOWS `caller` at some path — the tree +/// analog of [`allowed_blob_set_for_caller`]. `GET /ipfs/{cid}` gates tree objects +/// with this so the CID surface matches `get_tree`: a tree reachable only at a +/// withheld path is absent from the set and 404'd; the root tree ("/") and any tree +/// on the path to an allowed subtree are present. Fails closed on a +/// dangling/unreachable tree (never enumerated by the reachable walk, so never in +/// the set — the #126 geometry, for trees). A tree reachable at an allowed path is +/// included even when also reachable at a withheld one (its structure is visible to +/// this caller elsewhere). +#[cfg(test)] +pub fn allowed_tree_set_for_caller( + repo_path: &Path, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, + caller: Option<&str>, +) -> Result> { + allowed_tree_set_for_caller_bounded( + repo_path, + "git", + WALK_TIMEOUT, + rules, + is_public, + owner_did, + caller, + ) +} + +/// [`allowed_tree_set_for_caller`] with an injectable `git_bin` and walk `timeout`, +/// for the `GET /ipfs/{cid}` tree gate. One deadline spans the whole walk (the HEAD +/// probe, rev-list, every per-commit ls-tree, and the root-tree pass), matching +/// `blob_paths`, so a slow or hung walk is bounded as a unit while the handler holds +/// its /ipfs walk permit (#174 F5). +pub fn allowed_tree_set_for_caller_bounded( + repo_path: &Path, + git_bin: &str, + timeout: Duration, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, + caller: Option<&str>, +) -> Result> { + let deadline = Instant::now() + timeout; + Ok(allowed_set_from_pairs( + &tree_paths(repo_path, git_bin, deadline)?, + rules, + is_public, + owner_did, + caller, + )) +} + +/// Object bound for the annotated-tag reachability walk (#173, jatmn tag fan-out). +/// A path-scoped pinned-CID request drives this walk while holding one per-request +/// and one per-IP walk slot, so the total tag work must be finite regardless of how +/// many tag refs the repo has. 8192 is far past any real repo's annotated-tag count +/// (the Linux kernel has a few hundred), yet finite: a repo beyond it fails closed +/// (Err), matching this function's fail-closed-on-any-git-error contract, rather than +/// truncating silently (which would under-withhold a still-reachable tag object). +const MAX_TAG_OBJECTS: usize = 8192; + +/// Walk the annotated-tag chains rooted at `seeds`, inserting every tag object they +/// pass through into `set`. A tag whose target is itself a tag (tag-of-a-tag) +/// discovers the inner tag, which is walked in a later round. +/// +/// #173 (jatmn): the tag inspection is BATCHED, not one process per tag. Each round +/// feeds every not-yet-inspected tag oid to a SINGLE `git cat-file --batch` child on +/// stdin and reads back framed ` \n\n` records, so the +/// number of child processes is bounded by the tag-chain DEPTH (rounds), not the tag +/// COUNT. Oids go on stdin, never argv, so a large tag set cannot overflow ARG_MAX. +/// The child runs through [`run_bounded_git`], which drains stdout concurrently with +/// the stdin write (subsuming #173's F4 writer-thread drain — a round large enough to +/// fill both pipes cannot deadlock) and tears the child down at `deadline`, so a hung +/// cat-file cannot pin the caller's /ipfs walk permit (#174 F5). Total tag objects +/// inspected are capped at `max_tag_objects`; exceeding it is an error (fail closed), +/// not a silent truncation. Takes the bound as a parameter so a test can drive a tiny +/// value while the caller passes the real `MAX_TAG_OBJECTS`. +fn walk_tag_chain( + repo_path: &Path, + git_bin: &str, + seeds: Vec, + set: &mut HashSet, + max_tag_objects: usize, + deadline: Instant, +) -> Result<()> { + // Tag oids known but not yet inspected. Seeds may repeat / already be present; + // the `set.insert` gate below is what actually dedups and terminates cycles. + let mut pending: Vec = seeds; + let mut inspected: usize = 0; + + while !pending.is_empty() { + // Inspect only oids new to `set`; a re-seen oid was already walked. + let round: Vec = pending + .drain(..) + .filter(|oid| set.insert(oid.clone())) + .collect(); + if round.is_empty() { + break; + } + inspected += round.len(); + if inspected > max_tag_objects { + anyhow::bail!( + "annotated-tag walk exceeded the object bound ({max_tag_objects}); refusing to serve" + ); + } + + // One bounded child for the whole round: feed all oids on stdin, read the + // framed records from the returned stdout. + let mut buf = String::with_capacity(round.len() * 65); + for oid in &round { + buf.push_str(oid); + buf.push('\n'); + } + let stdout = run_bounded_git( + git_bin, + &["cat-file", "--batch"], + repo_path, + buf.as_bytes(), + deadline, + )?; + + // Parse one record per requested oid: ` \n\n`. + // A ` missing\n` record has no size/body and is anomalous here (every + // oid came from a ref tip or a prior tag body), so fail closed. + let mut i = 0usize; + for _ in 0..round.len() { + let hdr_end = stdout[i..] + .iter() + .position(|&b| b == b'\n') + .map(|p| i + p) + .context("git cat-file --batch: truncated record header")?; + let header = std::str::from_utf8(&stdout[i..hdr_end]) + .context("git cat-file --batch: non-utf8 record header")?; + i = hdr_end + 1; + let mut fields = header.split(' '); + let _oid = fields.next().unwrap_or(""); + let ty = fields.next().unwrap_or(""); + if ty == "missing" || fields.clone().next().is_none() { + anyhow::bail!("git cat-file --batch: object {header:?} missing or malformed"); + } + let size: usize = fields + .next() + .unwrap_or("") + .parse() + .context("git cat-file --batch: bad record size")?; + let body_end = i + .checked_add(size) + .filter(|&e| e <= stdout.len()) + .context("git cat-file --batch: truncated record body")?; + // Only a tag object can point at an inner tag; walk its header. + if ty == "tag" { + let body = std::str::from_utf8(&stdout[i..body_end]) + .context("git cat-file --batch: non-utf8 tag body")?; + let mut target = None; + let mut is_tag = false; + for line in body.lines() { + if let Some(oid) = line.strip_prefix("object ") { + target = Some(oid.trim().to_string()); + } else if line == "type tag" { + is_tag = true; + } else if line.is_empty() { + break; // end of header + } + } + if is_tag { + if let Some(t) = target { + pending.push(t); + } + } + } + // Skip body plus its trailing newline to the next record. + i = body_end + 1; + } + } + Ok(()) +} + +/// The reachable-commit/tag gate set for the `/ipfs/{cid}` resolver (#173, F2): +/// every reachable commit oid UNION every reachable annotated-tag OBJECT oid. A +/// DANGLING commit/tag (referenced by no ref, directly or via a tag chain) is in +/// neither part, so the resolver denies it under a path-scoped rule instead of +/// leaking its message; a reachable one still serves. +#[cfg(test)] +pub fn reachable_commit_tag_oids(repo_path: &Path) -> Result> { + reachable_commit_tag_oids_bounded(repo_path, "git", WALK_TIMEOUT) +} + +/// [`reachable_commit_tag_oids`] with an injectable `git_bin` and walk `timeout`, +/// for the `GET /ipfs/{cid}` commit/tag gate. One deadline spans the whole walk. +/// +/// Reachable commits come from bounded `git rev-list --all` (+ HEAD for the +/// detached case). Unlike the blob allowed-set, this does NOT run +/// `assert_all_refs_are_commits`: that guard fail-closes a repo's whole walk when +/// any ref peels to a non-commit (an annotated tag of a tree is pushable through +/// receive-pack), which would 404 every reachable commit/tag CID here for a +/// legitimate reader. The guard exists to stop blob/tree UNDER-withholding; it is +/// unnecessary for reachability, since a dangling object is absent from +/// `rev-list --all` and the ref walk below regardless of odd refs — so dropping it +/// recovers availability without admitting any dangling object (no leak). +/// +/// Reachable tag OBJECTS: `rev-list --all` dereferences annotated tags to commits, +/// so the tag objects are absent from it. Collect them by walking every ref tip and +/// peeling each tag's chain, so a nested tag-of-a-tag's INNER tag object (reachable +/// and pinnable, but not itself a ref tip) is included too. Fails closed on any git +/// error. +pub fn reachable_commit_tag_oids_bounded( + repo_path: &Path, + git_bin: &str, + timeout: Duration, +) -> Result> { + let deadline = Instant::now() + timeout; + // Reachable commits — no ref-commit assertion (see docstring). The HEAD probe + // doubles as the seed source for the tag-valued detached HEAD below: + // `rev-parse --verify HEAD` returns the tag oid UNPEELED when HEAD names a tag + // object. Failing to resolve HEAD (unborn/absent) is not fatal — there is + // simply no HEAD to walk or seed. + let head_oid: Option = run_bounded_git( + git_bin, + &["rev-parse", "--verify", "HEAD"], + repo_path, + b"", + deadline, + ) + .ok() + .map(|out| String::from_utf8_lossy(&out).trim().to_string()) + .filter(|s| !s.is_empty()); + let mut rev_args = vec!["rev-list", "--all"]; + if head_oid.is_some() { + rev_args.push("HEAD"); + } + let rev = run_bounded_git(git_bin, &rev_args, repo_path, b"", deadline)?; + let mut set: HashSet = String::from_utf8_lossy(&rev) + .lines() + .map(|l| l.trim().to_string()) + .filter(|l| !l.is_empty()) + .collect(); + + // Ref tips that are annotated tag objects seed the tag-chain walk. + let refs = run_bounded_git( + git_bin, + &["for-each-ref", "--format=%(objectname) %(objecttype)"], + repo_path, + b"", + deadline, + )?; + let mut worklist: Vec = Vec::new(); + for line in String::from_utf8_lossy(&refs).lines() { + let mut it = line.split_whitespace(); + if let (Some(oid), Some("tag")) = (it.next(), it.next()) { + worklist.push(oid.to_string()); + } + } + // A detached/direct HEAD may name an annotated tag object with no ref at that tag + // (#173 review, finding 3): `rev-list --all HEAD` above peels it to its commit and + // `for-each-ref` has no tag row, so the tag OBJECT would be omitted and its pinned + // CID would 404 for an authorized reader. Seed a tag-valued HEAD into the tag-chain + // walk; a `commit` HEAD adds nothing. A cat-file failure here only skips the seed + // (over-withholds that one tag — fail-closed), matching the original's tolerance. + if let Some(head_oid) = head_oid { + if let Ok(ty) = run_bounded_git( + git_bin, + &["cat-file", "-t", &head_oid], + repo_path, + b"", + deadline, + ) { + if String::from_utf8_lossy(&ty).trim() == "tag" { + worklist.push(head_oid); + } + } + } + // Peel every tag object's chain into `set`, adding each tag object it passes + // through. Bounded and batched (#173, jatmn tag fan-out): see `walk_tag_chain`. + walk_tag_chain( + repo_path, + git_bin, + worklist, + &mut set, + MAX_TAG_OBJECTS, + deadline, + )?; + Ok(set) +} + /// Objects safe to replicate, failing closed on blobs (#99). A candidate /// replicates iff it is NOT a blob (`all_blob_oids` — commits and trees are /// structural, never content-withheld) OR it is in `allowed_blobs` (reachable @@ -1113,6 +1584,614 @@ esac\n"; (td, bare, secret, public) } + /// #173 (jatmn round 8, F4 — load-bearing): a repo with enough annotated tags that + /// one `cat-file --batch` round fills BOTH pipes (stdin > 64 KiB of oids while the + /// child blocks on a full stdout) must not deadlock. The old order wrote the whole + /// round to stdin before draining stdout and hung indefinitely, stranding a blocking- + /// pool thread; `run_bounded_git`'s concurrent writer/drain completes. Driven with a + /// completion timeout: GREEN finishes in well under a second, RED (old order) hangs + /// and the recv_timeout fires. ~3000 tags is well past the ~2030-oid deadlock + /// threshold (41 bytes/oid, 64 KiB pipes) and under MAX_TAG_OBJECTS (8192). + /// Bulk-created via one fast-import stream so the fixture cost is one git process, + /// not 3000 `git tag -a` spawns. + #[test] + fn walk_tag_chain_large_batch_does_not_deadlock() { + use std::io::Write; + let td = TempDir::new().unwrap(); + let work = td.path().join("work"); + let bare = td.path().join("bare.git"); + let run = |args: &[&str], dir: &Path| { + assert!( + Command::new("git") + .args(args) + .current_dir(dir) + .status() + .unwrap() + .success(), + "git {args:?} failed" + ); + }; + std::fs::create_dir_all(&work).unwrap(); + std::fs::write(work.join("f.txt"), b"x\n").unwrap(); + run(&["init", "-q"], &work); + run(&["config", "user.email", "t@t"], &work); + run(&["config", "user.name", "t"], &work); + run(&["add", "."], &work); + run(&["commit", "-qm", "init"], &work); + let head = { + let out = Command::new("git") + .args(["rev-parse", "HEAD"]) + .current_dir(&work) + .output() + .unwrap(); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + + // Bulk-create ~3000 annotated tags via one fast-import stream. + const N: usize = 3000; + let mut stream = String::new(); + for i in 0..N { + let msg = format!("annotated tag {i}\n"); + stream.push_str(&format!("tag t{i}\n")); + stream.push_str(&format!("from {head}\n")); + stream.push_str("tagger t 1700000000 +0000\n"); + stream.push_str(&format!("data {}\n", msg.len())); + stream.push_str(&msg); + } + let mut fi = Command::new("git") + .args(["fast-import", "--quiet"]) + .current_dir(&work) + .stdin(std::process::Stdio::piped()) + .spawn() + .unwrap(); + fi.stdin + .take() + .unwrap() + .write_all(stream.as_bytes()) + .unwrap(); + assert!(fi.wait().unwrap().success(), "fast-import failed"); + + run( + &[ + "clone", + "-q", + "--bare", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + td.path(), + ); + + // Drive the walk on a worker thread with a completion timeout. The old + // write-all-before-drain order hangs here; the fix completes near-instantly. + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = tx.send(reachable_commit_tag_oids(&bare).map(|s| s.len())); + }); + match rx.recv_timeout(std::time::Duration::from_secs(20)) { + Ok(Ok(n)) => assert!( + n >= N, + "the walk must resolve every annotated tag object (got {n}, expected >= {N})" + ), + Ok(Err(e)) => panic!("walk errored: {e}"), + Err(_) => panic!("walk_tag_chain deadlocked on a large tag batch (F4 regression)"), + } + } + + /// #173 review (finding 3): an annotated tag reachable ONLY through a tag-valued + /// detached HEAD (raw HEAD naming a tag object, with no ref at that tag) must still + /// enter `reachable_commit_tag_oids`. `rev-list --all HEAD` peels such a HEAD to its + /// commit and `for-each-ref` has no tag row, so without a HEAD tag-seed the tag + /// OBJECT is omitted and its pinned CID would 404 for an authorized reader. RED + /// before the HEAD tag-seed (the tag oid is absent); GREEN after. + #[test] + fn reachable_commit_tag_oids_includes_tag_valued_detached_head() { + use std::io::Write; + let td = TempDir::new().unwrap(); + let work = td.path().join("work"); + let bare = td.path().join("bare.git"); + let run = |args: &[&str], dir: &Path| -> String { + let out = Command::new("git") + .args(args) + .current_dir(dir) + .output() + .unwrap(); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + std::fs::create_dir_all(&work).unwrap(); + std::fs::write(work.join("a.txt"), b"hi\n").unwrap(); + run(&["init", "-q"], &work); + run(&["config", "user.email", "t@t"], &work); + run(&["config", "user.name", "t"], &work); + run(&["add", "."], &work); + run(&["commit", "-qm", "seed"], &work); + run( + &[ + "clone", + "-q", + "--bare", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + td.path(), + ); + let commit = run(&["rev-parse", "HEAD"], &bare); + + // An annotated tag OBJECT in the bare ODB, with NO ref pointing at it. + let tag_body = format!( + "object {commit}\ntype commit\ntag htag\ntagger t 0 +0000\n\nHEAD-only tag\n" + ); + let tag_oid = { + let mut child = Command::new("git") + .args(["hash-object", "-t", "tag", "-w", "--stdin"]) + .current_dir(&bare) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .spawn() + .unwrap(); + child + .stdin + .as_mut() + .unwrap() + .write_all(tag_body.as_bytes()) + .unwrap(); + let out = child.wait_with_output().unwrap(); + assert!(out.status.success()); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + assert_eq!(run(&["cat-file", "-t", &tag_oid], &bare), "tag"); + // Raw-write HEAD directly to the tag object (the only way this state arises; + // update-ref / checkout both refuse a non-commit HEAD). + std::fs::write(bare.join("HEAD"), format!("{tag_oid}\n")).unwrap(); + + let set = reachable_commit_tag_oids(&bare).unwrap(); + assert!( + set.contains(&tag_oid), + "a tag reachable only via a tag-valued detached HEAD must be in the reachable set" + ); + assert!( + set.contains(&commit), + "the commit the HEAD tag peels to stays reachable (no regression)" + ); + } + + /// #173: `reachable_commit_tag_oids` on an empty repo (unborn HEAD) must return an + /// empty set, not error — exercising the `rev-parse HEAD` fail branch of the + /// detached-HEAD tag seed (there is simply no HEAD to seed). + #[test] + fn reachable_commit_tag_oids_handles_unborn_head() { + let td = TempDir::new().unwrap(); + let bare = td.path().join("empty.git"); + let ok = Command::new("git") + .args(["init", "-q", "--bare", bare.to_str().unwrap()]) + .status() + .unwrap() + .success(); + assert!(ok, "git init --bare failed"); + let set = reachable_commit_tag_oids(&bare).unwrap(); + assert!( + set.is_empty(), + "an empty repo (unborn HEAD) yields an empty reachable set with no error" + ); + } + + #[test] + fn object_paths_emits_trees_and_blob_paths_is_the_blob_slice() { + let (_td, bare, secret_oid, public_oid) = fixture(); + let deadline = Instant::now() + WALK_TIMEOUT; + // The lenient enumeration; on this clean fixture it matches the strict one. + let commits = reachable_commit_oids(&bare, "git", deadline).unwrap(); + let objs = object_paths(&bare, "git", &commits, deadline).unwrap(); + + // Blob records survive the `-rzt` change, at their paths (unchanged). + assert!(objs.contains(&(secret_oid.clone(), "/secret/b.txt".into(), "blob".into()))); + assert!(objs.contains(&(public_oid.clone(), "/public/a.txt".into(), "blob".into()))); + + // The #135 addition: subtree tree objects at their directory paths. + assert!( + objs.iter().any(|(_, p, k)| k == "tree" && p == "/secret"), + "the /secret subtree tree must be emitted at its dir path" + ); + assert!( + objs.iter().any(|(_, p, k)| k == "tree" && p == "/public"), + "the /public subtree tree must be emitted at its dir path" + ); + + // blob_paths must equal the blob slice of object_paths exactly — compared as + // SETS (both walks dedup via HashSet; the collected order is nondeterministic). + let bp: HashSet<(String, String)> = blob_paths(&bare, "git", WALK_TIMEOUT) + .unwrap() + .into_iter() + .collect(); + let bp_from_obj: HashSet<(String, String)> = objs + .iter() + .filter(|(_, _, k)| k == "blob") + .map(|(o, p, _)| (o.clone(), p.clone())) + .collect(); + assert_eq!( + bp, bp_from_obj, + "blob_paths output must be byte-identical to object_paths' blob slice" + ); + } + + #[test] + fn allowed_tree_set_gates_withheld_subtree_tree() { + let (_td, bare, _s, _p) = fixture(); + let oid = |rev: &str| { + let out = Command::new("git") + .args(["rev-parse", rev]) + .current_dir(&bare) + .output() + .unwrap(); + assert!(out.status.success(), "rev-parse {rev}"); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + let secret_tree = oid("HEAD:secret"); + let public_tree = oid("HEAD:public"); + let root_tree = oid("HEAD^{tree}"); + let reader = "did:key:z6MkReader"; + let rules = [rule("/secret/**", &[reader])]; + + // anon: the withheld /secret tree is excluded; root ("/") and /public are in. + let anon = allowed_tree_set_for_caller(&bare, &rules, true, OWNER, None).unwrap(); + assert!( + !anon.contains(&secret_tree), + "withheld /secret subtree tree excluded for anon" + ); + assert!(anon.contains(&root_tree), "root tree included (path /)"); + assert!(anon.contains(&public_tree), "/public subtree tree included"); + + // listed reader: sees the /secret tree (caller-aware, not a blanket deny). + let rd = allowed_tree_set_for_caller(&bare, &rules, true, OWNER, Some(reader)).unwrap(); + assert!( + rd.contains(&secret_tree), + "listed reader sees the /secret tree" + ); + + // owner: sees every reachable tree. + let ow = allowed_tree_set_for_caller(&bare, &rules, true, OWNER, Some(OWNER)).unwrap(); + assert!( + ow.contains(&secret_tree) && ow.contains(&public_tree) && ow.contains(&root_tree), + "owner sees all reachable trees" + ); + } + + #[test] + fn allowed_tree_set_excludes_dangling_tree() { + use std::io::Write; + let (_td, bare, secret_oid, _p) = fixture(); + // A DANGLING tree: written to the ODB but referenced by no commit. Uses a + // UNIQUE entry name so its oid is content-distinct from every reachable tree + // (a content-identical tree would dedup to a reachable oid — that is T2, not + // danglingness). The reachable-only walk never enumerates it -> fail closed. + let mut child = Command::new("git") + .args(["mktree"]) + .current_dir(&bare) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + writeln!( + child.stdin.as_mut().unwrap(), + "100644 blob {secret_oid}\tdangling-only-unreferenced.txt" + ) + .unwrap(); + let out = child.wait_with_output().unwrap(); + assert!(out.status.success(), "git mktree"); + let dangling = String::from_utf8_lossy(&out.stdout).trim().to_string(); + + let rules = [rule("/secret/**", &[])]; + for caller in [None, Some(OWNER)] { + let set = allowed_tree_set_for_caller(&bare, &rules, true, OWNER, caller).unwrap(); + assert!( + !set.contains(&dangling), + "dangling tree must never be in the reachable allowed-set (caller={caller:?})" + ); + } + } + + #[test] + fn allowed_tree_set_includes_tree_shared_across_allowed_and_denied_paths() { + // T2 (content-dedup): the SAME tree oid reachable at both an allowed and a + // withheld path is INCLUDED for anon (allowed-wins) — its structure is + // visible to the caller at the allowed path. Mirrors the blob analog + // `same_blob_at_allowed_and_denied_path_is_not_withheld`. + let td = TempDir::new().unwrap(); + let work = td.path().join("work"); + std::fs::create_dir_all(work.join("pub/sub")).unwrap(); + std::fs::create_dir_all(work.join("sec/sub")).unwrap(); + std::fs::write(work.join("pub/sub/f.txt"), b"same bytes\n").unwrap(); + std::fs::write(work.join("sec/sub/f.txt"), b"same bytes\n").unwrap(); + let run = |args: &[&str]| { + assert!( + Command::new("git") + .args(args) + .current_dir(&work) + .status() + .unwrap() + .success(), + "git {args:?}" + ); + }; + run(&["init", "-q"]); + run(&["config", "user.email", "t@t"]); + run(&["config", "user.name", "t"]); + run(&["add", "."]); + run(&["commit", "-qm", "seed"]); + let oid = |rev: &str| { + let out = Command::new("git") + .args(["rev-parse", rev]) + .current_dir(&work) + .output() + .unwrap(); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + let pub_sub = oid("HEAD:pub/sub"); + let sec_sub = oid("HEAD:sec/sub"); + assert_eq!(pub_sub, sec_sub, "identical content dedups to one tree oid"); + + // Withhold /sec from anon; the shared oid is still reachable at /pub/sub. + let rules = [rule("/sec/**", &[])]; + let anon = allowed_tree_set_for_caller(&work, &rules, true, OWNER, None).unwrap(); + assert!( + anon.contains(&pub_sub), + "a tree reachable at an allowed path is included even when also at a withheld path" + ); + } + + #[test] + fn allowed_tree_set_includes_root_trees_of_all_reachable_commits() { + // The batched root-tree pass (root_tree_pairs) must return EVERY reachable + // commit's root tree, not just HEAD's — two commits with distinct root trees + // both land in the set. Guards the git-log-over-N-commits root derivation. + let td = TempDir::new().unwrap(); + let work = td.path().join("work"); + std::fs::create_dir_all(&work).unwrap(); + let run = |args: &[&str]| { + assert!( + Command::new("git") + .args(args) + .current_dir(&work) + .status() + .unwrap() + .success(), + "git {args:?}" + ); + }; + run(&["init", "-q"]); + run(&["config", "user.email", "t@t"]); + run(&["config", "user.name", "t"]); + let oid = |rev: &str| { + let out = Command::new("git") + .args(["rev-parse", rev]) + .current_dir(&work) + .output() + .unwrap(); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + std::fs::write(work.join("a.txt"), b"one\n").unwrap(); + run(&["add", "."]); + run(&["commit", "-qm", "c1"]); + let root1 = oid("HEAD^{tree}"); + std::fs::write(work.join("b.txt"), b"two\n").unwrap(); + run(&["add", "."]); + run(&["commit", "-qm", "c2"]); + let root2 = oid("HEAD^{tree}"); + assert_ne!(root1, root2, "the two commits have distinct root trees"); + + // Public repo, no rules: every reachable tree is allowed for anon. + let set = allowed_tree_set_for_caller(&work, &[], true, OWNER, None).unwrap(); + assert!( + set.contains(&root1) && set.contains(&root2), + "root trees of BOTH reachable commits are in the set (batched root pass)" + ); + } + + #[test] + fn root_tree_pairs_returns_every_root_tree_at_scale() { + // Parity + liveness at scale for root_tree_pairs (#173 P2): feed every + // reachable commit oid to `git log --format=%T --stdin` and collect each + // commit's root tree. With N commits that is ~N*41 bytes of oids in and + // ~N*41 bytes of %T out — past the ~64 KiB pipe buffer in both directions — + // so this exercises the large-bidirectional-IO path the 2-commit test above + // cannot, and asserts parity: every distinct root tree comes back. + // + // NOTE: this is NOT a deadlock guard. `git log --stdin` reads its whole + // revision list to EOF before emitting any %T, so the naive "write all of + // stdin, then drain stdout" form does not deadlock at any scale for this + // invocation. `run_bounded_git`'s concurrent writer/drain is cheap defensive + // isolation, not load-bearing, and this test does not claim otherwise. The + // 30s watchdog is a general liveness bound so a future regression that + // genuinely hangs fails fast here rather than stalling the suite. + const N: usize = 2500; + let td = TempDir::new().unwrap(); + let bare = td.path().join("many.git"); + assert!(Command::new("git") + .args(["init", "-q", "--bare", bare.to_str().unwrap()]) + .status() + .unwrap() + .success()); + + // fast-import a linear chain of N commits, each adding a distinct file so + // every root tree is distinct (dedup cannot shrink the output). One + // subprocess, ~1s — far cheaper than N `git commit` spawns. + let mut stream = String::new(); + for i in 0..N { + let (b, cm) = (2 * i + 1, 2 * i + 2); + let content = format!("v{i}"); + let msg = format!("c{i}"); + stream.push_str(&format!( + "blob\nmark :{b}\ndata {}\n{content}\n", + content.len() + )); + stream.push_str(&format!( + "commit refs/heads/main\nmark :{cm}\ncommitter t 0 +0000\ndata {}\n{msg}\n", + msg.len() + )); + if i > 0 { + stream.push_str(&format!("from :{}\n", 2 * (i - 1) + 2)); + } + stream.push_str(&format!("M 100644 :{b} f{i}\n\n")); + } + let mut fi = Command::new("git") + .args(["fast-import", "--quiet"]) + .current_dir(&bare) + .stdin(std::process::Stdio::piped()) + .spawn() + .unwrap(); + { + use std::io::Write; + fi.stdin + .take() + .unwrap() + .write_all(stream.as_bytes()) + .unwrap(); + } + assert!(fi.wait().unwrap().success(), "fast-import failed"); + + let commits = reachable_commit_oids(&bare, "git", Instant::now() + WALK_TIMEOUT).unwrap(); + assert_eq!(commits.len(), N, "all {N} commits reachable"); + + // Call root_tree_pairs directly (private, same module) under a liveness + // watchdog, then assert it returned every distinct root tree. + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = tx.send( + root_tree_pairs(&bare, "git", &commits, Instant::now() + WALK_TIMEOUT) + .map(|s| s.len()), + ); + }); + match rx.recv_timeout(std::time::Duration::from_secs(30)) { + Ok(Ok(len)) => assert_eq!(len, N, "every distinct root tree returned"), + Ok(Err(e)) => panic!("root_tree_pairs errored: {e}"), + Err(_) => panic!("root_tree_pairs did not return within 30s"), + } + } + + /// #173 (jatmn tag fan-out): the batched `git cat-file --batch` tag walk must + /// return the SAME reachable set as the old per-tag `cat-file tag` loop — every + /// commit, the outer tag object, AND the inner tag object of a tag-of-a-tag chain + /// (the inner tag is reachable but is not itself a ref tip, so it is only found by + /// peeling the outer tag's target). Behavior-preservation proof for the rewrite. + #[test] + fn reachable_commit_tag_oids_includes_nested_tag_objects() { + let (_td, bare, _secret, _public) = fixture(); + let run = |args: &[&str]| -> String { + let out = Command::new("git") + .args(args) + .current_dir(&bare) + .output() + .unwrap(); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + run(&["config", "user.email", "t@t"]); + run(&["config", "user.name", "t"]); + // v1 -> commit, v2 -> v1 (tag-of-a-tag), plus a couple of sibling tags so the + // round batches more than one oid. Capture v1's oid, then DELETE the v1 ref so + // the inner tag object survives in the ODB but is NOT a ref tip: it is then + // reachable ONLY by peeling v2's target chain. That makes the peel load-bearing + // (breaking the inner-tag enqueue drops v1 from the set), unlike leaving v1 as + // its own ref where `for-each-ref` would seed it directly. + run(&["tag", "-a", "-m", "inner", "v1", "HEAD"]); + run(&["tag", "-a", "-m", "outer", "v2", "v1"]); + run(&["tag", "-a", "-m", "s1", "s1", "HEAD"]); + run(&["tag", "-a", "-m", "s2", "s2", "HEAD"]); + let commit = run(&["rev-parse", "HEAD"]); + let v1 = run(&["rev-parse", "v1"]); + let v2 = run(&["rev-parse", "v2"]); + let s1 = run(&["rev-parse", "s1"]); + let s2 = run(&["rev-parse", "s2"]); + run(&["tag", "-d", "v1"]); + + let set = reachable_commit_tag_oids(&bare).unwrap(); + assert!(set.contains(&commit), "the commit must be reachable"); + assert!( + set.contains(&v2), + "the outer tag object (ref tip) must be present" + ); + assert!( + set.contains(&v1), + "the INNER tag object of a tag-of-a-tag must be present (peeled from v2, no ref)" + ); + assert!(set.contains(&s1), "sibling tag s1 must be present"); + assert!(set.contains(&s2), "sibling tag s2 must be present"); + } + + /// #173 (jatmn tag fan-out): the object bound is load-bearing. A repo whose tag + /// count exceeds the bound must FAIL CLOSED (Err), not return a truncated set that + /// would under-withhold a still-reachable tag. Drives `walk_tag_chain` with a tiny + /// injected bound (the public fn uses the real `MAX_TAG_OBJECTS`); with the bound + /// check removed this would collect all tags and return Ok. + #[test] + fn walk_tag_chain_fails_closed_over_object_bound() { + let (_td, bare, _secret, _public) = fixture(); + let run = |args: &[&str]| { + assert!( + Command::new("git") + .args(args) + .current_dir(&bare) + .status() + .unwrap() + .success(), + "git {args:?} failed" + ); + }; + run(&["config", "user.email", "t@t"]); + run(&["config", "user.name", "t"]); + let mut seeds = Vec::new(); + for n in 0..5 { + let name = format!("t{n}"); + run(&["tag", "-a", "-m", &name, &name, "HEAD"]); + let oid = Command::new("git") + .args(["rev-parse", &name]) + .current_dir(&bare) + .output() + .unwrap(); + seeds.push(String::from_utf8_lossy(&oid.stdout).trim().to_string()); + } + + // Within a generous bound: the walk succeeds and collects the tags. + let mut ok_set = HashSet::new(); + walk_tag_chain( + &bare, + "git", + seeds.clone(), + &mut ok_set, + 8192, + Instant::now() + WALK_TIMEOUT, + ) + .unwrap(); + assert!( + seeds.iter().all(|s| ok_set.contains(s)), + "all 5 tags collected under a generous bound" + ); + + // Under a bound of 2 with 5 tags: fail closed (Err), not a partial set. + let mut small_set = HashSet::new(); + let result = walk_tag_chain( + &bare, + "git", + seeds, + &mut small_set, + 2, + Instant::now() + WALK_TIMEOUT, + ); + assert!( + result.is_err(), + "a tag count exceeding the object bound must fail closed (Err), not truncate" + ); + } + #[test] fn anonymous_caller_withholds_only_private_blob() { let (_td, bare, secret_oid, public_oid) = fixture(); diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 4a632b6d..5d4579a3 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -11,1154 +11,3592 @@ use anyhow::Result; use gitlawb_core::cid::Cid; use std::time::{Duration, Instant}; -/// Wall-clock ceiling on one [`pin_new_objects`] batch. -/// -/// The loop runs under a `pin_semaphore` permit and that pool defers rather than -/// sheds, so without a ceiling the hold is O(N) with N (the push's object count) -/// chosen by the pusher. This bounds the drain of a saturated pool instead. -/// -/// 120s is 12x the shared client's 10s whole-request ceiling, so a single large -/// healthy upload that needs more than the client default still has room to -/// finish (the per-request timeout is set to the remainder, not the default), -/// while a batch of them still cannot hold the permit indefinitely. Deliberately -/// a constant and not a config knob: the value only has to be large enough to be -/// uninteresting on a healthy node, and a knob is operator surface that would -/// have to be documented, validated, and kept meaningful. -pub const PIN_BATCH_BUDGET: Duration = Duration::from_secs(120); - -/// The smallest remainder worth starting a bounded git read (or an add) with. +/// How much READ work one source-less legacy row may cost a boot-time sweep. /// -/// A 1ms remainder otherwise buys a child spawned already past its deadline, which -/// can only be reaped: the watchdog's SIGTERM grace plus its post-SIGKILL settle are -/// paid in full for work that produces nothing, once per remaining object. Breaking -/// the batch instead is the same spawn-to-reap amplification the bounded type probe -/// already refuses when it declines a confirming re-probe it cannot afford. +/// The contract this number encodes: discovery for one pre-provenance row is allowed +/// at most this many bounded object reads from warm local repos, whatever the node's +/// repo count. The unit counted is the expensive one, a `git cat-file` pair against a +/// candidate repo; a candidate rejected at filter time (quarantined, cold, an unsafe +/// path) costs nothing against it. Without the cap the sweep is the O(repos x objects) +/// fan-out this subsystem's cost rule exists to forbid, paid at boot on the node with +/// the most history. /// -/// ~1100ms tracks `visibility_pack`'s 1s SIGTERM grace plus its 20ms settle plus -/// margin. Both of those are private to that module, so the value is named once here -/// and documented rather than guessed separately in each loop. -pub(crate) const PIN_READ_FLOOR: Duration = Duration::from_millis(1100); +/// It happens to equal the resolver's serve-time per-request source cap +/// (`db::MAX_PIN_SOURCES`), but it is a different bound with a different owner: that +/// one bounds how many sources ONE `/ipfs` request may gate, this one bounds how many +/// repos ONE background row may read. If either moves, the other does not follow. +pub(crate) const MAX_LEGACY_DISCOVERY_PROBES: usize = 16; -/// The shared outbound client for both IPFS sinks. -/// -/// `pin_new_objects` runs while holding a `pin_semaphore` permit and that pool -/// defers rather than sheds, so an unbounded await here parks the pool. A bare -/// `reqwest::Client::new()` has no timeout, which is exactly that. Built from -/// `crate::build_http_client` rather than a local builder: its docstring forbids -/// hand-rolling an equivalent, so that the redirect and timeout guarantees the -/// node's tests bind stay bound to the client every outbound path actually uses. -fn http_client() -> &'static reqwest::Client { - static CLIENT: std::sync::OnceLock = std::sync::OnceLock::new(); - CLIENT - .get_or_init(|| crate::build_http_client().expect("failed to build production http client")) +// Test-only cost counters for the sweep's discovery load: how many keyset PAGES of +// `repos` one `load_discovery_ctx` bought, and how many ROWS they carried. A load that +// pages the table to exhaustion and one that stops as soon as the probe window is full +// are indistinguishable by outcome, so the window contents cannot go red on the +// difference; the paging cost is the only thing that can. +#[cfg(test)] +thread_local! { + static DISCOVERY_REPO_PAGES: std::cell::Cell = const { std::cell::Cell::new(0) }; + static DISCOVERY_REPO_ROWS: std::cell::Cell = const { std::cell::Cell::new(0) }; } -/// Pin a single git object to the local IPFS/Kubo node. -/// -/// - `ipfs_api`: base URL of the Kubo HTTP API, e.g. `http://127.0.0.1:5001`. -/// If empty the function returns `Ok("")` immediately. -/// - `sha256_hex`: the git SHA-256 hex object ID (used only for logging). -/// - `data`: raw git object content bytes (same bytes used for CID computation). -/// - `request_timeout`: overrides the shared client's whole-request timeout for -/// THIS request only. `RequestBuilder::timeout` replaces the client-level value -/// per request and leaks nothing to other calls on the same client, so the -/// batch loop can hand each add whatever is left of its budget without -/// loosening or tightening any other outbound path. `None` keeps the client's -/// own ceiling. -/// -/// Returns the CID string on success, or `""` when IPFS is not configured. -pub async fn pin_git_object( - ipfs_api: &str, - sha256_hex: &str, - data: &[u8], - request_timeout: Option, -) -> Result { - if ipfs_api.is_empty() { - return Ok(String::new()); - } - - // Compute the expected CIDv1 from the content bytes - let expected_cid = Cid::from_git_object_bytes(data).to_string(); +#[cfg(test)] +pub(crate) fn reset_discovery_paging() { + DISCOVERY_REPO_PAGES.with(|c| c.set(0)); + DISCOVERY_REPO_ROWS.with(|c| c.set(0)); +} - let url = format!( - "{}/api/v0/add?cid-version=1&raw-leaves=true&pin=true", - ipfs_api.trim_end_matches('/') - ); +#[cfg(test)] +pub(crate) fn discovery_repo_pages() -> usize { + DISCOVERY_REPO_PAGES.with(|c| c.get()) +} - // Build multipart form with the object data - let part = reqwest::multipart::Part::bytes(data.to_vec()) - .file_name("object") - .mime_str("application/octet-stream")?; - let form = reqwest::multipart::Form::new().part("file", part); +#[cfg(test)] +pub(crate) fn discovery_repo_rows() -> usize { + DISCOVERY_REPO_ROWS.with(|c| c.get()) +} - let mut req = http_client().post(&url).multipart(form); - if let Some(t) = request_timeout { - req = req.timeout(t); - } +#[cfg(test)] +fn note_discovery_page(rows: usize) { + DISCOVERY_REPO_PAGES.with(|c| c.set(c.get() + 1)); + DISCOVERY_REPO_ROWS.with(|c| c.set(c.get() + rows)); +} - let resp = req - .send() - .await - // Keep the `reqwest::Error` as this error's source rather than - // formatting it away. Operators reading a pin failure want the concrete - // transport cause in the logged chain, not a single flattened line, and - // this module's tests downcast to it to prove a silent endpoint really - // surfaces as a timeout rather than as some other failure that happens - // to arrive in time. - // The context keeps the old message verbatim so the callers that log - // this at `%e` (here, `sync.rs`, `encrypted_pin.rs`) read the same. - .map_err(|e| { - let msg = format!("IPFS add request failed: {e}"); - anyhow::Error::new(e).context(msg) - })?; +/// Attempts (including the first) for a transient DB-record retry. +const PIN_RECORD_ATTEMPTS: u32 = 3; +/// Backoff between DB-record retry attempts. +const PIN_RECORD_BACKOFF: Duration = Duration::from_millis(50); - if !resp.status().is_success() { - let status = resp.status(); - let body = resp.text().await.unwrap_or_default(); - return Err(anyhow::anyhow!( - "IPFS /api/v0/add returned {status}: {body}" - )); +/// Run an idempotent DB-record operation with a bounded retry so a sub-second +/// transient error does not silently leave the pin-source set permanently +/// incomplete. The resolver treats a nonempty below-cap source set as complete, +/// so a dropped `record_pin_source`/`record_pinned_cid` makes `GET /ipfs/{cid}` +/// 404 a valid public copy. Every wrapped insert is idempotent (`ON CONFLICT DO +/// NOTHING` / provenance-preserving upsert), so re-running is safe. On exhausted +/// attempts the last error is returned and the caller records the durable +/// `pin_sources_incomplete` marker (U3, #173), which is what keeps the resolver's +/// bounded scan fallback available for that object instead of 404ing a public copy. +/// Shared with the `pinata.rs` twin so both pin paths retry identically. Runs +/// inside the already-detached post-push task, so the backoff adds no push latency. +pub(crate) async fn retry_db_record(mut op: F) -> Result<()> +where + F: FnMut() -> Fut, + Fut: std::future::Future>, +{ + let mut attempt = 1; + loop { + match op().await { + Ok(()) => return Ok(()), + Err(e) => { + if attempt >= PIN_RECORD_ATTEMPTS { + return Err(e); + } + tokio::time::sleep(PIN_RECORD_BACKOFF).await; + attempt += 1; + } + } } +} - // Kubo returns newline-delimited JSON; we only care about the last object - // (there's typically just one for a single-file add). - let body = resp - .text() - .await - .map_err(|e| anyhow::anyhow!("IPFS add response body read failed: {e}"))?; - let cid = body - .lines() - .filter(|l| !l.trim().is_empty()) - .filter_map(|line| { - let v: serde_json::Value = serde_json::from_str(line).ok()?; - v["Hash"].as_str().map(|s| s.to_string()) - }) - .next_back() - .unwrap_or(expected_cid.clone()); +/// The smallest bound a durability write is given, however little of the batch +/// deadline is left. +/// +/// `batch_budget_gate` only guarantees [`PIN_READ_FLOOR`] before an object STARTS, +/// and the add is handed the whole remainder, so a successful add can finish with +/// ~0 left. An unfloored bound would then fail a write that today completes in +/// milliseconds: on the add path the bytes would sit in Kubo with no `pinned_cids` +/// row, so nothing could resolve the CID, and on the skip branch the source record +/// would fail AND its compensating `mark_pin_sources_incomplete` would fail with +/// it, producing exactly the incomplete-set-without-marker state the marker exists +/// to prevent. The grace exists so a spent batch deadline degrades to a slightly +/// late permit release, never to a dropped durability write. +pub(crate) const DB_RECORD_GRACE: Duration = Duration::from_secs(2); - tracing::debug!(sha256 = %sha256_hex, %cid, "pinned git object to IPFS"); - Ok(cid) +/// Why a DB call bounded by the batch deadline did not return a value. +/// +/// The two arms are kept apart because an operator has to be able to tell a stalled +/// batch (every object timing out at once) from scattered per-object DB failures, and +/// because what an elapsed bound MEANS is not the same claim as a definite error even +/// where the two lead to the same compensation. Every warn line at a bounded site +/// names which arm fired. +#[derive(Debug)] +pub(crate) enum BoundedDbError { + /// The batch deadline was reached with the call still in flight. + /// + /// Whether this means "definitely did not happen" or "outcome unknown" is a + /// property of the OPERATION, not of the timeout, so each call site has to decide + /// it from the shape of the call it wrapped. `tokio::time::timeout` cancels the + /// client future; it does not cancel a statement Postgres has already started. + /// The two shapes that follow from that: + /// + /// - a MULTI-STATEMENT operation that ends in an explicit `tx.commit()` + /// DEFINITELY did not land. The cancelled future never reaches the commit, so no + /// COMMIT is ever sent and Postgres discards the transaction when the connection + /// is reset. `Db::record_pin_source` and `Db::record_pinned_cid_with_source` are + /// this shape, and a site that compensates for a definite error must compensate + /// here too; + /// - a SINGLE AUTOCOMMIT statement may still land server-side after this arm is + /// taken, because the statement is already running and nothing cancels it. + /// `Db::mark_pin_sources_incomplete` and `Db::record_pinata_cid` are this shape, + /// and nothing downstream may treat this arm as evidence the write did not + /// happen. + Elapsed, + /// The DB operation itself failed, definitely and with a cause. + Db(anyhow::Error), } -/// Fetch raw bytes for a CID from the local Kubo node (`/api/v0/cat`). -pub async fn cat(ipfs_api: &str, cid: &str) -> Result> { - if ipfs_api.is_empty() { - return Err(anyhow::anyhow!("IPFS not configured")); +impl std::fmt::Display for BoundedDbError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Elapsed => write!(f, "batch deadline reached with the DB call in flight"), + Self::Db(e) => write!(f, "{e}"), + } } - let url = format!("{}/api/v0/cat?arg={}", ipfs_api.trim_end_matches('/'), cid); - let resp = http_client().post(&url).send().await?; - if !resp.status().is_success() { - return Err(anyhow::anyhow!("ipfs cat {cid}: {}", resp.status())); +} + +impl From for anyhow::Error { + fn from(e: BoundedDbError) -> Self { + match e { + BoundedDbError::Elapsed => anyhow::anyhow!("{e}"), + // Keep the real cause chain rather than flattening it to a string. + BoundedDbError::Db(inner) => inner, + } } - Ok(resp.bytes().await?.to_vec()) } -/// The batch's remaining wall-clock, or `None` once too little is left to be worth -/// starting an object's work with, after logging the truncation exactly once. +/// Bound one DB operation by the batch deadline. /// -/// "Too little" is [`PIN_READ_FLOOR`], not zero: a remainder that cannot cover a -/// bounded git read's teardown buys a child spawned already past its deadline, and a -/// sub-millisecond per-request timeout buys a doomed add whose failure reads as an -/// endpoint fault rather than as budget truncation. +/// What this bounds is the PERMIT HOLD. Both pin loops run under a global +/// `pin_semaphore` permit and that pool defers rather than sheds, so a bare DB +/// await inside the budgeted region parks the permit for as long as the query is +/// stuck; once every pin permit is so held, post-push IPFS replication stops for +/// every repository on the node. `batch_budget_gate` cannot fix that, because it +/// only gates BETWEEN objects and cannot preempt a call already in flight. /// -/// Shaped like `api::ipfs`'s `budget_gate` on purpose: the nonzero-ness rides in -/// the returned value, so every call site must consume it as -/// `let Some(x) = ... else { break }` and a zero `Duration` can never reach a -/// request as its timeout. +/// Takes the ABSOLUTE `deadline`, not a duration, so a slow predecessor cannot hand +/// a later call a fresh full budget: the remainder is measured from the same fixed +/// point every time, which is what keeps N calls inside ONE budget instead of N. /// -/// `sink` labels the backend in the truncation warn ("IPFS" or "Pinata"). The gate -/// is shared by both pin loops rather than copied per loop, so the two cannot drift -/// apart in how they report a truncated batch. -pub(crate) fn batch_budget_gate( - sink: &str, - deadline: Instant, - pinned: usize, - unattempted: usize, -) -> Option { +/// Callers must map the elapsed arm PER SITE, from the shape of the operation they +/// wrapped, rather than folding it into their existing error arm or assuming one +/// meaning for all of them. `timeout` cancels the client future, never the statement +/// Postgres is already running, so an autocommit statement can land server-side after +/// this returns [`BoundedDbError::Elapsed`] while a multi-statement transaction whose +/// `tx.commit()` is never reached definitely cannot. See the arm's own docs for which +/// operations here are which. +pub(crate) async fn db_bounded(deadline: Instant, fut: F) -> Result +where + F: std::future::Future>, +{ let left = deadline.saturating_duration_since(Instant::now()); - if left < PIN_READ_FLOOR { - tracing::warn!( - sink, - pinned, - unattempted, - "pin batch deadline reached; the remaining objects are left unpinned" - ); - return None; + match tokio::time::timeout(left, fut).await { + Ok(Ok(v)) => Ok(v), + Ok(Err(e)) => Err(BoundedDbError::Db(e)), + Err(_elapsed) => Err(BoundedDbError::Elapsed), } - Some(left) } -/// Pin any of the given candidate git objects that are not yet recorded in -/// `pinned_cids`. -/// -/// `object_list` is the already-withheld-filtered OID set to pin: the caller -/// applies `visibility_pack::replicable_objects` on the delta path or the -/// `..._fail_closed` filter on the full-scan path before calling, so this -/// function never sees a withheld blob. `repo_path` is still needed to read each -/// object's bytes, and `git_bin` names the binary those reads run: the production -/// callers pass the literal `"git"`, and a test passes a fake so the loop's own bound -/// can be driven with a git that never answers. +/// The deadline a durability write gets: the batch deadline, floored at +/// [`DB_RECORD_GRACE`] from now so a spent budget cannot drop the write. +pub(crate) fn db_record_deadline(deadline: Instant) -> Instant { + std::cmp::max(deadline, Instant::now() + DB_RECORD_GRACE) +} + +/// Opportunistically repair a legacy provider-CID row on the already-pinned skip +/// path (#173 R8, KTD8). Releases before this branch stored the PROVIDER CID +/// (Kubo dag-pb / Pinata CIDv0) in `pinned_cids.cid`; the `/ipfs` resolver +/// recomputes the raw CID from object bytes and 404s any row whose key does not +/// match, yet `list_pinned_cids` still advertises the stored key — so a client +/// gets a CID the resolver deliberately withholds. When a re-push carries the +/// object again, rewrite the key to the raw CID and stash the old provider value +/// in `legacy_provider_cid`. /// -/// # What `batch_budget` does and does not bound +/// COST GATE: candidacy is decided from the stored key's codec alone — a +/// CIDv1/raw key is already the resolver key and reads NO bytes, keeping the +/// steady-state skip cost DB-only. Only a legacy-codec row reads the object to +/// recompute. A row whose bytes are gone stays withheld (no destructive rewrite). /// -/// The loop holds a `pin_semaphore` permit and that pool defers rather than -/// sheds, so the hold has to be bounded by something other than the pusher's -/// object count. Three things here are: +/// The read's `deadline` is the CALLER's to set, because the two kinds of caller can +/// afford very different holds. Both pin loops run this while holding a `pin_semaphore` +/// permit, so they clamp it to the batch deadline: left at `git_service_timeout_secs` +/// (600s by default) one wedged `cat-file` would hold a GLOBAL pin slot for five times +/// `PIN_BATCH_BUDGET` and starve every other repo's pin work, and the loop's own budget +/// gate cannot preempt a call already in flight. The boot sweep holds no permit and has +/// no batch to overrun, so it passes the plain `git_timeout`. +pub(crate) async fn repair_legacy_provider_cid( + repo_path: &std::path::Path, + git_bin: &str, + deadline: std::time::Instant, + sha: &str, + db: &crate::db::Db, +) -> Result { + // Bounded by the SAME `deadline` the git read below uses (F3, #173): both pin + // loops call this with the pin permit held, so a bare await here parked that + // permit exactly the way the loop bodies' own awaits did. A grep over the loop + // bodies cannot see this site, which is why it is bounded from inside. + let stored = match db_bounded(deadline, db.cid_for_oid(sha)).await? { + Some(c) => c, + None => return Ok(RepairOutcome::Settled), + }; + // Cost gate: a canonical raw CIDv1 key is already correct — never read bytes. + if gitlawb_core::cid::is_raw_cidv1(&stored) { + return Ok(RepairOutcome::Settled); + } + // Legacy-codec row: read the object to recompute. Counted so a test can prove + // the gate above spares non-legacy rows this read. + #[cfg(test)] + note_legacy_repair_read(); + // `read_object_bounded` is SYNCHRONOUS `git cat-file`, and its budget is + // `git_service_timeout_secs` (600 by default), so running it inline parks a tokio + // worker for as long as git takes: one wedged read on the sweep's first pass at boot + // holds a worker for ten minutes, per legacy row. Push it to the blocking pool, the + // same shape `replication_withheld_set` uses in api/repos.rs (#173 round 11, F4). + // Both callers of this function are async, so neither changes shape. The read-counter + // increment above stays on THIS thread so the thread_local cost-gate assertion holds. + let read = { + let repo_path = repo_path.to_path_buf(); + let git_bin = git_bin.to_string(); + let sha = sha.to_string(); + // The shared-deadline form: `read_object_bounded` composes its type probe and + // content read under ONE deadline, rather than granting each stage a full budget, + // so a legacy row's repair read is bounded in total by whatever the caller set. + tokio::task::spawn_blocking(move || { + crate::git::store::read_object_bounded(&git_bin, &repo_path, &sha, deadline) + }) + .await + }; + let data = match read { + Ok(Ok(Some((_ty, bytes)))) => bytes, + // Bytes gone: the row stays withheld, never destructively rewritten. Nothing a + // later pass changes, so this is a TERMINAL outcome for the sweep's re-walk gate. + Ok(Ok(None)) => return Ok(RepairOutcome::Settled), + // A wedged/D-state `git cat-file` (timeout/infra): the repair is opportunistic + // and best-effort, so skip it and return Ok so the pin task PROCEEDS to + // requeue_or_release rather than hanging the coalescing key until process death + // (grok F2, #173). A later re-push or the deferred sweep retries the repair. + Ok(Err(e)) => { + tracing::warn!(sha = %sha, err = %e, "skipping legacy provider-CID repair: bounded object read failed"); + return Ok(RepairOutcome::Retryable); + } + // The blocking task panicked or was cancelled: same best-effort treatment, and + // worth another walk because it says nothing about the row itself. + Err(e) => { + tracing::warn!(sha = %sha, err = %e, "skipping legacy provider-CID repair: object read task failed"); + return Ok(RepairOutcome::Retryable); + } + }; + let raw = Cid::from_git_object_bytes(&data).to_string(); + if raw == stored { + return Ok(RepairOutcome::Settled); + } + db_bounded(deadline, db.repair_legacy_provider_cid(sha, &raw, &stored)).await?; + Ok(RepairOutcome::Repaired) +} + +/// What one opportunistic repair did with a row, so the sweep can tell a skip a later +/// run could fix from one nothing will (U4 re-walk, #173 round 11). The push skip path +/// ignores the value: it repairs whatever the push happens to carry and a failure there +/// is already warn-only. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum RepairOutcome { + /// Nothing to do, or nothing a re-walk would change: the stored key is already the + /// raw resolver key, the recomputed key matches it, or the object's bytes are gone. + Settled, + /// The bounded object read failed (a wedged `git cat-file`, an unreadable repo). + /// The bytes may be readable later, so the row is worth walking again. + Retryable, + /// The row's key was rewritten to the raw-content CID. + Repaired, +} + +/// What one sweep pass (or a whole sweep run) did. `scanned` counts `pinned_cids` +/// rows READ, which is the quantity the batch size bounds; `repaired` counts rows +/// whose key was actually rewritten to the raw CID. +#[derive(Debug, Default, PartialEq, Eq)] +pub(crate) struct SweepStats { + pub scanned: usize, + pub repaired: usize, + pub passes: usize, + /// Rows left unrepaired for a reason a LATER run could fix (the source repo is not + /// on this node's local disk, a DB read failed, a bounded object read failed). Rows + /// that are unrepairable in principle (no provenance, the repo row is gone, the + /// bytes are gone) are NOT counted here. + /// + /// This drives NO control decision. It gated the cursor rewind under round 11; the + /// rewind now fires on reaching the end of the table, whatever happened on the way + /// (see [`sweep_legacy_provider_cids`]). Re-gating it on this field reopens the + /// below-cursor rolling-upgrade hole, because the run that parks the cursor is a + /// clean one by definition. The field is reporting only. + pub retryable_skips: usize, + /// Object reads spent on rows that turned out to be unrepairable in principle: the + /// bytes are gone, so the read is pure waste and the next run will waste it again. + /// [`MAX_DEAD_ROW_READS_PER_RUN`] bounds this per run. + pub dead_row_reads: usize, + /// Whether at least one source-less row was reached with the pass's whole discovery + /// budget already spent, so it was skipped without a probe (see + /// [`DISCOVERY_ROW_BUDGET_DIVISOR`]). Reporting only, like `retryable_skips`: it + /// drives no control decision, it just keeps a starved pass from being silent. + pub discovery_budget_spent: bool, + /// Why the run stopped. Meaningful on a RUN (`sweep_legacy_provider_cids` and the + /// re-arm wrapper); on a single pass it is always `Completed` and says nothing. + pub stop: SweepStop, +} + +/// Why a sweep run ended, which is what [`run_sweep_rearmed`] dispatches on. /// -/// - this loop's own wall-clock: the deadline is taken once at loop start and -/// checked at the top of every iteration, so no object's work begins with less -/// than [`PIN_READ_FLOOR`] left. It is a gate, not a hard ceiling, since a started -/// iteration still runs to completion; -/// - the git read: `store::read_object_bounded` runs under `spawn_blocking` against the -/// ABSOLUTE batch deadline (not the loop-top remainder, which the `is_pinned` round-trip -/// sitting between the two would push past it), with SIGTERM-then-SIGKILL -/// process-group teardown, so a hung `git cat-file` costs this batch its remaining -/// budget plus one watchdog teardown instead of holding the permit for the child's -/// whole lifetime and blocking a runtime worker while it does; -/// - each HTTP add: `pin_git_object` is handed the remainder measured AFTER the read -/// as its per-request timeout, which is what lets one large healthy upload run past -/// the shared client's 10s default without letting the batch run forever. Measuring -/// it after the read is what keeps the read-plus-add pair inside one budget rather -/// than up to two of them. +/// All three arms are re-armable; what differs is how long the wrapper waits. A run that +/// walked to the end of the table and a run that paused on +/// [`MAX_DEAD_ROW_READS_PER_RUN`] both left the node in a state a later run improves, so +/// the wrapper sleeps and goes again. A failing pass QUERY is a broken database, so it +/// waits far longer (see [`SWEEP_REARM_DELAY`]) rather than turning one logged +/// failure into a stream of them, but it does go again: exiting for good made a single +/// deadlock or connection reset disable legacy-CID repair for the whole process +/// lifetime. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub(crate) enum SweepStop { + /// The ordered walk reached the end of the table (a short batch). + #[default] + Completed, + /// Enough fruitless reads for one run; the cursor stays mid-table. + PausedOnDeadReadCap, + /// A pass's batch query or cursor write failed. + PassFailed, +} + +/// How many fruitless object reads one sweep run will spend before it stops and leaves +/// the rest of the table for the next run (#173 round 12, second-model pass). /// -/// So the LOOP's hold is bounded by roughly `batch_budget` plus one teardown. Two -/// things inside that region still are not, and the gate cannot fix either: +/// A completed run rewinds, so every later run re-attempts the read for every row whose +/// bytes are permanently gone. Without a bound that is `O(dead rows)` `git cat-file` +/// invocations on every single boot, and a node that accumulated a lot of them (a +/// deleted repo, a force-pushed history, a failed migration) pays it forever. Stopping +/// early keeps the cursor, so the next boot resumes past the rows already walked rather +/// than repeating them, and the table still gets covered across boots. +pub(crate) const MAX_DEAD_ROW_READS_PER_RUN: usize = 64; + +/// How the pass's discovery budget is sliced per source-less row (#173 round 13, F6). /// -/// - the DB round-trips (`is_pinned`, `record_pinned_cid`). -/// - the pool. `api::repos` acquires the same `pin_semaphore` for the Pinata -/// replication task and holds it across `pinata_object_list_for_refs`, a full git -/// re-derivation that runs BEFORE `pinata::pin_new_objects` is entered and whose -/// per-child timeouts carry no aggregate deadline. What is bounded is each pin -/// loop's own hold, not the permit's total hold and not the semaphore's worst-case -/// queue. +/// The pass budget alone is not enough. It is one `git_timeout` shared by every +/// source-less row the pass reaches, so a single wedged candidate on the first row spent +/// all of it and every later row arrived with a dead deadline, came back retryable +/// without a real probe, and starved. `sha256_hex` order is stable, so the same row won +/// the race on every boot and the rows behind it were never probed at all. /// -/// # Truncation semantics +/// The trade this number sets, stated both ways so neither half is silent: at least four +/// rows are guaranteed a live probe out of one pass budget, and no single row may spend +/// more than a quarter of it. Raising the divisor guarantees more rows per pass and gives +/// each a shorter probe; lowering it does the reverse. Four keeps a row's slice generous +/// against the default 600s `git_service_timeout_secs` (150s, far past any healthy +/// `cat-file`) while still bounding the damage one wedged candidate can do. /// -/// A batch stopped at the deadline leaves its remaining objects unpinned, and -/// nothing sweeps them up afterwards. There is no reconciliation pass over -/// `pinned_cids`; recovery is opportunistic, happening only if some later push -/// on the repo takes the full-scan fallback (`push_delta::list_all_objects`) and -/// re-derives the whole object set, which then re-offers the skipped OIDs. +/// It is NOT a bound on a single probe. Within a row's slice the per-row deadline is +/// shared by up to [`MAX_LEGACY_DISCOVERY_PROBES`] candidates exactly as the pass +/// deadline used to be shared by rows, so one wedged candidate can still consume its +/// row's whole slice and leave that row's later candidates unprobed. What this bounds is +/// the blast radius: the row, not the pass. +const DISCOVERY_ROW_BUDGET_DIVISOR: u32 = 4; + +/// The warm, non-quarantined repos one pass may probe for a source-less legacy row, +/// plus the absolute deadline bounding the pass's discovery as a whole, from which each +/// row takes a slice. /// -/// The twin in `pinata.rs` is back at parity on the two things that bound a -/// batch: it runs the same shared budget gate at the top of every iteration and -/// the same bounded, reaped git read. It still has no per-request override, since -/// `pinata::pin_object` takes no timeout argument and its uploads are bounded by -/// the shared client's own ceiling. Everything else about the shape (the -/// skip-if-pinned check, the fault arms, the returned pairs) changes in lockstep. +/// Loaded LAZILY, once per pass, on the first source-less row, mirroring the resolver's +/// own legacy-scan context: a pass with no such row pays nothing. The `is_dir` warm +/// filter runs ONCE here rather than per row, on the blocking pool, because O(repos) +/// stat calls per row would park a tokio worker for the whole boot sweep. +struct DiscoveryCtx { + /// Warm candidates with their RAW `(created_at, id)` keyset key and validated disk + /// path, ROTATED so the traversal's window starts at the head. + /// + /// The key is `ScanRepoRow::created_at_key`, the stored text, carried through rather + /// than re-derived from `RepoRecord::created_at`: re-serializing the parsed + /// `DateTime` is not guaranteed to reproduce the stored bytes (that struct says so + /// itself), and the keyset comparison this feeds is a TEXT comparison against the + /// SQL order, so a key off by one character rotates the list to a boundary the query + /// never had. + candidates: Vec<(crate::db::RepoRecord, String, std::path::PathBuf)>, + /// Whether the node's WHOLE warm candidate set fits in one window, which is the + /// condition the traversal's continuation reset arm turns on. + /// + /// A separate field because `candidates` can no longer answer it. The load stops as + /// soon as the window is full, so `candidates.len()` is `MAX_LEGACY_DISCOVERY_PROBES` + /// on a node with seventeen warm repos and on a node with seventeen thousand alike. + /// `load_discovery_ctx` collects one candidate past the window purely to decide this. + warm_fits_under_cap: bool, + /// The ceiling on the whole pass's discovery, so one pass costs at most one + /// `git_timeout` in total on top of the per-row probe cap. Per PASS, not per run: + /// `load_discovery_ctx` runs once per `sweep_pass` and a run loops passes. + /// + /// No row gets all of it. Each takes at most + /// `git_timeout / DISCOVERY_ROW_BUDGET_DIVISOR`, clamped to what is left here, and a + /// row reached with this already past is skipped without a probe. + pass_deadline: Instant, +} + +/// What one TRAVERSAL of the `pinned_cids` table learned about how far its discovery +/// window actually got, and therefore where the next traversal's window may start. /// -/// Returns a list of `(sha256_hex, cid)` pairs for objects pinned this call. -pub async fn pin_new_objects( - ipfs_api: &str, - repo_path: &std::path::Path, - git_bin: &str, - object_list: Vec, - db: &crate::db::Db, - batch_budget: Duration, -) -> Vec<(String, String)> { - if ipfs_api.is_empty() { - return vec![]; +/// Owned by [`run_sweep_rearmed`] and passed `&mut` through every run and every pass of +/// the traversal, which is the whole point of the type: the dead-read cap can PAUSE a +/// run in the middle of a traversal, and the run that later reaches the short batch is a +/// different run. Rebuilding this per run means that final run sees an empty accumulator +/// and applies the hold arm (or the reset arm) for windows the earlier run really did +/// probe, so the traversal advances by nothing and the sweep stalls on the same window +/// forever. Its lifetime is the traversal, so that is what it is scoped to. +#[derive(Debug, Default)] +pub(crate) struct DiscoveryTraversalState { + /// The `(created_at_key, id)` of the last candidate whose probe STARTED with the + /// row's deadline still live. + /// + /// A probe started against a dead deadline is charged a read (the U3 boundary row is + /// exactly this) but learns nothing: `db_bounded` returns immediately and the + /// candidate is left unread. Advancing over one would skip a candidate nobody looked + /// at, which is the same hole the continuation exists to close, one window narrower. + last_live_probe: Option<(String, String)>, + /// A row reached the probe cap with candidates still unprobed AND spent at least one + /// live-budget probe doing it. This is the arm that ADVANCES: there is a next window + /// and the traversal earned the right to move to it. + cap_exhausted_with_budget: bool, + /// The whole warm list fit inside one window, observed by a row with live budget. + /// There is no next window, so the continuation RESETS: leaving a stale key behind + /// on a list that has since shrunk below it would rotate every later traversal to an + /// empty tail and then wrap to the same prefix forever. + fit_under_cap: bool, +} + +impl DiscoveryTraversalState { + /// The advance to apply at the end of a completed traversal, or `None` to hold the + /// continuation where it is. + /// + /// Three arms, in this order. ADVANCE when a row ran out of window with budget left + /// to spend, to the last candidate actually read live. RESET when the list fit under + /// the cap, because there is nothing past the window to advance to. HOLD otherwise, + /// which is the starved traversal: nothing was probed live, so burning a window + /// would skip candidates on the strength of reads that never happened. + fn advance(&self) -> Option<(String, String)> { + if self.cap_exhausted_with_budget { + return self.last_live_probe.clone(); + } + if self.fit_under_cap { + return Some((String::new(), String::new())); + } + None } - let deadline = Instant::now() + batch_budget; - let total = object_list.len(); - let mut pinned = Vec::new(); + /// Fold one finished row's window observation in. + /// + /// A row that read NOTHING with live budget contributes nothing at all, neither arm. + /// Such a row is evidence about the clock, not about the candidates: letting it set + /// either flag would move or reset the window on the strength of probes that were + /// charged but never made. + fn note_row(&mut self, live_probes: usize, fits_under_cap: bool) { + if live_probes == 0 { + return; + } + if fits_under_cap { + self.fit_under_cap = true; + } else { + self.cap_exhausted_with_budget = true; + } + } +} - for (attempted, sha) in object_list.into_iter().enumerate() { - // Top of the iteration, before any of this object's work: an object is - // never started with a remainder too small to cover a bounded read's - // teardown. Consumed as a guard only: the read below runs against the - // absolute batch deadline, and the add's timeout is measured again after - // the read, so this remainder has no other consumer here. - if batch_budget_gate("IPFS", deadline, pinned.len(), total - attempted).is_none() { +/// Build one pass's discovery candidate list. +/// +/// Three filters, all applied before any probe so a rejected candidate costs nothing +/// against [`MAX_LEGACY_DISCOVERY_PROBES`]: +/// +/// - QUARANTINE. A quarantined repo is hidden from every reader, so it must not become +/// a discovery source either. Each page row carries its own `quarantined` flag, so +/// the drop is a filter over the rows this pass already read rather than a second +/// whole-node query. The resolver's legacy scan reads the same rows and drops on the +/// same flag. Private, non-quarantined repos DO stay in the list: an additive source +/// record binds nothing to one repo's ACL, because the resolver gates every source +/// independently at serve time, so probing a private repo leaks nothing. +/// - WARM ONLY. The path is resolved through the repo store's validated resolver and +/// kept only if it is on local disk. Nothing here goes through `repo_store.acquire`: +/// the sweep is opportunistic background maintenance over every pinned row on the +/// node, and pulling cold repos back from remote storage would turn a repair pass +/// into a bulk restore. +/// - UNSAFE PATH. A name that fails the validated resolver is dropped with a warn and +/// is terminal; nothing a later run changes. +/// +/// The candidates are ordered oldest-first by `(created_at, id)` rather than by id +/// alone. `repo_id` derives from the owner DID, which anyone can grind, so an id sort +/// would let an attacker register low-sorting repos and push the true holder past the +/// probe cap. Source-less rows predate provenance and their holders are old repos, +/// while freshly registered repos sort last and cannot be backdated. That order is now +/// the QUERY's (`ORDER BY created_at ASC, id ASC`, index-backed by migration v25) and +/// pages concatenate in it, so the list is globally sorted as it is built and no +/// client-side sort is involved. +async fn load_discovery_ctx( + repos_dir: &std::path::Path, + git_timeout: Duration, + db: &crate::db::Db, +) -> Result { + // Paged only as far as the WINDOW needs, not to exhaustion. + // + // The exhaustive load was defended as "background maintenance on a timer" whose + // "paging cost is paid once", and that was true when the sweep ran once per boot: one + // full-table pass per process lifetime to choose sixteen candidates. The sweep now + // re-arms on a timer, so the cost is paid on every run for as long as the node holds a + // single unrepairable source-less row. The idle backoff stretches that to hourly; it + // does not bound it. Same query as the resolver's legacy scan and still a different + // threat model (no caller to amplify, no scarce permit pinned), but an unbounded read + // that repeats forever is worth stopping on its own account. + // + // The window is unchanged. It is still the first `MAX_LEGACY_DISCOVERY_PROBES` WARM + // candidates strictly after the persisted continuation, wrapping to the front of the + // `(created_at, id)` order when the tail runs out, so the candidates picked here are + // byte for byte the ones the exhaustive load rotated to. What changed is that the + // rotation now STEERS the paging instead of being applied to a list already read: + // phase 0 reads forward from the continuation, phase 1 wraps to the front and stops + // where phase 0 began, and either may stop early once the window is full. + // + // Ordering is still the QUERY's `(created_at, id)` ASC, so non-steerability is + // untouched: `repo_id` derives from a grindable owner DID, but minted repos carry a + // fresh `created_at` and sort LAST, where they can only ever be reached after the + // older true holder rather than instead of it. + // + // Per PASS, not per row: `load_discovery_ctx` still runs once per pass and its result + // is still reused for every source-less row in that pass, so all of them share one + // window. + let page_rows = crate::api::ipfs::LEGACY_SCAN_PAGE_ROWS; + + // Read BEFORE paging, because the load is steered by it now. + let (cont_created_at, cont_id) = db.discovery_continuation().await?; + let resumed = !cont_created_at.is_empty() || !cont_id.is_empty(); + + // ONE PAST the window. The exhaustive load could read "the whole warm list fits under + // the cap" off a total count it had in hand; a bounded load has no total. Collecting + // one extra candidate restores the decision without restoring the cost: a load that + // stops at `MAX + 1` has PROVEN there are more than `MAX` warm candidates, and a load + // that ends at `MAX` or fewer can only have done so by running the whole warm set to + // its end. So `warm.len() <= MAX` after the fact is exactly the old condition. + let want = MAX_LEGACY_DISCOVERY_PROBES + 1; + + let repos_dir = repos_dir.to_path_buf(); + let mut warm: Vec<(crate::db::RepoRecord, String, std::path::PathBuf)> = Vec::new(); + + for phase in 0..2 { + if warm.len() >= want { break; } - // Skip if already pinned - match db.is_pinned(&sha).await { - Ok(true) => continue, - Ok(false) => {} - Err(e) => { - tracing::warn!(sha = %sha, err = %e, "DB error checking pinned status"); - continue; + // Nothing persisted means phase 0 already started at the front, so there is no + // prefix left to wrap into. + if phase == 1 && !resumed { + break; + } + let mut cursor: Option<(String, String)> = if phase == 0 && resumed { + Some((cont_created_at.clone(), cont_id.clone())) + } else { + None + }; + // Phase 1 must not run past the point phase 0 started at, or the wrap would probe + // the same candidates twice and the window would be short by however many it + // repeated. + let stop_after: Option<(&str, &str)> = if phase == 1 { + Some((cont_created_at.as_str(), cont_id.as_str())) + } else { + None + }; + loop { + let need = want.saturating_sub(warm.len()); + if need == 0 { + break; + } + let page = db + .list_repos_page_for_scan( + cursor + .as_ref() + .map(|(created_at, id)| (created_at.as_str(), id.as_str())), + page_rows as i64, + ) + .await?; + #[cfg(test)] + note_discovery_page(page.len()); + let Some(last) = page.last() else { break }; + let last_page = page.len() < page_rows; + cursor = Some((last.created_at_key.clone(), last.repo.id.clone())); + let mut wrapped_to_start = false; + let mut candidates: Vec<(crate::db::RepoRecord, String)> = Vec::new(); + for r in page { + if let Some((created_at, id)) = stop_after { + if (r.created_at_key.as_str(), r.repo.id.as_str()) > (created_at, id) { + wrapped_to_start = true; + break; + } + } + // QUARANTINE, dropped before the stat so a hidden repo costs nothing. + if r.quarantined { + continue; + } + candidates.push((r.repo, r.created_at_key)); + } + warm.extend(warm_candidates(&repos_dir, candidates, need).await?); + if wrapped_to_start || last_page { + break; } } + } - // Read raw object content, bounded and reaped, under `spawn_blocking`: this is - // synchronous blocking work (child spawn, pipe drain, watchdog join), so - // calling it from the runtime task would block a worker thread for its whole - // duration. Placement mirrors the `/ipfs` serve path; the admission guard is - // deliberately NOT moved into the closure there, since the pin permit is not - // held by a future a client disconnect can drop and the child is reaped on its - // own deadline regardless. - // - // The read runs against the ABSOLUTE batch deadline, not against the remainder - // measured at the top of the iteration: the `is_pinned` round-trip above sits - // between the two, so `Instant::now() + budget_left` would land past `deadline` - // by however long the DB took, and under a saturated pool that is the dominant - // term. A slow DB check must not push the read's own bound out. - let read_deadline = deadline; - let read_path = repo_path.to_path_buf(); - let read_sha = sha.clone(); - let read_git = git_bin.to_string(); - let read = tokio::task::spawn_blocking(move || { - crate::git::store::read_object_bounded(&read_git, &read_path, &read_sha, read_deadline) - }) - .await; - let data = match read { - Ok(Ok(Some((_obj_type, bytes)))) => bytes, - // A verified absence, and the only outcome that is not a fault. - Ok(Ok(None)) => continue, - // A Transient fault does NOT by itself mean the store is gone. It also - // covers a spawn or watchdog-timeout failure of the reaped child, an - // unaffordable confirming re-probe, and, because readability is judged FOR - // one oid, a single unreadable `objects/` fan-out, which is 1/256 of the - // store. So re-check store-wide before deciding what the fault costs. - Ok(Err(e @ crate::git::store::ProbeError::Transient(_))) => { - if !crate::git::store::object_store_readable_store_wide(repo_path) { - // Genuinely store-wide: every remaining object fails identically, and - // continuing would spawn one doomed bounded child per object and spend - // the batch budget reaping them. - tracing::warn!( - sha = %sha, - err = %e, - unattempted = total - attempted, - "object store unreadable while pinning; stopping the batch" - ); - break; - } - // The store still reads store-wide, so the fault is object-scoped or - // transient to this read. Breaking would forfeit a healthy store's - // remaining objects permanently: the documented recovery re-derives the - // same list and breaks at the same index. - tracing::warn!( - sha = %sha, - err = %e, - "transient fault reading git object for pinning; the object store is \ - still readable store-wide, so this costs only this object" - ); - continue; + // The fit-under-cap arm the traversal's continuation reset depends on, decided from + // the one extra candidate rather than from a whole-table count (see `want`). + let warm_fits_under_cap = warm.len() <= MAX_LEGACY_DISCOVERY_PROBES; + warm.truncate(MAX_LEGACY_DISCOVERY_PROBES); + + Ok(DiscoveryCtx { + candidates: warm, + warm_fits_under_cap, + pass_deadline: Instant::now() + git_timeout, + }) +} + +/// Keep the WARM ones out of a batch of candidate rows, stopping after `need` of them. +/// +/// The stat runs on the blocking pool because it is O(rows) filesystem calls and would +/// otherwise park a tokio worker for the length of a sweep. `need` is what keeps a page's +/// tail from being stat'd once the window is already full: the caller stops paging at that +/// point, so those rows are never looked at again this pass either. +/// +/// An UNSAFE PATH is dropped with a warn and is terminal, and a COLD repo is simply +/// absent: neither is evidence about any row (see `discover_legacy_row`). +async fn warm_candidates( + repos_dir: &std::path::Path, + candidates: Vec<(crate::db::RepoRecord, String)>, + need: usize, +) -> Result> { + let repos_dir = repos_dir.to_path_buf(); + Ok(tokio::task::spawn_blocking(move || { + let mut out = Vec::new(); + for (repo, created_at_key) in candidates { + if out.len() >= need { + break; } - // The store is readable and git still failed: a corrupt object, or a - // repo-wide fault git reports immediately. Either way it is per-object - // work that stays inside the budget, and breaking would forfeit a healthy - // store's remaining objects over one bad one, permanently (a later - // full-scan push re-offers the same object and breaks in the same place). - Ok(Err(e)) => { - tracing::warn!(sha = %sha, err = %e, "failed to read git object for pinning"); - continue; + match crate::git::repo_store::validated_repo_disk_path( + &repos_dir, + &repo.owner_did, + &repo.name, + ) { + Ok(p) if p.is_dir() => out.push((repo, created_at_key, p)), + Ok(_) => {} + Err(e) => { + tracing::warn!(repo_id = %repo.id, err = %e, "sweep discovery: rejected unsafe repo path"); + } } - // A panic in the read closure leaves no evidence that the failure is - // object-scoped, so fail toward the conservative arm. + } + out + }) + .await?) +} + +/// What discovery did with one source-less legacy row, in the same three-way shape +/// [`RepairOutcome`] uses so the row accounting is unchanged. +enum DiscoveryOutcome { + /// Nothing here a later run would find either. + Settled, + /// Worth walking again: a warm candidate's read failed, the candidate list could + /// not be loaded, or the probe cap was reached with candidates still unprobed. + Retryable, + /// The row's key was rewritten from bytes verified in a warm local repo. + Repaired, + /// The pass's whole discovery budget was already spent when this row was reached, so + /// nothing was probed. Accounted RETRYABLE like the arm above (nothing was learned + /// about the row), but kept distinct because it must cost NOTHING: charging it the + /// reads it never made would burn [`MAX_DEAD_ROW_READS_PER_RUN`] on rows that were + /// only ever skipped, pausing the run early for no information. + PassBudgetSpent, +} + +/// Probe a bounded set of warm local repos for a source-less legacy row's object. +/// +/// On a hit, record ONLY what discovery actually knows. Reading identical bytes proves +/// the repo HOLDS the object, not that it is the first pinner: forks, a shared LICENSE +/// blob and the empty tree all collide, and `backfill_pin_provenance`'s +/// `AND repo_id IS NULL` guard would make a guessed exclusive claim permanent. Worse, +/// the resolver's `needs_scan` is `sources.is_empty() || at_cap || incomplete`, so an +/// exclusive claim would permanently disable the fallback scan for that object. So +/// `pinned_cids.repo_id` stays NULL, the discovered repo goes in ADDITIVELY, and the +/// incomplete marker goes with it because one discovered holder never proves the set +/// complete. +/// +/// Both rows are written by ONE transaction (`record_discovered_pin_source`, U5), never +/// as two independent best-effort calls. Split, a failed sentinel left the row with a +/// nonempty, below-cap, UNMARKED source set, which `needs_scan` reads as complete: the +/// fallback scan is dropped and an unrecorded public duplicate is 404'd permanently. +/// Together they either both land or neither does. +/// +/// The record as a whole is still best-effort and warn-only, and the degradation is +/// stated rather than deferred to a healing pass that does not exist: if it fails the row +/// is raw-CIDv1 with an EMPTY source set, which is exactly the state `needs_scan` routes +/// to the bounded legacy scan, so the object stays servable. The sweep itself never +/// revisits it (the cost gate skips a raw row free from then on), so the resolver's +/// fallback is the healing path, not a retry. +async fn discover_legacy_row( + sha: &str, + ctx: &mut Option>, + repos_dir: &std::path::Path, + git_bin: &str, + git_timeout: Duration, + db: &crate::db::Db, + traversal: &mut DiscoveryTraversalState, +) -> (DiscoveryOutcome, usize) { + let mut reads = 0usize; + if ctx.is_none() { + *ctx = Some(match load_discovery_ctx(repos_dir, git_timeout, db).await { + Ok(c) => Some(c), Err(e) => { - tracing::warn!(sha = %sha, err = %e, "bounded git read task failed; stopping the batch"); - break; + tracing::warn!(err = %e, "sweep discovery: failed to load the candidate list"); + None } - }; + }); + } + let ctx = match ctx.as_ref().expect("the candidate list was just loaded") { + Some(c) => c, + // A failed load says nothing about the row, so a later run retries it. + None => return (DiscoveryOutcome::Retryable, reads), + }; - // Recompute the remainder AFTER the read rather than reusing `budget_left`: - // the read is now allowed to spend the whole remainder, so handing the add the - // loop-top value would make the pair a hold of up to 2x the batch budget. The - // same gate, so a remainder too small to be worth a request truncates the batch - // with one warn instead of shedding a doomed add that logs as an endpoint fault. - let Some(add_timeout) = - batch_budget_gate("IPFS", deadline, pinned.len(), total - attempted) - else { - break; - }; + // Reached with the pass's discovery budget already gone: probing now buys nothing but + // a spent-deadline error per candidate, so return before any read is charged. + if Instant::now() >= ctx.pass_deadline { + return (DiscoveryOutcome::PassBudgetSpent, reads); + } + // This row's slice of the pass budget, clamped to what is left of it. Without the + // clamp a row reached near the end of the pass would overrun the pass's own ceiling; + // without the slice one wedged candidate would spend the whole pass on this row. + let row_deadline = std::cmp::min( + Instant::now() + git_timeout / DISCOVERY_ROW_BUDGET_DIVISOR, + ctx.pass_deadline, + ); - // Pin to IPFS - match pin_git_object(ipfs_api, &sha, &data, Some(add_timeout)).await { - Ok(cid) if !cid.is_empty() => { - if let Err(e) = db.record_pinned_cid(&sha, &cid).await { - tracing::warn!(sha = %sha, err = %e, "failed to record pinned CID in DB"); + let mut retryable = false; + // How many of this row's probes actually STARTED with budget to spend, and whether + // the whole warm list fits in one window. Together they pick the traversal's advance + // arm once the row is done. + let mut live_probes = 0usize; + let fits_under_cap = ctx.warm_fits_under_cap; + // Every candidate that gets this far is READ, so taking the first + // MAX_LEGACY_DISCOVERY_PROBES bounds the expensive work exactly. Candidates the + // filters already rejected never reach here and so cost nothing against the cap. + for (repo, created_at_key, repo_path) in ctx.candidates.iter().take(MAX_LEGACY_DISCOVERY_PROBES) + { + // The live-budget test, taken BEFORE the probe and against the SAME deadline the + // probe is handed. Two shapes reach a probe with the deadline already gone and + // both are charged a read for it: U3's boundary row, admitted by a skip guard of + // `now >= pass_deadline` with a sliver of budget that `row_deadline` clamps to + // nothing, and every candidate queued behind a wedged one inside a row. In both, + // `db_bounded` returns on the spent deadline and the repo is never opened. They + // are reads, not looks, and the continuation must not advance over them: doing so + // skips candidates nobody examined, which is the hole the continuation exists to + // close, one window narrower. + let live = Instant::now() < row_deadline; + if live { + live_probes += 1; + traversal.last_live_probe = Some((created_at_key.clone(), repo.id.clone())); + } + // Counted before the match, because the read is spent whatever it returns. This + // is the quantity the caller charges against the per-run budget. + reads += 1; + match repair_legacy_provider_cid(repo_path, git_bin, row_deadline, sha, db).await { + Ok(RepairOutcome::Repaired) => { + // ONE transaction for both writes (U5, #173). Discovery found ONE holder + // out of a bounded, warm-only candidate set, so the source set is still + // not known complete and the resolver must keep its scan fallback for + // this row; the sentinel that arms it is therefore not a separate + // best-effort write but part of the same commit as the source row. + // + // Marked against the UNKNOWN-repo sentinel rather than the repo just + // recorded, which would be a lie (that repo IS recorded). The sentinel is + // the same one the v24 migration carries pre-upgrade markers under, and it + // means what it means here: a source may be missing and nobody knows + // which, so no real record clears it. + // + // Rebase note (#321 onto the per-(oid, repo) marker): the original wrote + // this marker because `record_pin_source` used to clear the whole + // per-object boolean. It no longer does, so the sentinel went from + // compensating for a clear to being the only thing arming the fallback, + // which is why it may not be allowed to fail on its own. + match db_bounded( + db_record_deadline(row_deadline), + retry_db_record(|| db.record_discovered_pin_source(sha, &repo.id)), + ) + .await + { + Ok(()) => {} + // Elapsed is a DEFINITE non-write here, and that follows from the + // shape of what was wrapped: the record is commit-terminated, so a + // cancelled future never sends the COMMIT. The arm stays separate + // only so the warn tells an operator a stalled DB from a scattered + // per-row failure; both leave the same benign end state below. + Err(e @ BoundedDbError::Elapsed) => { + tracing::warn!( + sha = %sha, + repo_id = %repo.id, + err = %e, + "sweep discovery: the discovered pin source record did not \ + complete inside the row deadline; a cancelled \ + commit-terminated transaction definitely did not land, so \ + the row keeps an empty source set and the resolver falls back" + ); + } + Err(e) => { + tracing::warn!(sha = %sha, repo_id = %repo.id, err = %e, "sweep discovery: failed to record the discovered pin source and its sentinel"); + } } - pinned.push((sha, cid)); + traversal.note_row(live_probes, fits_under_cap); + return (DiscoveryOutcome::Repaired, reads); } - Ok(_) => {} + // The bytes could not be read from this WARM candidate right now, which IS + // evidence about the row: try the next one and walk the row again later. + Ok(RepairOutcome::Retryable) => retryable = true, + // Absent here, or the row was repaired concurrently. Next candidate. + Ok(RepairOutcome::Settled) => {} Err(e) => { - tracing::warn!(sha = %sha, err = %e, "failed to pin git object to IPFS"); + tracing::warn!(sha = %sha, repo_id = %repo.id, err = %e, "sweep discovery: probe failed"); + retryable = true; } } } - - pinned + traversal.note_row(live_probes, fits_under_cap); + if !ctx.warm_fits_under_cap { + // Cap exhausted with candidates left unprobed: RETRYABLE, never terminal. The + // probe order is deterministic, but "a re-walk finds the same nothing" only + // holds if the candidate set cannot be steered, and it can: repo ids derive + // from grindable owner DIDs, so a terminal verdict would let an attacker bury + // the true holder past the cap permanently. The oldest-first order makes that + // expensive, and this arm makes it non-permanent. + return (DiscoveryOutcome::Retryable, reads); + } + if retryable { + (DiscoveryOutcome::Retryable, reads) + } else { + (DiscoveryOutcome::Settled, reads) + } } -#[cfg(test)] -mod tests { - use super::*; - use std::time::Duration; +/// One bounded pass of the U4 sweep: read at most `batch` `pinned_cids` rows after +/// the persisted cursor, repair the legacy ones, and persist the new cursor. +/// +/// The batch is what bounds the pass. It caps rows READ, not rows repaired, because +/// the legacy predicate is a codec decode SQL cannot express; a table of raw rows +/// therefore costs one indexed range scan per pass and nothing else. +/// +/// The cursor advances to the LAST row read whatever happened to each row, including +/// rows that were skipped as unrepairable. A cursor that only advanced on success +/// would re-read the same unrepairable row on every pass and the sweep would never +/// reach the rows behind it. +async fn sweep_pass( + repos_dir: &std::path::Path, + git_bin: &str, + git_timeout: Duration, + batch: i64, + db: &crate::db::Db, + traversal: &mut DiscoveryTraversalState, +) -> Result { + let cursor = db.pin_repair_cursor().await?; + let rows = db.pinned_cids_after(&cursor, batch).await?; + let scanned = rows.len(); + let mut repaired = 0usize; + let mut retryable_skips = 0usize; + let mut dead_row_reads = 0usize; + let mut discovery_budget_spent = false; + let mut last = cursor; + // Loaded on the first source-less row and reused by every later one. The outer + // `None` is "not loaded yet"; `Some(None)` is "the load failed this pass", which is + // remembered so a broken DB is not re-queried once per row. + let mut discovery: Option> = None; - /// Write `n` loose blobs into a fresh bare repo and return their oids. - /// `read_object` shells to `git cat-file`, so the objects must genuinely - /// exist on disk — a fabricated oid would `continue` past the pin call and - /// the loop scenario below would prove nothing. - fn seed_loose_blobs(repo_path: &std::path::Path, n: usize) -> Vec { - crate::git::store::init_bare(repo_path).expect("init bare repo"); - (0..n) - .map(|i| { - let mut cmd = std::process::Command::new("git"); - cmd.args(["hash-object", "-w", "--stdin"]) - .current_dir(repo_path) - .stdin(std::process::Stdio::piped()) - .stdout(std::process::Stdio::piped()); - let mut child = cmd.spawn().expect("spawn git hash-object"); - { - use std::io::Write; - child - .stdin - .as_mut() - .expect("stdin") - .write_all(format!("pin loop object {i}\n").as_bytes()) - .expect("write stdin"); + for (sha, stored) in rows { + // Advance FIRST: every path below this line may skip the row, and none of them + // may wedge the walk (scenario 7). + last = sha.clone(); + // Same cost gate as the skip-path repair: a canonical raw CIDv1 key is already + // the resolver key, so it reads no bytes and resolves no repo. + if gitlawb_core::cid::is_raw_cidv1(&stored) { + continue; + } + // Resolve the row's repo from its recorded provenance (first-pinner plus the + // bounded additional source set). An empty set is a pin recorded before + // provenance existed, which is the pre-provenance-est shape of row and exactly + // what this sweep is for, so it is not skipped: discovery below probes a + // bounded, quarantine-filtered set of warm local repos for the object and + // records what it finds ADDITIVELY. + let sources = match db.pin_sources_for_oid(&sha).await { + Ok(s) => s, + Err(e) => { + tracing::warn!(sha = %sha, err = %e, "sweep: failed to read pin sources"); + // A DB read error says nothing about the row, so a later run retries it. + retryable_skips += 1; + continue; + } + }; + // Whether this row ended the source walk repaired, and whether anything it hit + // along the way was a transient obstacle rather than a permanent one. + let mut row_repaired = false; + let mut row_retryable = false; + // Whether any source got as far as spending an object read on this row, which is + // what makes an unrepairable row COST something rather than just being skipped. + let mut row_read_attempted = false; + if sources.is_empty() { + let (outcome, reads) = discover_legacy_row( + &sha, + &mut discovery, + repos_dir, + git_bin, + git_timeout, + db, + traversal, + ) + .await; + match outcome { + DiscoveryOutcome::Repaired => { + repaired += 1; + row_repaired = true; } - let out = child.wait_with_output().expect("hash-object output"); - assert!( - out.status.success(), - "git hash-object: {}", - String::from_utf8_lossy(&out.stderr) - ); - String::from_utf8_lossy(&out.stdout).trim().to_string() - }) - .collect() + DiscoveryOutcome::Retryable => row_retryable = true, + DiscoveryOutcome::Settled => {} + // Worth walking again, like any retryable row, but it read nothing and so + // is charged nothing below (`reads` is zero). The flag is what keeps a + // starved pass from being silent. + DiscoveryOutcome::PassBudgetSpent => { + row_retryable = true; + discovery_budget_spent = true; + } + } + // Charge every probe that did not end in a repair, INCLUDING a retryable + // one, which is where discovery differs from the provenance loop below. + // There a retryable read is against a repo the row names as a holder, so it + // is expected to succeed once that repo warms. Discovery probes repos the + // row does not name, re-derives its candidate list from scratch on every + // run, and re-probes from the top, so a row that stays unrepaired costs the + // same reads again on the next boot whatever its outcome was. Leaving the + // retryable arm uncharged would also leave the cost open to steering, since + // the cap-reached arm is retryable by design and repo ids are grindable. + if !row_repaired { + dead_row_reads += reads; + } + } + for repo_id in sources { + let repo = match db.get_repo_by_id(&repo_id).await { + Ok(Some(r)) => r, + // The repo row is gone: a later source may still hold the bytes. A + // deleted repo does not come back, so this is not a retryable skip. + Ok(None) => continue, + Err(e) => { + tracing::warn!(repo_id = %repo_id, err = %e, "sweep: failed to read repo"); + row_retryable = true; + continue; + } + }; + // Derive the LOCAL disk path rather than going through `repo_store.acquire`. + // The sweep is opportunistic background maintenance over every pinned row on + // the node, so it must never pull a cold repo back from remote storage: that + // would turn a repair pass into a bulk restore. A repo that is not on local + // disk simply reads no bytes here and stays withheld, but it IS a retryable + // skip: on a Tigris-backed node the repo is cold now and warm later, and + // without the re-walk that row would never be repaired by anything. + // The path goes through the repo store's VALIDATED resolver (allowlisted + // components, rooted at repos_dir, no ParentDir/CurDir segment), not the raw + // join: the sweep is a second caller of that path logic and gets the same + // barrier the acquire path has (#173 round 11, F3). It is the non-fetching + // variant, so the no-cold-pull property above is untouched. + let repo_path = match crate::git::repo_store::validated_repo_disk_path( + repos_dir, + &repo.owner_did, + &repo.name, + ) { + Ok(p) => p, + // An unsafe name is not something a later run fixes, so it is terminal. + Err(e) => { + tracing::warn!(repo_id = %repo_id, err = %e, "sweep: rejected unsafe repo path"); + continue; + } + }; + if !repo_path.is_dir() { + row_retryable = true; + continue; + } + // The sweep holds no pin permit and has no batch to overrun, so the plain + // `git_timeout` is the right budget here. + row_read_attempted = true; + match repair_legacy_provider_cid( + &repo_path, + git_bin, + std::time::Instant::now() + git_timeout, + &sha, + db, + ) + .await + { + Ok(RepairOutcome::Repaired) => { + repaired += 1; + row_repaired = true; + break; + } + // The bytes could not be read from this source right now: try the next + // source, and if none of them works, walk the row again on a later run. + Ok(RepairOutcome::Retryable) => row_retryable = true, + // Nothing to repair from this source and nothing a re-walk changes. + Ok(RepairOutcome::Settled) => {} + Err(e) => { + tracing::warn!(sha = %sha, err = %e, "sweep: legacy provider-CID repair failed"); + row_retryable = true; + } + } + } + if !row_repaired && row_retryable { + retryable_skips += 1; + } + // Read, not repaired, and nothing a later run would change: pure waste, and the + // rewind means the next run repeats it. This is the quantity the run bounds. + if row_read_attempted && !row_repaired && !row_retryable { + dead_row_reads += 1; + } } - /// A live endpoint that answers every add with `500`. Counts the requests it - /// received so a test can tell "the loop kept going" from "the loop stopped", - /// which the returned pin list cannot (it is empty either way). Reads the - /// full request, headers plus the `Content-Length` body, before answering: - /// responding early and closing would surface as a write failure on the - /// client and turn a rejection into something else. - async fn rejecting_endpoint( - requests: std::sync::Arc, - ) -> String { - use tokio::io::{AsyncReadExt, AsyncWriteExt}; - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let endpoint = format!("http://{}", listener.local_addr().unwrap()); - tokio::spawn(async move { - while let Ok((mut sock, _)) = listener.accept().await { - requests.fetch_add(1, std::sync::atomic::Ordering::SeqCst); - tokio::spawn(async move { - let mut acc = Vec::new(); - let mut buf = [0u8; 4096]; - loop { - let n = match sock.read(&mut buf).await { - Ok(0) | Err(_) => break, - Ok(n) => n, - }; - acc.extend_from_slice(&buf[..n]); - // Once the headers are complete, keep reading until the - // declared body has arrived. - if let Some(hdr_end) = - acc.windows(4).position(|w| w == b"\r\n\r\n").map(|p| p + 4) - { - let headers = String::from_utf8_lossy(&acc[..hdr_end]).to_lowercase(); - let len: usize = headers - .lines() - .find_map(|l| l.strip_prefix("content-length:")) - .and_then(|v| v.trim().parse().ok()) - .unwrap_or(0); - if acc.len() >= hdr_end + len { - break; - } - } - } - let _ = sock - .write_all( - b"HTTP/1.1 500 Internal Server Error\r\nContent-Length: 0\r\n\r\n", - ) - .await; - let _ = sock.flush().await; - }); + db.set_pin_repair_cursor(&last).await?; + // A short batch is the end of the table, so this pass ended the TRAVERSAL: apply the + // window advance the traversal earned, then start a fresh accumulator for the next + // one. Persisting from HERE, not from the end of the run, is what survives the + // shutdown `select!` in `spawn_legacy_cid_sweep`: a drop mid-traversal loses only + // the accumulator, so the next traversal repeats a window rather than skipping one. + // + // The write is warn-only. A failed persist leaves the old continuation, and the next + // traversal probes the same window again, which is wasted work and never a gap. + if (scanned as i64) < batch { + if let Some((created_at_key, id)) = traversal.advance() { + if let Err(e) = db.set_discovery_continuation(&created_at_key, &id).await { + tracing::warn!(err = %e, "failed to persist the sweep discovery continuation"); } - }); + } + *traversal = DiscoveryTraversalState::default(); + } + Ok(SweepStats { + scanned, + repaired, + passes: 1, + retryable_skips, + dead_row_reads, + discovery_budget_spent, + stop: SweepStop::Completed, + }) +} + +/// Test seam for a single bounded pass (scenarios 4 and 5 drive passes by hand to +/// observe the batch bound and the restart-resumes-from-cursor behavior). +#[cfg_attr(not(test), allow(dead_code))] +pub(crate) async fn sweep_legacy_provider_cids_once( + repos_dir: &std::path::Path, + git_bin: &str, + git_timeout: Duration, + batch: i64, + db: &crate::db::Db, + traversal: &mut DiscoveryTraversalState, +) -> Result { + sweep_pass(repos_dir, git_bin, git_timeout, batch, db, traversal).await +} + +/// U4 (#173): the one-shot legacy provider-CID migration sweep. +/// +/// Releases before this branch stored the PROVIDER CID (Kubo dag-pb / Pinata CIDv0) +/// in `pinned_cids.cid`. This branch's `/ipfs/{cid}` resolver recomputes the raw +/// content CID and withholds any row whose key does not match, so those rows are +/// unresolvable. The opportunistic repair on the already-pinned skip path only fires +/// when a later push re-carries the object, and normal git negotiation omits objects +/// the node already has, so on an upgraded node that push generally never comes. This +/// walks the table instead. +/// +/// A row with NO recorded source is the pre-provenance case this exists for, so it is +/// not skipped: the pass probes a bounded, quarantine-filtered set of WARM local repos +/// for the object (at most [`MAX_LEGACY_DISCOVERY_PROBES`] reads per row, each row +/// taking a [`DISCOVERY_ROW_BUDGET_DIVISOR`] slice of the pass's one discovery deadline) +/// and, on a hit, rewrites the key from the verified bytes and +/// records the discovered repo ADDITIVELY alongside the incomplete marker. It never +/// writes an exclusive first-pinner claim and never pulls a cold repo back from remote +/// storage. See `discover_legacy_row` for why both of those matter. +/// +/// Runs until a pass comes back short of a full batch, which is the end of the table. +/// Sleeps `delay` between full batches so it cannot monopolize the DB, and persists +/// its cursor every pass so a restart continues instead of rewinding. Errors reading +/// or repairing an individual row are warn-and-skip; only a failure of the batch query +/// or the cursor write ends the run, and a later run picks up from the stored cursor. +/// +/// A run that REACHES THE END OF THE TABLE rewinds the cursor to the start on its way +/// out, so the next run walks the whole table again (#173 rounds 11 and 12). Without +/// that the cursor parked at the maximum `sha256_hex` for good and every later boot +/// read zero rows, which stranded two different kinds of row: one skipped for a +/// transient reason (its repo cold on a Tigris-backed node, a DB or object read error), +/// and one written BELOW the parked cursor afterwards by another node mid-rolling- +/// upgrade. Either way the row was unadvertised and unresolvable with nothing left to +/// fix it. +/// +/// Round 11 gated the rewind on a transient skip having happened. That could not cover +/// the second case, because the run that parks the cursor is a clean one by definition: +/// the row it strands does not exist yet. So the rewind is unconditional on completion. +/// +/// It is a per-RUN decision made after the walk has finished, never mid-walk, so it +/// cannot spin. The cost is one extra ordered scan per run, plus a repair attempt for +/// each row that is unrepairable in principle (bytes gone, provenance gone): the read is +/// attempted before the bytes are found missing. Those reads are the one cost that does +/// not shrink as the migration progresses, so `MAX_DEAD_ROW_READS_PER_RUN` bounds them +/// per run and the run stops early rather than paying `O(dead rows)` on every boot. A +/// row already carrying the canonical raw key costs a codec decode and no read at all, +/// so a node that has finished repairing pays the scan and nothing more. +/// +/// A run that stops on a pass ERROR does NOT rewind: its cursor is mid-table and +/// discarding it would restart the walk from the beginning on a node whose DB is +/// failing part-way through. +pub(crate) async fn sweep_legacy_provider_cids( + repos_dir: &std::path::Path, + git_bin: &str, + git_timeout: Duration, + batch: i64, + delay: Duration, + db: &crate::db::Db, + traversal: &mut DiscoveryTraversalState, +) -> SweepStats { + let mut totals = SweepStats::default(); + let mut completed = false; + loop { + let pass = match sweep_pass(repos_dir, git_bin, git_timeout, batch, db, traversal).await { + Ok(p) => p, + Err(e) => { + tracing::warn!(err = %e, "legacy provider-CID sweep pass failed; stopping"); + totals.stop = SweepStop::PassFailed; + break; + } + }; + totals.scanned += pass.scanned; + totals.repaired += pass.repaired; + totals.retryable_skips += pass.retryable_skips; + totals.dead_row_reads += pass.dead_row_reads; + totals.discovery_budget_spent |= pass.discovery_budget_spent; + totals.passes += 1; + // A short batch means the ordered walk reached the end of the table. Stop here + // rather than after an extra empty pass, and do NOT sleep on the way out. + if (pass.scanned as i64) < batch { + completed = true; + totals.stop = SweepStop::Completed; + break; + } + // Enough fruitless reads for one run. Stop WITHOUT completing, so the cursor + // stays where the walk got to and the next run carries on from there instead of + // re-reading these rows. Checked between passes, so a run can overshoot by at + // most one batch. + if totals.dead_row_reads >= MAX_DEAD_ROW_READS_PER_RUN { + tracing::info!( + dead_row_reads = totals.dead_row_reads, + "legacy provider-CID sweep pausing: too many unrepairable rows this run" + ); + totals.stop = SweepStop::PausedOnDeadReadCap; + break; + } + tokio::time::sleep(delay).await; + } + if totals.discovery_budget_spent { + tracing::info!( + passes = totals.passes, + "legacy provider-CID sweep: a pass spent its whole discovery budget before \ + reaching every source-less row; the rest were skipped unprobed" + ); + } + if completed { + if let Err(e) = db.set_pin_repair_cursor("").await { + tracing::warn!(err = %e, "failed to rewind the legacy provider-CID sweep cursor"); + } + } + totals +} + +/// How long the sweep waits between runs before walking the table again. +/// +/// Coverage of the discovery window is per TRAVERSAL, and a node with more warm repos +/// than one window needs several of them, so how long a source-less row waits for its +/// holder's window is set by how often traversals happen. Tying that to reboots would +/// make it a reboot count on a node that never reboots, which is the healthy node. +/// +/// Five minutes is chosen against what a run COSTS on a settled node, not against how +/// fast the migration should finish: a fully repaired table is one indexed range scan +/// per batch and a codec decode per row, no object reads at all, so the standing cost is +/// a few queries every five minutes and the migration still converges in hours rather +/// than never. It is also the anti-hot-spin floor for the degenerate case, an empty or +/// fully repaired table where a run returns immediately. +/// +/// That pricing holds for a table that settles. It does NOT hold for the table that +/// never does: a node carrying rows whose source bytes are permanently gone spends up to +/// [`MAX_DEAD_ROW_READS_PER_RUN`] (64) object reads on every run, repairs nothing, and +/// arrives back at exactly the same rows next time. At this interval alone that is 64 +/// fruitless `git cat-file` invocations every five minutes for the life of the process. +/// This constant is therefore the interval after a run that REPAIRED something; +/// [`SWEEP_IDLE_REARM_MULTIPLIER`] is what a run that repaired nothing backs off to (one +/// hour), and it is what keeps the unrepairable case from costing that forever. A failed +/// pass waits [`SWEEP_FAILURE_REARM_MULTIPLIER`] times this (30 minutes). +pub(crate) const SWEEP_REARM_DELAY: Duration = Duration::from_secs(300); + +/// How much longer the sweep waits after a run that repaired NOTHING, as a multiple of +/// the base interval. +/// +/// The base interval above is priced against a settled table, where a run is an indexed +/// range scan and a codec decode per row. It is not priced against the case that never +/// settles: a node carrying rows whose source bytes are permanently gone spends up to +/// [`MAX_DEAD_ROW_READS_PER_RUN`] object reads on every run, repairs nothing, and does +/// it again on the next one, forever. At the base interval that is 64 fruitless object +/// reads every five minutes, for the life of the process, against a table that will +/// never repair. +/// +/// So a run that repaired nothing backs off to the longer interval instead. Any run that +/// repairs at least one row resets to the base, because a table still yielding repairs +/// is one worth walking often. A single longer interval, not an exponential ladder: the +/// point is to stop paying a fixed waste every five minutes. +/// +/// Expressed as a multiple of the base rather than as an absolute so that shortening the +/// base (which the wrapper's tests do) shortens all three intervals coherently. +/// One hour is what it comes to in production. +const SWEEP_IDLE_REARM_MULTIPLIER: u32 = 12; + +/// How much longer the sweep waits after a pass QUERY failed, as a multiple of the base. +/// +/// A failing pass is a broken database, not a broken sweep, and retrying it on the base +/// interval would turn one fault into a stream of failing queries. But the alternative +/// the wrapper used to take, returning for good, is worse: one deadlock or connection +/// reset permanently disabled legacy-CID repair for the whole process lifetime, and +/// nothing joins the task, so the only trace was a single warn. Half an hour in +/// production is long enough not to hammer a database that is down, short enough that a +/// transient fault costs one window rather than a reboot. +const SWEEP_FAILURE_REARM_MULTIPLIER: u32 = 6; + +/// Consecutive failed runs before the per-failure log escalates from `warn!` to +/// `error!`. A single failure is a transient the next run recovers from; a standing +/// stream of them is a database that needs an operator, and at the production failure +/// interval this is reached in a couple of hours. +const SWEEP_FAILURE_ESCALATE_AFTER: u32 = 3; + +/// Run the legacy provider-CID sweep on a timer until shutdown. +/// +/// Owns the [`DiscoveryTraversalState`] across runs, which is the reason this is a +/// wrapper and not a loop inside `sweep_legacy_provider_cids`: a run can PAUSE +/// mid-traversal on [`MAX_DEAD_ROW_READS_PER_RUN`], and the traversal it was in is +/// finished by a later run, which has to apply the advance the earlier run earned. +/// +/// Sleeps after EVERY run, unconditionally, at the interval its outcome earns: +/// `rearm_delay` after a run that repaired something, that scaled by +/// [`SWEEP_IDLE_REARM_MULTIPLIER`] after one that repaired nothing, and by +/// [`SWEEP_FAILURE_REARM_MULTIPLIER`] after a failed pass query. Not conditional on the +/// run having done work: a run over an empty or fully repaired table returns +/// immediately, and without the sleep this loop would spin the database as fast as it +/// can answer. +/// +/// It NEVER returns, which is why it yields nothing: shutdown preempts it from the +/// outside, through the `tokio::select!` the caller wraps it in, so there is no awaited +/// value for a caller to log and the per-run summary is logged HERE. +pub(crate) async fn run_sweep_rearmed( + repos_dir: &std::path::Path, + git_bin: &str, + git_timeout: Duration, + batch: i64, + delay: Duration, + rearm_delay: Duration, + db: &crate::db::Db, +) { + let mut traversal = DiscoveryTraversalState::default(); + let mut consecutive_failures: u32 = 0; + loop { + let run = sweep_legacy_provider_cids( + repos_dir, + git_bin, + git_timeout, + batch, + delay, + db, + &mut traversal, + ) + .await; + if run.repaired > 0 { + tracing::info!( + scanned = run.scanned, + repaired = run.repaired, + passes = run.passes, + stop = ?run.stop, + "legacy provider-CID sweep run finished" + ); + } + #[cfg(test)] + note_sweep_run(); + + // A failed pass RE-ARMS, on its own longer interval, and never returns. Returning + // was the whole defect: the wrapper exists so coverage is wall-clock rather than + // a reboot count, and one deadlock or connection reset used to disable + // legacy-CID repair for the entire process lifetime. Nothing joins this task, so + // the only trace was a single warn and the node quietly kept withholding every + // unrepaired row. The longer interval is what keeps a genuinely broken database + // from being hammered, and the escalation is what keeps it from being quiet. + let next = if run.stop == SweepStop::PassFailed { + consecutive_failures = consecutive_failures.saturating_add(1); + if consecutive_failures > SWEEP_FAILURE_ESCALATE_AFTER { + tracing::error!( + consecutive_failures, + "legacy provider-CID sweep has failed every run for a while; the \ + database looks broken and legacy CID repair is not progressing" + ); + } else { + tracing::warn!( + consecutive_failures, + "legacy provider-CID sweep pass failed; re-arming on the longer \ + failure interval" + ); + } + rearm_delay.saturating_mul(SWEEP_FAILURE_REARM_MULTIPLIER) + } else { + consecutive_failures = 0; + if run.repaired == 0 { + // Nothing repaired: either the table is settled, or it holds rows that + // will never repair and this run just paid up to + // MAX_DEAD_ROW_READS_PER_RUN fruitless object reads to learn that + // again. Back off rather than pay it every base interval forever. Any + // run that does repair something resets to the base above. + rearm_delay.saturating_mul(SWEEP_IDLE_REARM_MULTIPLIER) + } else { + rearm_delay + } + }; + tokio::time::sleep(next).await; + } +} + +// Test-only wrapper-loop seam: how many RUNS the re-arm loop has completed. The loop +// never returns, so a test cannot observe its behaviour off a return value, and the +// interval it chose is only visible as "did another run happen inside this window". +// A process-wide counter rather than a `thread_local`, because the loop is awaited on a +// multi-thread runtime and can move between threads; the sweep tests that read it +// serialize on `sweep_run_lock` so they never see each other's increments. +#[cfg(test)] +static SWEEP_RUNS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + +#[cfg(test)] +fn note_sweep_run() { + SWEEP_RUNS.fetch_add(1, std::sync::atomic::Ordering::Relaxed); +} + +#[cfg(test)] +pub(crate) fn reset_sweep_runs() { + SWEEP_RUNS.store(0, std::sync::atomic::Ordering::Relaxed); +} + +#[cfg(test)] +pub(crate) fn sweep_runs() -> usize { + SWEEP_RUNS.load(std::sync::atomic::Ordering::Relaxed) +} + +/// Serializes the tests that read [`sweep_runs`], since the counter is process-wide. +#[cfg(test)] +pub(crate) fn sweep_run_lock() -> &'static tokio::sync::Mutex<()> { + static LOCK: std::sync::OnceLock> = std::sync::OnceLock::new(); + LOCK.get_or_init(|| tokio::sync::Mutex::new(())) +} + +// Test-only cost-gate counter (R8, U7): how many times the opportunistic repair +// read an object's bytes on the skip path. The codec gate must spare a CIDv1/raw +// row this read; the counter is the both-ways guard (removing the gate reads the +// raw row and increments it). Same thread_local discipline as the serve-path +// oversize counter — the pin tests await `pin_new_objects` on a current-thread +// runtime, so the increment and the assertion share one thread. +#[cfg(test)] +thread_local! { + static LEGACY_REPAIR_READS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +#[cfg(test)] +pub(crate) fn reset_legacy_repair_reads() { + LEGACY_REPAIR_READS.with(|c| c.set(0)); +} + +#[cfg(test)] +pub(crate) fn legacy_repair_reads() -> usize { + LEGACY_REPAIR_READS.with(|c| c.get()) +} + +#[cfg(test)] +fn note_legacy_repair_read() { + LEGACY_REPAIR_READS.with(|c| c.set(c.get() + 1)); +} + +/// Wall-clock ceiling on one [`pin_new_objects`] batch. +/// +/// The loop runs under a `pin_semaphore` permit and that pool defers rather than +/// sheds, so without a ceiling the hold is O(N) with N (the push's object count) +/// chosen by the pusher. This bounds the drain of a saturated pool instead. +/// +/// 120s is 12x the shared client's 10s whole-request ceiling, so a single large +/// healthy upload that needs more than the client default still has room to +/// finish (the per-request timeout is set to the remainder, not the default), +/// while a batch of them still cannot hold the permit indefinitely. Deliberately +/// a constant and not a config knob: the value only has to be large enough to be +/// uninteresting on a healthy node, and a knob is operator surface that would +/// have to be documented, validated, and kept meaningful. +pub const PIN_BATCH_BUDGET: Duration = Duration::from_secs(120); + +/// The smallest remainder worth starting a bounded git read (or an add) with. +/// +/// A 1ms remainder otherwise buys a child spawned already past its deadline, which +/// can only be reaped: the watchdog's SIGTERM grace plus its post-SIGKILL settle are +/// paid in full for work that produces nothing, once per remaining object. Breaking +/// the batch instead is the same spawn-to-reap amplification the bounded type probe +/// already refuses when it declines a confirming re-probe it cannot afford. +/// +/// ~1100ms tracks `visibility_pack`'s 1s SIGTERM grace plus its 20ms settle plus +/// margin. Both of those are private to that module, so the value is named once here +/// and documented rather than guessed separately in each loop. +pub(crate) const PIN_READ_FLOOR: Duration = Duration::from_millis(1100); + +/// The shared outbound client for both IPFS sinks. +/// +/// `pin_new_objects` runs while holding a `pin_semaphore` permit and that pool +/// defers rather than sheds, so an unbounded await here parks the pool. A bare +/// `reqwest::Client::new()` has no timeout, which is exactly that. Built from +/// `crate::build_http_client` rather than a local builder: its docstring forbids +/// hand-rolling an equivalent, so that the redirect and timeout guarantees the +/// node's tests bind stay bound to the client every outbound path actually uses. +fn http_client() -> &'static reqwest::Client { + static CLIENT: std::sync::OnceLock = std::sync::OnceLock::new(); + CLIENT + .get_or_init(|| crate::build_http_client().expect("failed to build production http client")) +} + +/// Pin a single git object to the local IPFS/Kubo node. +/// +/// - `ipfs_api`: base URL of the Kubo HTTP API, e.g. `http://127.0.0.1:5001`. +/// If empty the function returns `Ok("")` immediately. +/// - `sha256_hex`: the git SHA-256 hex object ID (used only for logging). +/// - `data`: raw git object content bytes (same bytes used for CID computation). +/// - `request_timeout`: overrides the shared client's whole-request timeout for +/// THIS request only. `RequestBuilder::timeout` replaces the client-level value +/// per request and leaks nothing to other calls on the same client, so the +/// batch loop can hand each add whatever is left of its budget without +/// loosening or tightening any other outbound path. `None` keeps the client's +/// own ceiling. +/// +/// Returns the CID string on success, or `""` when IPFS is not configured. +pub async fn pin_git_object( + ipfs_api: &str, + sha256_hex: &str, + data: &[u8], + request_timeout: Option, +) -> Result { + if ipfs_api.is_empty() { + return Ok(String::new()); + } + + // Compute the expected CIDv1 from the content bytes + let expected_cid = Cid::from_git_object_bytes(data).to_string(); + + let url = format!( + "{}/api/v0/add?cid-version=1&raw-leaves=true&pin=true", + ipfs_api.trim_end_matches('/') + ); + + // Build multipart form with the object data + let part = reqwest::multipart::Part::bytes(data.to_vec()) + .file_name("object") + .mime_str("application/octet-stream")?; + let form = reqwest::multipart::Form::new().part("file", part); + + let mut req = http_client().post(&url).multipart(form); + if let Some(t) = request_timeout { + req = req.timeout(t); + } + + let resp = req + .send() + .await + // Keep the `reqwest::Error` as this error's source rather than + // formatting it away. Operators reading a pin failure want the concrete + // transport cause in the logged chain, not a single flattened line, and + // this module's tests downcast to it to prove a silent endpoint really + // surfaces as a timeout rather than as some other failure that happens + // to arrive in time. + // The context keeps the old message verbatim so the callers that log + // this at `%e` (here, `sync.rs`, `encrypted_pin.rs`) read the same. + .map_err(|e| { + let msg = format!("IPFS add request failed: {e}"); + anyhow::Error::new(e).context(msg) + })?; + + if !resp.status().is_success() { + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + return Err(anyhow::anyhow!( + "IPFS /api/v0/add returned {status}: {body}" + )); + } + + // Kubo returns newline-delimited JSON; we only care about the last object + // (there's typically just one for a single-file add). + let body = resp + .text() + .await + .map_err(|e| anyhow::anyhow!("IPFS add response body read failed: {e}"))?; + let cid = body + .lines() + .filter(|l| !l.trim().is_empty()) + .filter_map(|line| { + let v: serde_json::Value = serde_json::from_str(line).ok()?; + v["Hash"].as_str().map(|s| s.to_string()) + }) + .next_back() + .unwrap_or(expected_cid.clone()); + + tracing::debug!(sha256 = %sha256_hex, %cid, "pinned git object to IPFS"); + Ok(cid) +} + +/// Fetch raw bytes for a CID from the local Kubo node (`/api/v0/cat`). +pub async fn cat(ipfs_api: &str, cid: &str) -> Result> { + if ipfs_api.is_empty() { + return Err(anyhow::anyhow!("IPFS not configured")); + } + let url = format!("{}/api/v0/cat?arg={}", ipfs_api.trim_end_matches('/'), cid); + let resp = http_client().post(&url).send().await?; + if !resp.status().is_success() { + return Err(anyhow::anyhow!("ipfs cat {cid}: {}", resp.status())); + } + Ok(resp.bytes().await?.to_vec()) +} + +/// The batch's remaining wall-clock, or `None` once too little is left to be worth +/// starting an object's work with, after logging the truncation exactly once. +/// +/// "Too little" is [`PIN_READ_FLOOR`], not zero: a remainder that cannot cover a +/// bounded git read's teardown buys a child spawned already past its deadline, and a +/// sub-millisecond per-request timeout buys a doomed add whose failure reads as an +/// endpoint fault rather than as budget truncation. +/// +/// Shaped like `api::ipfs`'s `budget_gate` on purpose: the nonzero-ness rides in +/// the returned value, so every call site must consume it as +/// `let Some(x) = ... else { break }` and a zero `Duration` can never reach a +/// request as its timeout. +/// +/// `sink` labels the backend in the truncation warn ("IPFS" or "Pinata"). The gate +/// is shared by both pin loops rather than copied per loop, so the two cannot drift +/// apart in how they report a truncated batch. +pub(crate) fn batch_budget_gate( + sink: &str, + deadline: Instant, + pinned: usize, + unattempted: usize, +) -> Option { + let left = deadline.saturating_duration_since(Instant::now()); + if left < PIN_READ_FLOOR { + tracing::warn!( + sink, + pinned, + unattempted, + "pin batch deadline reached; the remaining objects are left unpinned" + ); + return None; + } + Some(left) +} + +/// Pin any of the given candidate git objects that are not yet recorded in +/// `pinned_cids`. +/// +/// `object_list` is the already-withheld-filtered OID set to pin: the caller +/// applies `visibility_pack::replicable_objects` on the delta path or the +/// `..._fail_closed` filter on the full-scan path before calling, so this +/// function never sees a withheld blob. `repo_path` is still needed to read each +/// object's bytes, and `git_bin` names the binary those reads run: the production +/// callers pass the literal `"git"`, and a test passes a fake so the loop's own bound +/// can be driven with a git that never answers. `repo_id` records the pin's provenance +/// so `GET /ipfs/{cid}` resolves straight to this repo instead of scanning every repo +/// (#173). +/// +/// # What `batch_budget` does and does not bound +/// +/// The loop holds a `pin_semaphore` permit and that pool defers rather than +/// sheds, so the hold has to be bounded by something other than the pusher's +/// object count. Four things here are: +/// +/// - this loop's own wall-clock: the deadline is taken once at loop start and +/// checked at the top of every iteration, so no object's work begins with less +/// than [`PIN_READ_FLOOR`] left. It is a gate, not a hard ceiling, since a started +/// iteration still runs to completion; +/// - the git read: `store::read_object_bounded` runs under `spawn_blocking` against the +/// ABSOLUTE batch deadline (not the loop-top remainder, which the `is_pinned` round-trip +/// sitting between the two would push past it), with SIGTERM-then-SIGKILL +/// process-group teardown, so a hung `git cat-file` costs this batch its remaining +/// budget plus one watchdog teardown instead of holding the permit for the child's +/// whole lifetime and blocking a runtime worker while it does; +/// - each HTTP add: `pin_git_object` is handed the remainder measured AFTER the read +/// as its per-request timeout, which is what lets one large healthy upload run past +/// the shared client's 10s default without letting the batch run forever. Measuring +/// it after the read is what keeps the read-plus-add pair inside one budget rather +/// than up to two of them; +/// - the DB round-trips: every DB operation reachable from inside the region is +/// bounded by the same absolute deadline through [`db_bounded`], including the two +/// inside `repair_legacy_provider_cid`, which the loop body's own call sites do not +/// show. `retry_db_record` is wrapped as a whole so its ladder cannot multiply one +/// remainder, and the durability writes (the post-add record, the skip branch's +/// source record and its incomplete marker) take the floored remainder +/// `max(remaining, DB_RECORD_GRACE)` so a spent budget delays the permit release +/// rather than dropping a write. A bound is not a rollback, and what an elapsed +/// bound MEANS is a property of the operation, so each site maps that arm from the +/// shape of the call it wrapped: a multi-statement transaction whose `tx.commit()` +/// is never reached definitely did not land and is compensated like a definite +/// error, while a single autocommit statement may still land server-side and is +/// never treated as a failed write. See [`BoundedDbError::Elapsed`]. +/// +/// So the LOOP's hold is bounded by roughly `batch_budget` plus one teardown plus the +/// record graces one iteration can chain. `db_record_deadline` re-floors from +/// `Instant::now()` at EVERY call, so the graces inside a single iteration add up +/// rather than sharing one floor: this loop's worst case is the skip branch at +/// `deadline + 4s` (the source record, then its incomplete marker). It does NOT stack +/// per object, because the next iteration's first statement is `batch_budget_gate`, +/// which breaks the batch, so the overrun is one iteration's worth however many +/// objects the push carried. Against the 120s `PIN_BATCH_BUDGET` that is roughly a 5% +/// overrun for the batch, not an unbounded hold. One thing inside that region still is +/// not bounded at all, and the gate cannot fix it: +/// +/// - the pool. `api::repos` acquires the same `pin_semaphore` for the Pinata +/// replication task and holds it across `pinata_object_list_for_refs`, a full git +/// re-derivation that runs BEFORE `pinata::pin_new_objects` is entered and whose +/// per-child timeouts carry no aggregate deadline. What is bounded is each pin +/// loop's own hold, not the permit's total hold and not the semaphore's worst-case +/// queue. +/// +/// # Truncation semantics +/// +/// A batch stopped at the deadline leaves its remaining objects unpinned, and +/// nothing sweeps them up afterwards. There is no reconciliation pass over +/// `pinned_cids`; recovery is opportunistic, happening only if some later push +/// on the repo takes the full-scan fallback (`push_delta::list_all_objects`) and +/// re-derives the whole object set, which then re-offers the skipped OIDs. +/// +/// The twin in `pinata.rs` is back at parity on everything that bounds or repairs an +/// object: it runs the same shared budget gate at the top of every iteration, the same +/// bounded and reaped git read against the earlier of the batch deadline and +/// `git_timeout`, and the same opportunistic legacy provider-CID repair on its skip +/// branch. It still has no per-request override, since `pinata::pin_object` takes no +/// timeout argument and its uploads are bounded by the shared client's own ceiling. +/// Everything else about the shape (the skip-if-pinned check, the provenance recording, +/// the fault arms) changes in lockstep. The returned pairs are the one deliberate +/// exception: this side omits an object whose DB record exhausted its retries, because +/// the return here is consumed for logging only, while the pinata side still returns it +/// because its return feeds the announcement `cid_map`. See the record step for the +/// reasoning. +/// +/// Returns a list of `(sha256_hex, cid)` pairs pinned AND durably recorded this +/// call. +// Eight because #173's git seam (`git_bin`, `git_timeout`) and pin provenance +// (`repo_id`) sit alongside #174's batch budget. All four callers pass every one, and +// grouping them into a context struct would add a type whose only job is to be +// destructured back into these fields at the top of the loop. +#[allow(clippy::too_many_arguments)] +pub async fn pin_new_objects( + ipfs_api: &str, + repo_path: &std::path::Path, + git_bin: &str, + git_timeout: Duration, + object_list: Vec, + db: &crate::db::Db, + repo_id: &str, + batch_budget: Duration, +) -> Vec<(String, String)> { + if ipfs_api.is_empty() { + return vec![]; + } + + let deadline = Instant::now() + batch_budget; + let total = object_list.len(); + let mut pinned = Vec::new(); + + for (attempted, sha) in object_list.into_iter().enumerate() { + // Top of the iteration, before any of this object's work: an object is + // never started with a remainder too small to cover a bounded read's + // teardown. Consumed as a guard only: the read below runs against the + // absolute batch deadline, and the add's timeout is measured again after + // the read, so this remainder has no other consumer here. + if batch_budget_gate("IPFS", deadline, pinned.len(), total - attempted).is_none() { + break; + } + // Skip if already pinned, but first backfill provenance if the existing + // pin has none. A legacy pin (recorded before repo_id existed, #173, jatmn) + // is skipped here before record_pinned_cid ever runs, so its NULL provenance + // would never resolve to one repo and known CIDs keep hitting the scan. The + // backfill only sets repo_id (AND repo_id IS NULL guard preserves + // first-pinner-owns) and never re-pins the bytes: the object is already on IPFS. + // Every DB call from here to the end of the iteration is bounded by the + // ABSOLUTE batch deadline (F3, #173): the loop runs under a global pin permit + // and a bare await parked it for the whole stall. The elapsed arm is mapped per + // site below, never as a blanket "existing error arm": a timeout cancels the + // client future but not the statement Postgres is running, so it reports an + // UNKNOWN outcome, not a failed write. + match db_bounded(deadline, db.is_pinned(&sha)).await { + Ok(true) => { + // Elapsed here is free to skip: these are reads, so a late server-side + // completion costs nothing, and the backfill's own `AND repo_id IS NULL` + // guard makes a late-landing write idempotent. + match db_bounded(deadline, db.provenance_for_oid(&sha)).await { + Ok(None) => { + if let Err(e) = + db_bounded(deadline, db.backfill_pin_provenance(&sha, repo_id)).await + { + tracing::warn!(sha = %sha, err = %e, "failed to backfill pin provenance"); + } + } + Ok(Some(_)) => {} + Err(e) => { + tracing::warn!(sha = %sha, err = %e, "DB error reading pin provenance"); + } + } + // F1 (#173 round 8): record this repo as an ADDITIONAL source for the + // already-pinned object. This is the load-bearing skip-branch insert — + // a later repo pushing a shared object hits this path (already pinned), + // and without it `GET /ipfs/{cid}` only ever knows the first pinner, so a + // shared object first pinned from a private/quarantined repo 404s even + // when this repo would serve it. Bounded per object (MAX_PIN_SOURCES). + // The retry ladder is wrapped AS A WHOLE, not per attempt: three stalls + // plus their backoff otherwise multiply one remainder by three. Floored + // at DB_RECORD_GRACE because this is a durability write. + match db_bounded( + db_record_deadline(deadline), + retry_db_record(|| db.record_pin_source(&sha, repo_id)), + ) + .await + { + Ok(()) => {} + // Elapsed here is a DEFINITE non-write, not an unknown outcome, and + // that follows from what was wrapped rather than from the timeout. + // `record_pin_source` is an explicit transaction (`pool.begin()`, + // the insert, a conditional marker clear, `tx.commit()`), so a + // cancelled future never reaches the commit, no COMMIT is ever sent, + // and the row cannot have landed. The source set is therefore + // incomplete and must be marked, exactly as on the definite-error + // arm below; leaving it unmarked is the state the marker exists to + // prevent, since the resolver reads a non-empty below-cap set as + // COMPLETE and 404s a copy this repo would serve. The cost of the + // marker is bounded: the fallback legacy scan is capped at + // `ipfs_max_legacy_probes` and charges the per-IP work rate limiter + // per probe. The arm stays separate only so the warn tells an + // operator a stalled batch from a scattered per-object failure. + Err(e @ BoundedDbError::Elapsed) => { + tracing::warn!( + sha = %sha, + err = %e, + "pin source record did not complete inside the batch deadline; \ + a cancelled multi-statement transaction never commits, so the \ + source is definitely missing and the set is marked incomplete" + ); + if let Err(e) = db_bounded( + db_record_deadline(deadline), + db.mark_pin_sources_incomplete(&sha, repo_id), + ) + .await + { + tracing::warn!(sha = %sha, err = %e, "failed to mark pin sources incomplete"); + } + } + // U3 (#173): the retries are spent on REAL errors, so this repo is + // definitely NOT in the source set and the set is known incomplete. + // Persist that, or the resolver reads a non-empty below-cap set as + // COMPLETE and 404s an object this repo would serve. Warn-only in + // turn: if the marker write also fails the object degrades to the + // pre-U3 behavior, never worse. Floored for the same reason the + // record above is: a spent budget must not drop the compensation. + // The marker write itself is a single autocommit statement, so ITS + // own elapsed arm genuinely is an unknown outcome; nothing branches + // on it, which is why warn-only is the right handling there. + Err(e) => { + tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); + if let Err(e) = db_bounded( + db_record_deadline(deadline), + db.mark_pin_sources_incomplete(&sha, repo_id), + ) + .await + { + tracing::warn!(sha = %sha, err = %e, "failed to mark pin sources incomplete"); + } + } + } + // R8 (#173 round 10): opportunistically repair a legacy provider-CID + // row (Kubo dag-pb / Pinata) to the raw-content resolver key on this + // re-push. Cost-gated on the stored key's codec — a non-legacy row + // reads no bytes. Warn-only: a failure leaves the row as-is for a + // later re-push or the deferred one-shot sweep. + // Clamped to the batch deadline: this runs with the pin permit held, so + // an unclamped `git_timeout` would let one wedged read hold a global pin + // slot for 600s against a 120s budget. + if let Err(e) = repair_legacy_provider_cid( + repo_path, + git_bin, + std::cmp::min(deadline, std::time::Instant::now() + git_timeout), + &sha, + db, + ) + .await + { + tracing::warn!(sha = %sha, err = %e, "failed to repair legacy provider CID"); + } + continue; + } + Ok(false) => {} + Err(e) => { + tracing::warn!(sha = %sha, err = %e, "DB error checking pinned status"); + continue; + } + } + + // Read raw object content, bounded and reaped, under `spawn_blocking`: this is + // synchronous blocking work (child spawn, pipe drain, watchdog join), so + // calling it from the runtime task would block a worker thread for its whole + // duration. Placement mirrors the `/ipfs` serve path; the admission guard is + // deliberately NOT moved into the closure there, since the pin permit is not + // held by a future a client disconnect can drop and the child is reaped on its + // own deadline regardless. + // + // The read runs against the ABSOLUTE batch deadline, not against the remainder + // measured at the top of the iteration: the `is_pinned` round-trip above sits + // between the two, so `Instant::now() + budget_left` would land past `deadline` + // by however long the DB took, and under a saturated pool that is the dominant + // term. A slow DB check must not push the read's own bound out. + // + // Bounded by the EARLIER of the batch deadline (#174) and this object's own + // `git_timeout` (#173). Both bounds are load-bearing and neither implies the + // other: the batch deadline alone would let ONE wedged `cat-file` hold the pin + // permit for the whole 120s budget (the failure #173's reaper test drives), while + // `git_timeout` alone would let a batch of merely-slow reads run past the budget. + // Which arm actually binds depends on configuration, and at SHIPPED DEFAULTS it is + // always the batch deadline: `git_service_timeout_secs` is 600 against a 120s + // PIN_BATCH_BUDGET. The `git_timeout` arm is what an operator who tightens that + // knob below the remaining budget gets, so do not read this as two bounds both + // firing in a default deployment. + let read_deadline = std::cmp::min(deadline, std::time::Instant::now() + git_timeout); + let read_path = repo_path.to_path_buf(); + let read_sha = sha.clone(); + let read_git = git_bin.to_string(); + let read = tokio::task::spawn_blocking(move || { + crate::git::store::read_object_bounded(&read_git, &read_path, &read_sha, read_deadline) + }) + .await; + let data = match read { + Ok(Ok(Some((_obj_type, bytes)))) => bytes, + // A verified absence, and the only outcome that is not a fault. + Ok(Ok(None)) => continue, + // A Transient fault does NOT by itself mean the store is gone. It also + // covers a spawn or watchdog-timeout failure of the reaped child, an + // unaffordable confirming re-probe, and, because readability is judged FOR + // one oid, a single unreadable `objects/` fan-out, which is 1/256 of the + // store. So re-check store-wide before deciding what the fault costs. + Ok(Err(e @ crate::git::store::ProbeError::Transient(_))) => { + if !crate::git::store::object_store_readable_store_wide(repo_path) { + // Genuinely store-wide: every remaining object fails identically, and + // continuing would spawn one doomed bounded child per object and spend + // the batch budget reaping them. + tracing::warn!( + sha = %sha, + err = %e, + unattempted = total - attempted, + "object store unreadable while pinning; stopping the batch" + ); + break; + } + // The store still reads store-wide, so the fault is object-scoped or + // transient to this read. Breaking would forfeit a healthy store's + // remaining objects permanently: the documented recovery re-derives the + // same list and breaks at the same index. + tracing::warn!( + sha = %sha, + err = %e, + "transient fault reading git object for pinning; the object store is \ + still readable store-wide, so this costs only this object" + ); + continue; + } + // The store is readable and git still failed: a corrupt object, or a + // repo-wide fault git reports immediately. Either way it is per-object + // work that stays inside the budget, and breaking would forfeit a healthy + // store's remaining objects over one bad one, permanently (a later + // full-scan push re-offers the same object and breaks in the same place). + Ok(Err(e)) => { + tracing::warn!(sha = %sha, err = %e, "failed to read git object for pinning"); + continue; + } + // A panic in the read closure leaves no evidence that the failure is + // object-scoped, so fail toward the conservative arm. + Err(e) => { + tracing::warn!(sha = %sha, err = %e, "bounded git read task failed; stopping the batch"); + break; + } + }; + + // Recompute the remainder AFTER the read rather than reusing `budget_left`: + // the read is now allowed to spend the whole remainder, so handing the add the + // loop-top value would make the pair a hold of up to 2x the batch budget. The + // same gate, so a remainder too small to be worth a request truncates the batch + // with one warn instead of shedding a doomed add that logs as an endpoint fault. + let Some(add_timeout) = + batch_budget_gate("IPFS", deadline, pinned.len(), total - attempted) + else { + break; + }; + + // Pin to IPFS + match pin_git_object(ipfs_api, &sha, &data, Some(add_timeout)).await { + Ok(cid) if !cid.is_empty() => { + // The resolver key (`pinned_cids.cid`) must be the locally-computed + // raw-content CID, never the provider Hash: Kubo returns a dag-pb/UnixFS + // root for objects above its block size, which does not hash the raw + // content, so `GET /ipfs/{provider_cid}` would resolve then fail the F2 + // integrity check (list-then-404). The serve path reads bytes from git and + // verifies them against the requested CID, so the raw CID is the correct + // key. Mirrors the pinata twin, which already records the raw CID. + let raw_cid = gitlawb_core::cid::Cid::from_git_object_bytes(&data).to_string(); + // F1 (#173 round 8): the first pinner is recorded in pin_repo_sources too, + // so every source (first and subsequent) is tried uniformly by the + // resolver. U3 (#173): the pin and its source go down in ONE transaction. + // As two independent best-effort calls this path could land the pin while + // dropping its own source, producing a source set silently missing its + // first pinner; atomically there is no such window. When the transaction + // still fails after every retry, Kubo is holding the bytes but the DB has + // no row, so nothing can resolve that CID and there is no partial state to + // clean up. Recovery is the next push, which re-offers the object and + // retries the whole record; until then the object counts as unpinned, and + // the returned vector says so by carrying only durably recorded pins. + // + // Returning the provider Hash rather than the resolver key is deliberate: + // the DB `cid` is the raw resolver key (recorded above), the returned value + // is the provider CID. On the record-failed case the twins DIVERGE and must + // stay that way. This return is log-only (`api/repos.rs` turns it into a + // count log plus one line per pair and consumes it nowhere else), so + // dropping a record-failed pin costs nothing and stops the log claiming a + // pin the resolver cannot serve. The pinata twin keeps its unconditional + // push because ITS return is a real input: `api/repos.rs` builds the + // sha-to-cid `cid_map` from it, which drives `upsert_branch_cid` and the + // p2p `publish_ref_update` gossip CID. Do not re-align them without moving + // that consumer first. + // + // The bound here is FLOORED at DB_RECORD_GRACE. The add was handed the + // whole remainder, so a successful one can return with ~0 left, and an + // unfloored bound would fail a write that today completes in + // milliseconds, leaving the bytes in Kubo with no row to resolve them + // by. If it still fires, both arms mean the same thing here and the site + // takes one Err path for them: `record_pinned_cid_with_source` is an + // explicit transaction, so a cancelled future never reaches its + // `tx.commit()` and the rows definitely did not land, exactly as on a + // real error. Either way the pin is not returned and the next push + // re-offers the object. The warn still names the arm through the error's + // own Display, so an operator can tell a stalled batch from a scattered + // per-object failure. + match db_bounded( + db_record_deadline(deadline), + retry_db_record(|| db.record_pinned_cid_with_source(&sha, &raw_cid, repo_id)), + ) + .await + { + Ok(()) => pinned.push((sha, cid)), + Err(e) => { + tracing::warn!(sha = %sha, err = %e, "failed to record pinned CID in DB"); + } + } + } + Ok(_) => {} + Err(e) => { + tracing::warn!(sha = %sha, err = %e, "failed to pin git object to IPFS"); + } + } + } + + pinned +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::Cell; + + // The retry helper is the load-bearing unit: it converts a sub-second + // transient DB error at the three warn-only record sites into a landed row, + // instead of a permanently incomplete pin-source set. These drive the helper + // directly against a controlled closure (the record sites take a concrete + // `&Db` over a `PgPool`, so a failing-first wrapper cannot slot in without + // changing signatures — see U6 seam note). + + /// The re-arm intervals are expressed as multiples of the base so a test can shrink + /// all three coherently. This pins what they come to in production, which is the + /// number the constants' docs quote. + #[test] + fn the_rearm_multipliers_give_the_documented_production_intervals() { + assert_eq!( + SWEEP_REARM_DELAY.saturating_mul(SWEEP_IDLE_REARM_MULTIPLIER), + Duration::from_secs(3600), + "a run that repairs nothing waits an hour" + ); + assert_eq!( + SWEEP_REARM_DELAY.saturating_mul(SWEEP_FAILURE_REARM_MULTIPLIER), + Duration::from_secs(1800), + "a failed pass waits half an hour" + ); + } + + #[tokio::test] + async fn retry_lands_after_transient_failures() { + let calls = Cell::new(0u32); + let result = retry_db_record(|| { + let n = calls.get() + 1; + calls.set(n); + async move { + if n < PIN_RECORD_ATTEMPTS { + Err(anyhow::anyhow!("transient failure on attempt {n}")) + } else { + Ok(()) + } + } + }) + .await; + + assert!( + result.is_ok(), + "retry lands the row after transient failures" + ); + assert_eq!( + calls.get(), + PIN_RECORD_ATTEMPTS, + "op is retried until it succeeds" + ); + } + + #[tokio::test] + async fn retry_returns_last_err_after_exhaustion() { + let calls = Cell::new(0u32); + let result = retry_db_record(|| { + let n = calls.get() + 1; + calls.set(n); + async move { Err::<(), _>(anyhow::anyhow!("attempt {n} failed")) } + }) + .await; + + let err = result.expect_err("all attempts fail so the last error surfaces"); + assert_eq!( + calls.get(), + PIN_RECORD_ATTEMPTS, + "attempts are bounded to the cap" + ); + assert_eq!( + err.to_string(), + "attempt 3 failed", + "the LAST error is returned, not the first" + ); + } + + // Happy path against a real DB: a single-attempt success lands the row, and a + // redundant call is idempotent (`ON CONFLICT DO NOTHING`), so the source set + // holds exactly one row. + #[sqlx::test] + async fn retry_records_pin_source_once(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.unwrap(); + + let sha = "a".repeat(64); + let repo_id = "repo-retry-1"; + + retry_db_record(|| db.record_pin_source(&sha, repo_id)) + .await + .expect("happy-path record succeeds in one attempt"); + retry_db_record(|| db.record_pin_source(&sha, repo_id)) + .await + .expect("a redundant record is idempotent"); + + let sources = db.pin_sources_for_oid(&sha).await.unwrap(); + assert_eq!( + sources, + vec![repo_id.to_string()], + "exactly one source row lands under ON CONFLICT DO NOTHING" + ); + } + + use std::time::Duration; + + /// Write `n` loose blobs into a fresh bare repo and return their oids. + /// `read_object` shells to `git cat-file`, so the objects must genuinely + /// exist on disk — a fabricated oid would `continue` past the pin call and + /// the loop scenario below would prove nothing. + fn seed_loose_blobs(repo_path: &std::path::Path, n: usize) -> Vec { + crate::git::store::init_bare(repo_path).expect("init bare repo"); + (0..n) + .map(|i| { + let mut cmd = std::process::Command::new("git"); + cmd.args(["hash-object", "-w", "--stdin"]) + .current_dir(repo_path) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()); + let mut child = cmd.spawn().expect("spawn git hash-object"); + { + use std::io::Write; + child + .stdin + .as_mut() + .expect("stdin") + .write_all(format!("pin loop object {i}\n").as_bytes()) + .expect("write stdin"); + } + let out = child.wait_with_output().expect("hash-object output"); + assert!( + out.status.success(), + "git hash-object: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }) + .collect() + } + + /// A live endpoint that answers every add with `500`. Counts the requests it + /// received so a test can tell "the loop kept going" from "the loop stopped", + /// which the returned pin list cannot (it is empty either way). Reads the + /// full request, headers plus the `Content-Length` body, before answering: + /// responding early and closing would surface as a write failure on the + /// client and turn a rejection into something else. + async fn rejecting_endpoint( + requests: std::sync::Arc, + ) -> String { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { + while let Ok((mut sock, _)) = listener.accept().await { + requests.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + tokio::spawn(async move { + let mut acc = Vec::new(); + let mut buf = [0u8; 4096]; + loop { + let n = match sock.read(&mut buf).await { + Ok(0) | Err(_) => break, + Ok(n) => n, + }; + acc.extend_from_slice(&buf[..n]); + // Once the headers are complete, keep reading until the + // declared body has arrived. + if let Some(hdr_end) = + acc.windows(4).position(|w| w == b"\r\n\r\n").map(|p| p + 4) + { + let headers = String::from_utf8_lossy(&acc[..hdr_end]).to_lowercase(); + let len: usize = headers + .lines() + .find_map(|l| l.strip_prefix("content-length:")) + .and_then(|v| v.trim().parse().ok()) + .unwrap_or(0); + if acc.len() >= hdr_end + len { + break; + } + } + } + let _ = sock + .write_all( + b"HTTP/1.1 500 Internal Server Error\r\nContent-Length: 0\r\n\r\n", + ) + .await; + let _ = sock.flush().await; + }); + } + }); + endpoint + } + + /// A sleeping-but-live endpoint. Answers `200` with an empty body after + /// `delays[i]` for the i-th request it accepts (the last entry repeats), so + /// a test can make one add slow and the next fast. Drains the full request, + /// headers plus the declared `Content-Length` body, before sleeping: exactly + /// as in `rejecting_endpoint`, answering early and closing would surface as + /// a write failure on the client and turn a slow-but-healthy add into a + /// different failure shape. + /// + /// An empty body is a successful pin: `pin_git_object` falls back to the CID + /// it computed from the bytes when the response carries no `Hash`. + async fn delaying_endpoint(delays: Vec) -> String { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { + let mut seen = 0usize; + while let Ok((mut sock, _)) = listener.accept().await { + let delay = *delays + .get(seen) + .or_else(|| delays.last()) + .unwrap_or(&Duration::ZERO); + seen += 1; + tokio::spawn(async move { + let mut acc = Vec::new(); + let mut buf = [0u8; 4096]; + loop { + let n = match sock.read(&mut buf).await { + Ok(0) | Err(_) => break, + Ok(n) => n, + }; + acc.extend_from_slice(&buf[..n]); + if let Some(hdr_end) = + acc.windows(4).position(|w| w == b"\r\n\r\n").map(|p| p + 4) + { + let headers = String::from_utf8_lossy(&acc[..hdr_end]).to_lowercase(); + let len: usize = headers + .lines() + .find_map(|l| l.strip_prefix("content-length:")) + .and_then(|v| v.trim().parse().ok()) + .unwrap_or(0); + if acc.len() >= hdr_end + len { + break; + } + } + } + tokio::time::sleep(delay).await; + let _ = sock + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n") + .await; + let _ = sock.flush().await; + }); + } + }); endpoint } - /// A sleeping-but-live endpoint. Answers `200` with an empty body after - /// `delays[i]` for the i-th request it accepts (the last entry repeats), so - /// a test can make one add slow and the next fast. Drains the full request, - /// headers plus the declared `Content-Length` body, before sleeping: exactly - /// as in `rejecting_endpoint`, answering early and closing would surface as - /// a write failure on the client and turn a slow-but-healthy add into a - /// different failure shape. + /// A `tracing` sink a test can read back, so the deadline warn can be + /// asserted on rather than assumed. Installed with `set_default`, which is + /// thread-local and scoped to the guard, so it cannot bleed into any other + /// test in the binary. + #[derive(Clone, Default)] + struct CapturedLogs(std::sync::Arc>>); + + impl CapturedLogs { + fn text(&self) -> String { + String::from_utf8_lossy(&self.0.lock().unwrap()).to_string() + } + } + + impl std::io::Write for CapturedLogs { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0.lock().unwrap().extend_from_slice(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for CapturedLogs { + type Writer = CapturedLogs; + fn make_writer(&'a self) -> Self::Writer { + self.clone() + } + } + + fn capture_logs() -> (CapturedLogs, tracing::subscriber::DefaultGuard) { + let logs = CapturedLogs::default(); + let subscriber = tracing_subscriber::fmt() + .with_writer(logs.clone()) + .with_max_level(tracing::Level::WARN) + .with_ansi(false) + .finish(); + let guard = tracing::subscriber::set_default(subscriber); + (logs, guard) + } + + /// The add sink must be built from the shared no-redirect client, and the + /// per-request override must actually reach the request. Against a silent + /// endpoint (accept succeeds, no response ever written) a bare + /// `reqwest::Client::new()` blocks forever. With `Some(2s)` the call must + /// come back well inside that, and as a reqwest timeout: the elapsed + /// assertion is the real RED signal, and the outer `tokio::time::timeout` + /// is only a wedge guard so a regression fails the suite instead of hanging + /// it (`cargo test` has no per-test timeout). The old "no elapsed assertion + /// because the timeout is a process-global `OnceLock`" caveat no longer + /// holds now that `request_timeout` overrides it per call. + #[tokio::test] + async fn pin_git_object_against_silent_endpoint_errors_within_its_own_timeout() { + let endpoint = crate::test_support::silent_http_endpoint().await; + let started = std::time::Instant::now(); + let inner = tokio::time::timeout( + Duration::from_secs(30), + pin_git_object( + &endpoint, + "deadbeef", + b"some object bytes\n", + Some(Duration::from_secs(2)), + ), + ) + .await + .expect("wedge guard: pin_git_object must return long before 30s"); + let elapsed = started.elapsed(); + let err = inner.expect_err("a silent endpoint must not surface as a successful pin"); + assert!( + elapsed < Duration::from_secs(5), + "the 2s per-request override must bound this call, not the client's own ceiling (took {elapsed:?})" + ); + assert!( + err.downcast_ref::() + .is_some_and(|e| e.is_timeout()), + "a silent endpoint must surface as a reqwest timeout, preserved as the error's source: {err:#}" + ); + } + + /// The second unhardened sink, reached from `sync.rs`. Same shape as above. + #[tokio::test] + async fn cat_against_silent_endpoint_errors_within_its_own_timeout() { + let endpoint = crate::test_support::silent_http_endpoint().await; + let inner = tokio::time::timeout(Duration::from_secs(30), cat(&endpoint, "bafkqaaa")) + .await + .expect( + "cat must return before the outer 30s timeout — an unbounded client hangs here", + ); + assert!( + inner.is_err(), + "a silent endpoint must surface as a transport error, not successful bytes" + ); + } + + /// The permit-hold bound. `pin_new_objects` runs under a deferring + /// `pin_semaphore`, so without a batch deadline the hold is O(N) with N + /// chosen by the pusher. Five objects against an endpoint that takes 2s + /// each, under a 5.5s budget, must stop partway: only the first two can + /// finish inside the budget, so the batch is truncated and the remainder is + /// left unattempted with one warn naming how many. + /// + /// The windows are deliberately loose. Three pins would need every add to + /// answer in under 1.83s, which the endpoint's own 2s sleep forbids, and one + /// pin needs only the first add to land inside 5.5s, so both bounds hold + /// with more than a second of slack on a loaded box. + #[sqlx::test] + async fn pin_new_objects_stops_the_batch_at_its_deadline(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.expect("migrations"); + let tmp = tempfile::TempDir::new().unwrap(); + let repo_path = tmp.path().join("slow.git"); + let oids = seed_loose_blobs(&repo_path, 5); + let endpoint = delaying_endpoint(vec![Duration::from_secs(2)]).await; + + let (logs, _guard) = capture_logs(); + let pinned = tokio::time::timeout( + Duration::from_secs(30), + pin_new_objects( + &endpoint, + &repo_path, + "git", + Duration::from_secs(30), + oids, + &db, + "repo-batch-budget", + Duration::from_millis(5500), + ), + ) + .await + .expect("wedge guard: a 5.5s budget cannot take 30s"); + + assert!( + (1..=3).contains(&pinned.len()), + "the batch must stop partway, not pin all five and not stall on the first: pinned {}", + pinned.len() + ); + let text = logs.text(); + let warns: Vec<&str> = text + .lines() + .filter(|l| l.contains("pin batch deadline reached")) + .collect(); + assert_eq!( + warns.len(), + 1, + "the deadline must be reported exactly once for the batch, not per object: {text}" + ); + let unattempted: usize = warns[0] + .split("unattempted=") + .nth(1) + .and_then(|s| { + s.split(|c: char| !c.is_ascii_digit()) + .next() + .and_then(|d| d.parse().ok()) + }) + .unwrap_or_else(|| panic!("the deadline warn must name the unattempted count: {text}")); + assert!( + unattempted >= 1 && unattempted + pinned.len() <= 5, + "unattempted={unattempted} with {} pinned is not a partial batch of five", + pinned.len() + ); + } + + /// The must-not case, and the regression that killed the old transport + /// classifier: an endpoint that is slow but genuinely alive must NOT cost + /// the rest of the batch. The first add takes 13s, past the shared client's + /// 10s ceiling, which is exactly what the classifier used to read as a dead + /// endpoint; the second is immediate. Under a 90s budget both must pin, so + /// this fails if the per-request timeout is left at the client default and + /// fails if any error arm breaks the loop. Two objects, not one, because + /// with one object "did not abandon the rest" would be vacuous. + #[sqlx::test] + async fn pin_new_objects_does_not_abandon_the_batch_on_a_slow_but_alive_endpoint( + pool: sqlx::PgPool, + ) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.expect("migrations"); + let tmp = tempfile::TempDir::new().unwrap(); + let repo_path = tmp.path().join("slow_alive.git"); + let oids = seed_loose_blobs(&repo_path, 2); + let endpoint = + delaying_endpoint(vec![Duration::from_secs(13), Duration::from_millis(0)]).await; + + let pinned = tokio::time::timeout( + Duration::from_secs(60), + pin_new_objects( + &endpoint, + &repo_path, + "git", + Duration::from_secs(30), + oids, + &db, + "repo-batch-continues", + Duration::from_secs(90), + ), + ) + .await + .expect("wedge guard: a 13s add plus an immediate one cannot take 60s"); + assert_eq!( + pinned.len(), + 2, + "a slow but progressing endpoint must pin both objects: an upload past the client's \ + 10s default is not a dead endpoint" + ); + } + + /// The must-not case for the warn-and-continue arm: a live endpoint + /// rejecting each object with `500` is a per-object failure, so the loop + /// must still warn and continue and every object must be attempted. Without + /// this, a `break` arm could be reintroduced and the deadline test above + /// would not notice. + #[sqlx::test] + async fn pin_new_objects_continues_past_a_per_object_rejection(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.expect("migrations"); + let tmp = tempfile::TempDir::new().unwrap(); + let repo_path = tmp.path().join("rejecting.git"); + let oids = seed_loose_blobs(&repo_path, 4); + let requests = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let endpoint = rejecting_endpoint(std::sync::Arc::clone(&requests)).await; + + let pinned = tokio::time::timeout( + Duration::from_secs(30), + pin_new_objects( + &endpoint, + &repo_path, + "git", + Duration::from_secs(30), + oids, + &db, + "repo-batch-rejects", + Duration::from_secs(60), + ), + ) + .await + .expect("a rejecting endpoint answers immediately, so this cannot take 30s"); + assert!(pinned.is_empty(), "every add was rejected"); + assert_eq!( + requests.load(std::sync::atomic::Ordering::SeqCst), + 4, + "a non-2xx rejection is per-object: all four objects must still be attempted" + ); + } + + /// Write an executable `/bin/sh` script. Copied per module rather than shared: + /// `store.rs` and `visibility_pack.rs` each keep their own, since their test mods + /// are private and not reachable from here. + #[cfg(unix)] + fn write_script(path: &std::path::Path, body: &str) { + use std::os::unix::fs::PermissionsExt; + std::fs::write(path, body).expect("write fake git"); + let mut perm = std::fs::metadata(path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(path, perm).unwrap(); + } + + /// A `git_bin` wrapper that records every invocation's arguments and then execs + /// the real git, so a test can tell which objects the loop actually attempted. + /// The returned pin list cannot: it is empty both when the loop broke after one + /// object and when it continued past all of them. + #[cfg(unix)] + fn counting_git(dir: &std::path::Path, log: &std::path::Path) -> String { + let fake = dir.join("counting-git"); + write_script( + &fake, + &format!( + "#!/bin/sh\necho \"$*\" >> {}\nexec git \"$@\"\n", + log.display() + ), + ); + fake.to_str().unwrap().to_string() + } + + /// How many objects the loop actually attempted, read off the invocation log. /// - /// An empty body is a successful pin: `pin_git_object` falls back to the CID - /// it computed from the bytes when the response carries no `Hash`. - async fn delaying_endpoint(delays: Vec) -> String { - use tokio::io::{AsyncReadExt, AsyncWriteExt}; - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let endpoint = format!("http://{}", listener.local_addr().unwrap()); - tokio::spawn(async move { - let mut seen = 0usize; - while let Ok((mut sock, _)) = listener.accept().await { - let delay = *delays - .get(seen) - .or_else(|| delays.last()) - .unwrap_or(&Duration::ZERO); - seen += 1; - tokio::spawn(async move { - let mut acc = Vec::new(); - let mut buf = [0u8; 4096]; - loop { - let n = match sock.read(&mut buf).await { - Ok(0) | Err(_) => break, - Ok(n) => n, - }; - acc.extend_from_slice(&buf[..n]); - if let Some(hdr_end) = - acc.windows(4).position(|w| w == b"\r\n\r\n").map(|p| p + 4) - { - let headers = String::from_utf8_lossy(&acc[..hdr_end]).to_lowercase(); - let len: usize = headers - .lines() - .find_map(|l| l.strip_prefix("content-length:")) - .and_then(|v| v.trim().parse().ok()) - .unwrap_or(0); - if acc.len() >= hdr_end + len { - break; - } - } - } - tokio::time::sleep(delay).await; - let _ = sock - .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n") - .await; - let _ = sock.flush().await; - }); + /// Counts `--batch-check` invocations, not log lines and not oid occurrences: the + /// type probe carries its oid on stdin rather than in argv, so an oid appears in + /// the log only once an object has already got past its probe, and a healthy + /// object costs two invocations to a faulting one's one. + fn objects_attempted(log: &std::path::Path) -> usize { + std::fs::read_to_string(log) + .unwrap_or_default() + .lines() + .filter(|l| l.contains("--batch-check")) + .count() + } + + /// #174 F3, the finding itself: the git read runs while the `pin_semaphore` + /// permit is held, so a wedged `git cat-file` used to hold that permit for as + /// long as the child lived, with no deadline and no reaping, on a path a pusher + /// drives. With the read bounded, a git that never answers costs the batch its + /// budget plus one watchdog teardown and no more. + /// + /// The fake traps SIGTERM and sleeps a BOUNDED 30s, following the fixture in + /// `visibility_pack.rs`: with the deadline neutralized the read would otherwise + /// leave the blocking closure and its child alive long after the test-level + /// timeout fires, wedging the run instead of reporting a failure. The endpoint is + /// never reached, since no object's bytes are ever produced. + /// + /// The batch ends on the BUDGET, not on the fault arm: the repo is a healthy bare + /// store, so the timeout's `Transient` verdict is object-scoped and the loop moves on, + /// only to find the budget spent. Capturing that warn is not decoration. `tracing` + /// caches a callsite's interest globally the first time it is hit, and a hit from a + /// thread with no subscriber caches it as never-interested for the whole binary, which + /// silently blinds the deadline tests running beside this one. + #[cfg(unix)] + #[sqlx::test] + async fn pin_new_objects_returns_by_budget_with_a_hung_git(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.expect("migrations"); + let tmp = tempfile::TempDir::new().unwrap(); + let repo_path = tmp.path().join("wedged.git"); + let oids = seed_loose_blobs(&repo_path, 3); + let fake = tmp.path().join("hanging-git"); + write_script(&fake, "#!/bin/sh\ntrap '' TERM\necho $$ > pid\nsleep 30\n"); + + let (logs, _guard) = capture_logs(); + let started = std::time::Instant::now(); + let pinned = tokio::time::timeout( + Duration::from_secs(25), + pin_new_objects( + "http://127.0.0.1:9", + &repo_path, + fake.to_str().unwrap(), + Duration::from_secs(30), + oids, + &db, + "repo-merge-test", + Duration::from_secs(2), + ), + ) + .await + .expect( + "a wedged git must not hold the pin permit past the batch budget: the read is \ + bounded and reaped, so this cannot reach the outer timeout", + ); + let elapsed = started.elapsed(); + + assert!( + pinned.is_empty(), + "a git that never answers cannot produce a pinned object: {pinned:?}" + ); + assert!( + elapsed < Duration::from_secs(20), + "elapsed {elapsed:?} must stay inside the budget plus one watchdog teardown" + ); + let text = logs.text(); + assert_eq!( + text.lines() + .filter(|l| l.contains("pin batch deadline reached")) + .count(), + 1, + "one wedged read must spend the whole budget and stop the batch there, exactly \ + once: {text}" + ); + + // The child's process group must be gone once the call returns; a bounded read + // that leaves the child running has only moved the hold somewhere else. + let pid: i32 = std::fs::read_to_string(repo_path.join("pid")) + .expect("the fake git must have recorded its pid, or it was never on the read path") + .trim() + .parse() + .unwrap(); + let mut gone = false; + for _ in 0..200 { + // SAFETY: kill(2) with signal 0 only probes existence; ESRCH means gone. + if unsafe { libc::kill(pid, 0) } != 0 { + gone = true; + break; } - }); - endpoint + std::thread::sleep(Duration::from_millis(10)); + } + assert!( + gone, + "the reaped fake git ({pid}) must not outlive the call" + ); + } + + /// [`PIN_READ_FLOOR`] itself, which nothing else in the suite makes load-bearing: with + /// the floor at zero every test here still passes, because they all either finish + /// inside their budget or run it down to nothing. The dead zone is the interesting + /// region, a remainder that is nonzero but too small to cover a bounded read's + /// teardown, and it takes a fixture built to land in it. + /// + /// The arithmetic, with `r` the time one healthy read costs: a 1500ms budget and a + /// 700ms upload leave `800 - r` at the top of the second iteration, which is below the + /// 1100ms floor for every `r`, while the first iteration's post-read gate needs only + /// `r <= 400ms`. So the batch must stop after exactly one object, and it stops on the + /// FLOOR rather than on exhaustion: 800ms is still a perfectly nonzero remainder, and a + /// zero-floor gate would happily spend it spawning a child it cannot afford to reap. + #[cfg(unix)] + #[sqlx::test] + async fn pin_new_objects_stops_when_the_remainder_falls_below_the_read_floor( + pool: sqlx::PgPool, + ) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.expect("migrations"); + let tmp = tempfile::TempDir::new().unwrap(); + let repo_path = tmp.path().join("floor.git"); + let oids = seed_loose_blobs(&repo_path, 3); + let log = tmp.path().join("calls.log"); + let git_bin = counting_git(tmp.path(), &log); + let endpoint = delaying_endpoint(vec![Duration::from_millis(700)]).await; + + let (logs, _guard) = capture_logs(); + let pinned = tokio::time::timeout( + Duration::from_secs(30), + pin_new_objects( + &endpoint, + &repo_path, + &git_bin, + Duration::from_secs(30), + oids, + &db, + "repo-merge-test", + Duration::from_millis(1500), + ), + ) + .await + .expect("wedge guard: a 1.5s budget cannot take 30s"); + + // The named property first, so a regression reddens on it rather than on a + // downstream count that a zero floor also happens to change. + assert_eq!( + objects_attempted(&log), + 1, + "a remainder below the read floor must stop the batch, not buy a bounded child \ + that can only be spawned and reaped" + ); + assert_eq!( + pinned.len(), + 1, + "the first object is inside the budget and must pin: {pinned:?}" + ); + let text = logs.text(); + assert_eq!( + text.lines() + .filter(|l| l.contains("pin batch deadline reached")) + .count(), + 1, + "the truncation must be reported exactly once: {text}" + ); } - /// A `tracing` sink a test can read back, so the deadline warn can be - /// asserted on rather than assumed. Installed with `set_default`, which is - /// thread-local and scoped to the guard, so it cannot bleed into any other - /// test in the binary. - #[derive(Clone, Default)] - struct CapturedLogs(std::sync::Arc>>); + /// A store-wide fault breaks the batch instead of amplifying it. When the object + /// store itself cannot be read every remaining object fails identically, so + /// continuing would spawn one doomed bounded child per object and burn the budget + /// on reaping them. + /// + /// The fixture looks wrong and is not: with the objects LOOSE and only + /// `objects/pack` unreadable, git still resolves each object, but it prints an + /// `error:` diagnostic that the probe routes to a fault before the present/missing + /// parse, so the read reaches `classify_store_fault` and (the store being + /// unreadable) returns `Transient`. + #[cfg(unix)] + #[sqlx::test] + async fn pin_new_objects_breaks_the_batch_on_an_unreadable_store(pool: sqlx::PgPool) { + use std::os::unix::fs::PermissionsExt; + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.expect("migrations"); + let tmp = tempfile::TempDir::new().unwrap(); + let repo_path = tmp.path().join("unreadable.git"); + let oids = seed_loose_blobs(&repo_path, 5); + let log = tmp.path().join("calls.log"); + let git_bin = counting_git(tmp.path(), &log); + let endpoint = delaying_endpoint(vec![Duration::ZERO]).await; + + let pack_dir = repo_path.join("objects").join("pack"); + let chmod = |mode: u32| { + let mut perms = std::fs::metadata(&pack_dir).unwrap().permissions(); + perms.set_mode(mode); + std::fs::set_permissions(&pack_dir, perms).unwrap(); + }; + chmod(0o000); + // Root bypasses permission bits, so witness the exact operation the probe + // performs and skip rather than falsely fail. + let genuinely_unreadable = std::fs::read_dir(&pack_dir).is_err(); - impl CapturedLogs { - fn text(&self) -> String { - String::from_utf8_lossy(&self.0.lock().unwrap()).to_string() - } - } + let pinned = tokio::time::timeout( + Duration::from_secs(30), + pin_new_objects( + &endpoint, + &repo_path, + &git_bin, + Duration::from_secs(30), + oids.clone(), + &db, + "repo-merge-test", + Duration::from_secs(60), + ), + ) + .await + .expect("an immediately-faulting store cannot take 30s"); + let attempted = objects_attempted(&log); + chmod(0o755); // restore BEFORE any assertion that can panic, so TempDir cleans up - impl std::io::Write for CapturedLogs { - fn write(&mut self, buf: &[u8]) -> std::io::Result { - self.0.lock().unwrap().extend_from_slice(buf); - Ok(buf.len()) - } - fn flush(&mut self) -> std::io::Result<()> { - Ok(()) + if genuinely_unreadable { + assert!( + pinned.is_empty(), + "nothing can be pinned through a store that cannot be read: {pinned:?}" + ); + assert_eq!( + attempted, 1, + "a store-wide fault must break the batch after the first object, not spawn \ + one doomed bounded child per object: {attempted} of 5 objects were read" + ); } } - impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for CapturedLogs { - type Writer = CapturedLogs; - fn make_writer(&'a self) -> Self::Writer { - self.clone() + /// The failure mode the store-wide re-check exists for. A `Transient` fault does not + /// prove the store is gone: the readability verdict is judged FOR one oid, so a single + /// unreadable `objects/` fan-out (1/256 of the store) taints only the objects that + /// live in it. Breaking there forfeits every remaining object over a fault that costs + /// at most a few of them, and permanently: the documented recovery re-derives the same + /// list and breaks at the same index. + /// + /// Exactly ONE fan-out dir is chmod'd, and the expected pin count is derived from the + /// oids that actually land in it rather than assumed to be one: two seeded blobs can + /// share a fan-out prefix, and hardcoding "four must pin" would make this test flap on + /// a collision instead of reporting it. + #[cfg(unix)] + #[sqlx::test] + async fn pin_new_objects_continues_past_one_unreadable_fanout_dir(pool: sqlx::PgPool) { + use std::os::unix::fs::PermissionsExt; + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.expect("migrations"); + let tmp = tempfile::TempDir::new().unwrap(); + let repo_path = tmp.path().join("one_bad_fanout.git"); + let oids = seed_loose_blobs(&repo_path, 5); + let log = tmp.path().join("calls.log"); + let git_bin = counting_git(tmp.path(), &log); + let endpoint = delaying_endpoint(vec![Duration::ZERO]).await; + + let prefix = oids[0][0..2].to_string(); + let tainted: Vec = oids.iter().filter(|o| o[0..2] == prefix).cloned().collect(); + assert!( + tainted.len() < oids.len(), + "the fixture must leave healthy objects outside the tainted fan-out, or this \ + test cannot tell an object-scoped fault from a store-wide one" + ); + + let fanout = repo_path.join("objects").join(&prefix); + let chmod = |mode: u32| { + let mut perms = std::fs::metadata(&fanout).unwrap().permissions(); + perms.set_mode(mode); + std::fs::set_permissions(&fanout, perms).unwrap(); + }; + chmod(0o000); + // Root bypasses permission bits, so witness the exact operation the probe performs + // (an open of this oid's loose path) and skip rather than falsely fail. + let genuinely_unreadable = std::fs::File::open(fanout.join(&oids[0][2..])).is_err(); + + let pinned = tokio::time::timeout( + Duration::from_secs(30), + pin_new_objects( + &endpoint, + &repo_path, + &git_bin, + Duration::from_secs(30), + oids.clone(), + &db, + "repo-merge-test", + Duration::from_secs(60), + ), + ) + .await + .expect("an immediate endpoint and four healthy objects cannot take 30s"); + let attempted = objects_attempted(&log); + chmod(0o755); // restore BEFORE any assertion that can panic, so TempDir cleans up + + if genuinely_unreadable { + assert_eq!( + attempted, + oids.len(), + "one unreadable fan-out is 1/256 of the store, not the store: every object \ + must still be attempted, got {attempted} of {}", + oids.len() + ); + let pinned_shas: Vec<&String> = pinned.iter().map(|(sha, _)| sha).collect(); + let expected: Vec<&String> = oids.iter().filter(|o| !tainted.contains(o)).collect(); + assert_eq!( + pinned_shas, expected, + "every object outside the tainted fan-out must pin, and only those" + ); } } - fn capture_logs() -> (CapturedLogs, tracing::subscriber::DefaultGuard) { - let logs = CapturedLogs::default(); - let subscriber = tracing_subscriber::fmt() - .with_writer(logs.clone()) - .with_max_level(tracing::Level::WARN) - .with_ansi(false) - .finish(); - let guard = tracing::subscriber::set_default(subscriber); - (logs, guard) - } + /// The must-not direction of the arm above: an object-scoped fault must NOT break + /// the batch. One corrupt loose object among healthy ones is a `Deterministic` + /// fault (the store is readable, git still fails), and the documented recovery path + /// cannot repair it: a later full-scan push re-offers the same object and would + /// break at the same place, so breaking here stops the repo pinning permanently. + /// + /// Deliberately not the bad-config corruption, which is repo-wide: all five objects + /// would fault and the test would pin the store-wide case rather than the + /// object-scoped one this arm rests on. + #[cfg(unix)] + #[sqlx::test] + async fn pin_new_objects_continues_past_a_deterministic_fault(pool: sqlx::PgPool) { + use std::os::unix::fs::PermissionsExt; + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.expect("migrations"); + let tmp = tempfile::TempDir::new().unwrap(); + let repo_path = tmp.path().join("corrupt.git"); + let oids = seed_loose_blobs(&repo_path, 5); + let log = tmp.path().join("calls.log"); + let git_bin = counting_git(tmp.path(), &log); + let endpoint = delaying_endpoint(vec![Duration::ZERO]).await; - /// The add sink must be built from the shared no-redirect client, and the - /// per-request override must actually reach the request. Against a silent - /// endpoint (accept succeeds, no response ever written) a bare - /// `reqwest::Client::new()` blocks forever. With `Some(2s)` the call must - /// come back well inside that, and as a reqwest timeout: the elapsed - /// assertion is the real RED signal, and the outer `tokio::time::timeout` - /// is only a wedge guard so a regression fails the suite instead of hanging - /// it (`cargo test` has no per-test timeout). The old "no elapsed assertion - /// because the timeout is a process-global `OnceLock`" caveat no longer - /// holds now that `request_timeout` overrides it per call. - #[tokio::test] - async fn pin_git_object_against_silent_endpoint_errors_within_its_own_timeout() { - let endpoint = crate::test_support::silent_http_endpoint().await; - let started = std::time::Instant::now(); - let inner = tokio::time::timeout( + // Overwrite exactly one loose object with non-zlib garbage (0o444 by default). + let victim = repo_path + .join("objects") + .join(&oids[0][0..2]) + .join(&oids[0][2..]); + assert!(victim.is_file(), "fixture must leave the blob loose"); + let mut perms = std::fs::metadata(&victim).unwrap().permissions(); + perms.set_mode(0o644); + std::fs::set_permissions(&victim, perms).unwrap(); + std::fs::write(&victim, b"garbage not a zlib stream").unwrap(); + + let pinned = tokio::time::timeout( Duration::from_secs(30), - pin_git_object( + pin_new_objects( &endpoint, - "deadbeef", - b"some object bytes\n", - Some(Duration::from_secs(2)), + &repo_path, + &git_bin, + Duration::from_secs(30), + oids.clone(), + &db, + "repo-merge-test", + Duration::from_secs(60), ), ) .await - .expect("wedge guard: pin_git_object must return long before 30s"); - let elapsed = started.elapsed(); - let err = inner.expect_err("a silent endpoint must not surface as a successful pin"); - assert!( - elapsed < Duration::from_secs(5), - "the 2s per-request override must bound this call, not the client's own ceiling (took {elapsed:?})" + .expect("an immediate endpoint and four healthy objects cannot take 30s"); + + assert_eq!( + objects_attempted(&log), + 5, + "an object-scoped fault must not stop the batch: every object must be read" ); - assert!( - err.downcast_ref::() - .is_some_and(|e| e.is_timeout()), - "a silent endpoint must surface as a reqwest timeout, preserved as the error's source: {err:#}" + assert_eq!( + pinned.len(), + 4, + "one corrupt object must cost only itself: the other four must still pin" ); } - /// The second unhardened sink, reached from `sync.rs`. Same shape as above. - #[tokio::test] - async fn cat_against_silent_endpoint_errors_within_its_own_timeout() { - let endpoint = crate::test_support::silent_http_endpoint().await; - let inner = tokio::time::timeout(Duration::from_secs(30), cat(&endpoint, "bafkqaaa")) + // --------------------------------------------------------------------- + // F3 (#173, jatmn): the DB operations inside the budgeted region. + // + // `api/repos.rs` holds the GLOBAL `pin_semaphore` permit across the whole + // `pin_new_objects` call. `batch_budget_gate` only gates BETWEEN objects and + // the git read is already clamped, but every DB call in the region used to be + // a bare await, so one stalled query parked the permit past every budget and, + // once all pin permits were so held, post-push IPFS replication stopped for + // every repo on the node. The tests below drive that stall with a + // `LOCK TABLE .. IN ACCESS EXCLUSIVE MODE` held on a dedicated pooled + // connection, the same technique as `get_by_cid_stalled_metadata_query_frees_ + // walk_permit` in api/ipfs.rs, and copy its tolerances (a ~1s budget, an + // `elapsed < 3s` assertion, a 10s outer wrap). Pre-fix each one blocks on the + // lock until the outer wrap fires. + // --------------------------------------------------------------------- + + /// Take an `ACCESS EXCLUSIVE` lock on `table` on a dedicated pooled connection. + /// Every SELECT needs `ACCESS SHARE`, which conflicts, so the next statement + /// touching the table blocks at lock acquisition regardless of row count. + async fn lock_table( + pool: &sqlx::PgPool, + table: &str, + ) -> sqlx::pool::PoolConnection { + let mut conn = pool.acquire().await.unwrap(); + sqlx::raw_sql(&format!( + "BEGIN; LOCK TABLE {table} IN ACCESS EXCLUSIVE MODE;" + )) + .execute(&mut *conn) + .await + .unwrap(); + conn + } + + async fn rollback(conn: &mut sqlx::pool::PoolConnection) { + sqlx::raw_sql("ROLLBACK") + .execute(&mut **conn) .await - .expect( - "cat must return before the outer 30s timeout — an unbounded client hangs here", - ); + .unwrap(); + } + + /// A raw CIDv1 to seed a pin with, so the opportunistic legacy repair takes its + /// cost gate and reads no bytes. The value only has to be a canonical raw key. + fn seed_cid() -> String { + Cid::from_git_object_bytes(b"pin loop seed").to_string() + } + + /// The helper's zero-remainder path, where the absolute deadline is the whole + /// point: a spent deadline must error immediately rather than hand the call a + /// fresh budget. + /// + /// This is a unit test rather than a loop-driven one on purpose. Driving a ~0 + /// `batch_budget` through `pin_new_objects` is VACUOUS: `batch_budget_gate` + /// returns None below [`PIN_READ_FLOOR`] as the first statement of the loop body, + /// so the batch breaks before any DB call and the test passes identically with + /// `db_bounded` deleted. The helper is tested where the zero-remainder path + /// actually runs. + #[tokio::test] + async fn db_bounded_elapsed_deadline_errors_promptly() { + let spent = Instant::now() - Duration::from_secs(5); + let started = std::time::Instant::now(); + let out: Result = db_bounded(spent, async { + tokio::time::sleep(Duration::from_secs(30)).await; + Ok(7u8) + }) + .await; + assert!( - inner.is_err(), - "a silent endpoint must surface as a transport error, not successful bytes" + matches!(out, Err(BoundedDbError::Elapsed)), + "a spent deadline must yield the DISTINGUISHABLE timeout arm, not a value \ + and not a generic DB error: {out:?}" + ); + assert!( + started.elapsed() < Duration::from_secs(1), + "a spent deadline must error at once, not after a fresh full budget; got {:?}", + started.elapsed() ); } - /// The permit-hold bound. `pin_new_objects` runs under a deferring - /// `pin_semaphore`, so without a batch deadline the hold is O(N) with N - /// chosen by the pusher. Five objects against an endpoint that takes 2s - /// each, under a 5.5s budget, must stop partway: only the first two can - /// finish inside the budget, so the batch is truncated and the remainder is - /// left unattempted with one warn naming how many. + /// U5 (#173): the elapsed arm of the discovery record leaves neither row behind. + /// + /// What this proves and what it does NOT: it shows the wrapper composition + /// (`db_bounded` over `retry_db_record` over `record_discovered_pin_source`) returns + /// promptly and cleanly on the elapsed arm with a healthy pool, so the call site's + /// "definitely did not land" reading is not contradicted here. It is NOT evidence of + /// transactionality: a spent deadline reduces to `timeout(0, fut)`, the future never + /// starts, and "neither row landed" would hold just as well for two separate calls. + /// It kills no mutation. The atomicity property is proven by + /// `sweep_discovery_failed_marker_does_not_strand_public_copy` and its mutations + /// alone. /// - /// The windows are deliberately loose. Three pins would need every add to - /// answer in under 1.83s, which the endpoint's own 2s sleep forbids, and one - /// pin needs only the first add to land inside 5.5s, so both bounds hold - /// with more than a second of slack on a loaded box. + /// Driven directly with a past deadline rather than through `db_record_deadline`, + /// whose `DB_RECORD_GRACE` floor makes this arm near-unreachable in production. #[sqlx::test] - async fn pin_new_objects_stops_the_batch_at_its_deadline(pool: sqlx::PgPool) { - let db = crate::db::Db::for_testing(pool); + async fn discovery_record_elapsed_leaves_neither_row(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool.clone()); db.run_migrations().await.expect("migrations"); - let tmp = tempfile::TempDir::new().unwrap(); - let repo_path = tmp.path().join("slow.git"); - let oids = seed_loose_blobs(&repo_path, 5); - let endpoint = delaying_endpoint(vec![Duration::from_secs(2)]).await; + let sha = "d5".repeat(32); + // A `pinned_cids` row so the sentinel's `WHERE EXISTS` guard is satisfied and its + // absence below is the timeout's doing, not the guard's. + db.record_pinned_cid_with_source(&sha, &seed_cid(), "repo-first") + .await + .expect("seed the pinned row"); - let (logs, _guard) = capture_logs(); - let pinned = tokio::time::timeout( - Duration::from_secs(30), - pin_new_objects( - &endpoint, - &repo_path, - "git", - oids, - &db, - Duration::from_millis(5500), - ), + let spent = Instant::now() - Duration::from_secs(5); + let started = std::time::Instant::now(); + let out = db_bounded( + spent, + retry_db_record(|| db.record_discovered_pin_source(&sha, "repo-discovered")), ) - .await - .expect("wedge guard: a 5.5s budget cannot take 30s"); + .await; assert!( - (1..=3).contains(&pinned.len()), - "the batch must stop partway, not pin all five and not stall on the first: pinned {}", - pinned.len() + matches!(out, Err(BoundedDbError::Elapsed)), + "a spent deadline must yield the timeout arm, not a value: {out:?}" + ); + assert!( + started.elapsed() < Duration::from_secs(1), + "the wrapped retry ladder must not outlive the spent deadline; got {:?}", + started.elapsed() ); - let text = logs.text(); - let warns: Vec<&str> = text - .lines() - .filter(|l| l.contains("pin batch deadline reached")) - .collect(); assert_eq!( - warns.len(), - 1, - "the deadline must be reported exactly once for the batch, not per object: {text}" + db.pin_sources_for_oid(&sha).await.unwrap(), + vec!["repo-first".to_string()], + "no discovered source row landed" ); - let unattempted: usize = warns[0] - .split("unattempted=") - .nth(1) - .and_then(|s| { - s.split(|c: char| !c.is_ascii_digit()) - .next() - .and_then(|d| d.parse().ok()) - }) - .unwrap_or_else(|| panic!("the deadline warn must name the unattempted count: {text}")); assert!( - unattempted >= 1 && unattempted + pinned.len() <= 5, - "unattempted={unattempted} with {} pinned is not a partial batch of five", - pinned.len() + !db.pin_sources_incomplete(&sha).await.unwrap(), + "no sentinel landed either" ); } - /// The must-not case, and the regression that killed the old transport - /// classifier: an endpoint that is slow but genuinely alive must NOT cost - /// the rest of the batch. The first add takes 13s, past the shared client's - /// 10s ceiling, which is exactly what the classifier used to read as a dead - /// endpoint; the second is immediate. Under a 90s budget both must pin, so - /// this fails if the per-request timeout is left at the client default and - /// fails if any error arm breaks the loop. Two objects, not one, because - /// with one object "did not abandon the rest" would be vacuous. - #[sqlx::test] - async fn pin_new_objects_does_not_abandon_the_batch_on_a_slow_but_alive_endpoint( - pool: sqlx::PgPool, - ) { - let db = crate::db::Db::for_testing(pool); - db.run_migrations().await.expect("migrations"); - let tmp = tempfile::TempDir::new().unwrap(); - let repo_path = tmp.path().join("slow_alive.git"); - let oids = seed_loose_blobs(&repo_path, 2); - let endpoint = - delaying_endpoint(vec![Duration::from_secs(13), Duration::from_millis(0)]).await; + /// The ABSOLUTE half of the same helper, which no loop-level test actually binds. + /// + /// `db_bounded` takes an `Instant`, not a `Duration`, so every call sharing one + /// deadline shares ONE budget: whatever an earlier call spends, a later one no + /// longer has. `pin_new_objects_multi_object_stall_charges_one_budget` covers that + /// end to end, but it only goes red under a per-call duration LARGER than the batch + /// budget (its mutation grants `PIN_BATCH_BUDGET`, 120s, against a 1.5s budget). A + /// defect that handed every call a fresh duration at or below the budget would slip + /// straight past it, so the property is bound here instead, where it lives and where + /// no lock, endpoint, or budget gate stands between the assertion and the helper. + /// + /// Two calls against one 3s deadline: the first spends 2s and succeeds, so the + /// second sees ~1s left and must elapse even though its own work needs only 2s. + /// Any fresh per-call duration of 2s or more, INCLUDING one exactly equal to the 3s + /// budget, would return a value there instead. + #[tokio::test] + async fn db_bounded_shares_one_budget_across_sequential_calls() { + let deadline = Instant::now() + Duration::from_secs(3); - let pinned = tokio::time::timeout( - Duration::from_secs(60), - pin_new_objects( - &endpoint, - &repo_path, - "git", - oids, - &db, - Duration::from_secs(90), - ), - ) - .await - .expect("wedge guard: a 13s add plus an immediate one cannot take 60s"); - assert_eq!( - pinned.len(), - 2, - "a slow but progressing endpoint must pin both objects: an upload past the client's \ - 10s default is not a dead endpoint" + let first: Result = db_bounded(deadline, async { + tokio::time::sleep(Duration::from_secs(2)).await; + Ok(1u8) + }) + .await; + assert!( + matches!(first, Ok(1)), + "the first call fits well inside the shared budget and must return its \ + value: {first:?}" + ); + + let started = std::time::Instant::now(); + let second: Result = db_bounded(deadline, async { + tokio::time::sleep(Duration::from_secs(2)).await; + Ok(2u8) + }) + .await; + + assert!( + matches!(second, Err(BoundedDbError::Elapsed)), + "a SHARED deadline is consumed by the calls before it: with ~1s of the 3s \ + left, a 2s call must elapse. A fresh per-call DURATION would let it \ + succeed even when that duration is exactly the budget, and N calls would \ + then charge N budgets instead of one: {second:?}" + ); + assert!( + started.elapsed() < Duration::from_millis(1500), + "the second call must be cut off by the REMAINDER (~1s) rather than run its \ + full 2s; got {:?}", + started.elapsed() ); } - /// The must-not case for the warn-and-continue arm: a live endpoint - /// rejecting each object with `500` is a per-object failure, so the loop - /// must still warn and continue and every object must be attempted. Without - /// this, a `break` arm could be reintroduced and the deadline test above - /// would not notice. + /// Scenario 1: the FIRST DB call in the region (`is_pinned`) stalls. With the + /// batch deadline bounding it the loop abandons the object, the budget gate + /// then breaks the batch, and the call returns at ~budget with nothing pinned. + /// Pre-fix the bare await blocks on the lock for the lock's whole lifetime, + /// holding the caller's global pin permit with it. #[sqlx::test] - async fn pin_new_objects_continues_past_a_per_object_rejection(pool: sqlx::PgPool) { - let db = crate::db::Db::for_testing(pool); + async fn pin_new_objects_stalled_db_returns_by_budget(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool.clone()); db.run_migrations().await.expect("migrations"); let tmp = tempfile::TempDir::new().unwrap(); - let repo_path = tmp.path().join("rejecting.git"); - let oids = seed_loose_blobs(&repo_path, 4); - let requests = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let endpoint = rejecting_endpoint(std::sync::Arc::clone(&requests)).await; + let repo_path = tmp.path().join("stalled.git"); + let oids = seed_loose_blobs(&repo_path, 1); + let endpoint = delaying_endpoint(vec![Duration::ZERO]).await; + // Install a log capture even though nothing here asserts on it: `tracing` + // caches a callsite's interest globally the first time it is hit, and a hit + // from a thread with no subscriber caches it as never-interested for the whole + // binary, which silently blinds the sibling tests that DO assert on the batch + // deadline warn. + let (_logs, _log_guard) = capture_logs(); + + let mut lock = lock_table(&pool, "pinned_cids").await; + let started = std::time::Instant::now(); let pinned = tokio::time::timeout( - Duration::from_secs(30), + Duration::from_secs(10), pin_new_objects( &endpoint, &repo_path, "git", + Duration::from_secs(30), oids, &db, - Duration::from_secs(60), + "repo-stalled-db", + Duration::from_millis(1500), ), ) .await - .expect("a rejecting endpoint answers immediately, so this cannot take 30s"); - assert!(pinned.is_empty(), "every add was rejected"); - assert_eq!( - requests.load(std::sync::atomic::Ordering::SeqCst), - 4, - "a non-2xx rejection is per-object: all four objects must still be attempted" + .expect( + "a stalled DB must cost the batch its budget, not the lock's lifetime: the \ + bare await hangs past this wrap", ); - } - - /// Write an executable `/bin/sh` script. Copied per module rather than shared: - /// `store.rs` and `visibility_pack.rs` each keep their own, since their test mods - /// are private and not reachable from here. - #[cfg(unix)] - fn write_script(path: &std::path::Path, body: &str) { - use std::os::unix::fs::PermissionsExt; - std::fs::write(path, body).expect("write fake git"); - let mut perm = std::fs::metadata(path).unwrap().permissions(); - perm.set_mode(0o755); - std::fs::set_permissions(path, perm).unwrap(); - } + let elapsed = started.elapsed(); - /// A `git_bin` wrapper that records every invocation's arguments and then execs - /// the real git, so a test can tell which objects the loop actually attempted. - /// The returned pin list cannot: it is empty both when the loop broke after one - /// object and when it continued past all of them. - #[cfg(unix)] - fn counting_git(dir: &std::path::Path, log: &std::path::Path) -> String { - let fake = dir.join("counting-git"); - write_script( - &fake, - &format!( - "#!/bin/sh\necho \"$*\" >> {}\nexec git \"$@\"\n", - log.display() - ), + assert!( + pinned.is_empty(), + "a stalled pinned-status check cannot produce a pinned object: {pinned:?}" + ); + assert!( + elapsed < Duration::from_secs(3), + "the batch deadline must end the call at ~budget (1.5s); got {elapsed:?}" ); - fake.to_str().unwrap().to_string() - } - /// How many objects the loop actually attempted, read off the invocation log. - /// - /// Counts `--batch-check` invocations, not log lines and not oid occurrences: the - /// type probe carries its oid on stdin rather than in argv, so an oid appears in - /// the log only once an object has already got past its probe, and a healthy - /// object costs two invocations to a faulting one's one. - fn objects_attempted(log: &std::path::Path) -> usize { - std::fs::read_to_string(log) - .unwrap_or_default() - .lines() - .filter(|l| l.contains("--batch-check")) - .count() + rollback(&mut lock).await; } - /// #174 F3, the finding itself: the git read runs while the `pin_semaphore` - /// permit is held, so a wedged `git cat-file` used to hold that permit for as - /// long as the child lived, with no deadline and no reaping, on a path a pusher - /// drives. With the read bounded, a git that never answers costs the batch its - /// budget plus one watchdog teardown and no more. + /// Scenario 2, and the sharpest unknown-outcome must-not. The object is already + /// pinned, so the loop takes the skip branch and tries to record this repo as an + /// additional source; `pin_repo_sources` is locked, so that insert stalls inside + /// `retry_db_record`. Two properties: /// - /// The fake traps SIGTERM and sleeps a BOUNDED 30s, following the fixture in - /// `visibility_pack.rs`: with the deadline neutralized the read would otherwise - /// leave the blocking closure and its child alive long after the test-level - /// timeout fires, wedging the run instead of reporting a failure. The endpoint is - /// never reached, since no object's bytes are ever produced. + /// - the whole retry ladder (three attempts plus backoff) lives inside ONE + /// remainder, so the call still returns promptly; + /// - on the TIMEOUT arm the incomplete marker is NOT written. A cancelled client + /// future does not cancel the statement Postgres is running, so the source may + /// well be recorded; the marker would force every later `/ipfs` request for the + /// object onto the O(repos) legacy scan, from any unauthenticated caller, on the + /// strength of an outcome the code does not know. Only the definite-error arm + /// marks incomplete. /// - /// The batch ends on the BUDGET, not on the fault arm: the repo is a healthy bare - /// store, so the timeout's `Transient` verdict is object-scoped and the loop moves on, - /// only to find the budget spent. Capturing that warn is not decoration. `tracing` - /// caches a callsite's interest globally the first time it is hit, and a hit from a - /// thread with no subscriber caches it as never-interested for the whole binary, which - /// silently blinds the deadline tests running beside this one. - #[cfg(unix)] + /// The record site carries the durability floor, so the return lands at ~2s + /// rather than at the 1.5s budget; that is the floor working, not a missed bound. #[sqlx::test] - async fn pin_new_objects_returns_by_budget_with_a_hung_git(pool: sqlx::PgPool) { - let db = crate::db::Db::for_testing(pool); + async fn pin_new_objects_skip_branch_stalled_record_returns_by_budget(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool.clone()); db.run_migrations().await.expect("migrations"); let tmp = tempfile::TempDir::new().unwrap(); - let repo_path = tmp.path().join("wedged.git"); - let oids = seed_loose_blobs(&repo_path, 3); - let fake = tmp.path().join("hanging-git"); - write_script(&fake, "#!/bin/sh\ntrap '' TERM\necho $$ > pid\nsleep 30\n"); + let repo_path = tmp.path().join("skip_stalled.git"); + let oids = seed_loose_blobs(&repo_path, 1); + let sha = oids[0].clone(); + db.record_pinned_cid_with_source(&sha, &seed_cid(), "repo-seed") + .await + .unwrap(); + let endpoint = delaying_endpoint(vec![Duration::ZERO]).await; + // Install a log capture even though nothing here asserts on it: `tracing` + // caches a callsite's interest globally the first time it is hit, and a hit + // from a thread with no subscriber caches it as never-interested for the whole + // binary, which silently blinds the sibling tests that DO assert on the batch + // deadline warn. + let (_logs, _log_guard) = capture_logs(); + + let mut lock = lock_table(&pool, "pin_repo_sources").await; - let (logs, _guard) = capture_logs(); let started = std::time::Instant::now(); let pinned = tokio::time::timeout( - Duration::from_secs(25), + Duration::from_secs(10), pin_new_objects( - "http://127.0.0.1:9", + &endpoint, &repo_path, - fake.to_str().unwrap(), + "git", + Duration::from_secs(30), oids, &db, - Duration::from_secs(2), + "repo-skip-stalled", + Duration::from_millis(1500), ), ) .await .expect( - "a wedged git must not hold the pin permit past the batch budget: the read is \ - bounded and reaped, so this cannot reach the outer timeout", + "the wrapped retry ladder must fit inside one remainder: the bare \ + retry_db_record hangs past this wrap", ); let elapsed = started.elapsed(); assert!( pinned.is_empty(), - "a git that never answers cannot produce a pinned object: {pinned:?}" - ); - assert!( - elapsed < Duration::from_secs(20), - "elapsed {elapsed:?} must stay inside the budget plus one watchdog teardown" - ); - let text = logs.text(); - assert_eq!( - text.lines() - .filter(|l| l.contains("pin batch deadline reached")) - .count(), - 1, - "one wedged read must spend the whole budget and stop the batch there, exactly \ - once: {text}" + "an already-pinned object is skipped, never re-pinned: {pinned:?}" ); - - // The child's process group must be gone once the call returns; a bounded read - // that leaves the child running has only moved the hold somewhere else. - let pid: i32 = std::fs::read_to_string(repo_path.join("pid")) - .expect("the fake git must have recorded its pid, or it was never on the read path") - .trim() - .parse() - .unwrap(); - let mut gone = false; - for _ in 0..200 { - // SAFETY: kill(2) with signal 0 only probes existence; ESRCH means gone. - if unsafe { libc::kill(pid, 0) } != 0 { - gone = true; - break; - } - std::thread::sleep(Duration::from_millis(10)); - } assert!( - gone, - "the reaped fake git ({pid}) must not outlive the call" + elapsed < Duration::from_secs(3), + "the record's floored remainder must end the call promptly; got {elapsed:?}" ); - } - /// [`PIN_READ_FLOOR`] itself, which nothing else in the suite makes load-bearing: with - /// the floor at zero every test here still passes, because they all either finish - /// inside their budget or run it down to nothing. The dead zone is the interesting - /// region, a remainder that is nonzero but too small to cover a bounded read's - /// teardown, and it takes a fixture built to land in it. - /// - /// The arithmetic, with `r` the time one healthy read costs: a 1500ms budget and a - /// 700ms upload leave `800 - r` at the top of the second iteration, which is below the - /// 1100ms floor for every `r`, while the first iteration's post-read gate needs only - /// `r <= 400ms`. So the batch must stop after exactly one object, and it stops on the - /// FLOOR rather than on exhaustion: 800ms is still a perfectly nonzero remainder, and a - /// zero-floor gate would happily spend it spawning a child it cannot afford to reap. - #[cfg(unix)] + rollback(&mut lock).await; + drop(lock); + + assert!( + db.pin_sources_incomplete(&sha).await.unwrap(), + "a TIMED-OUT `record_pin_source` definitively did not land: it is an explicit \ + multi-statement transaction, and the cancelled future never reaches \ + `tx.commit()`, so no COMMIT is ever sent and the row cannot exist. The set is \ + therefore incomplete, and leaving it UNMARKED is the exact state the marker \ + exists to prevent: the resolver reads a non-empty below-cap set as complete \ + and 404s a copy this repo would serve" + ); + } + + /// Scenario 7: three objects, one budget. Every object's first DB call stalls on + /// the same lock, and the total must stay near ONE budget rather than one per + /// object. This is the only loop-level scenario where an absolute-deadline bound + /// and a per-call duration could differ; every single-object stall test above + /// passes under either. #[sqlx::test] - async fn pin_new_objects_stops_when_the_remainder_falls_below_the_read_floor( - pool: sqlx::PgPool, - ) { - let db = crate::db::Db::for_testing(pool); + async fn pin_new_objects_multi_object_stall_charges_one_budget(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool.clone()); db.run_migrations().await.expect("migrations"); let tmp = tempfile::TempDir::new().unwrap(); - let repo_path = tmp.path().join("floor.git"); + let repo_path = tmp.path().join("multi_stalled.git"); let oids = seed_loose_blobs(&repo_path, 3); - let log = tmp.path().join("calls.log"); - let git_bin = counting_git(tmp.path(), &log); - let endpoint = delaying_endpoint(vec![Duration::from_millis(700)]).await; + let endpoint = delaying_endpoint(vec![Duration::ZERO]).await; + // Install a log capture even though nothing here asserts on it: `tracing` + // caches a callsite's interest globally the first time it is hit, and a hit + // from a thread with no subscriber caches it as never-interested for the whole + // binary, which silently blinds the sibling tests that DO assert on the batch + // deadline warn. + let (_logs, _log_guard) = capture_logs(); - let (logs, _guard) = capture_logs(); + let mut lock = lock_table(&pool, "pinned_cids").await; + + let started = std::time::Instant::now(); let pinned = tokio::time::timeout( - Duration::from_secs(30), + Duration::from_secs(10), pin_new_objects( &endpoint, &repo_path, - &git_bin, + "git", + Duration::from_secs(30), oids, &db, + "repo-multi-stalled", Duration::from_millis(1500), ), ) .await - .expect("wedge guard: a 1.5s budget cannot take 30s"); + .expect("three stalled objects must still cost one budget, not three"); + let elapsed = started.elapsed(); - // The named property first, so a regression reddens on it rather than on a - // downstream count that a zero floor also happens to change. - assert_eq!( - objects_attempted(&log), - 1, - "a remainder below the read floor must stop the batch, not buy a bounded child \ - that can only be spawned and reaped" + assert!(pinned.is_empty(), "nothing can pin against a stalled DB"); + assert!( + elapsed < Duration::from_secs(3), + "three stalled objects must charge ONE budget (1.5s), not one each; got {elapsed:?}" ); - assert_eq!( - pinned.len(), - 1, - "the first object is inside the budget and must pin: {pinned:?}" + + rollback(&mut lock).await; + } + + /// Scenario 8, Kubo half of the durability floor. `batch_budget_gate` only + /// guarantees `PIN_READ_FLOOR` before an object STARTS and the add is handed the + /// whole remainder, so a successful add can finish with ~0 left. Without the + /// floor the post-add record would then be failed by a spent deadline, leaving + /// bytes in Kubo with no `pinned_cids` row and nothing able to resolve the CID. + /// + /// Fixture: a 2s budget, a 1.7s add, and `pinned_cids` locked from 500ms (well + /// after `is_pinned` has read it, and still well before the add returns) until + /// 2.4s. The record therefore starts at ~1.72s with ~280ms of budget left and + /// needs ~680ms of lock wait to land, which only the `DB_RECORD_GRACE` floor buys + /// it. + /// + /// The lock time is a MARGIN, not a boundary: taking it at 100ms left `is_pinned` + /// racing it on a loaded box, and losing that race makes the read block, time out, + /// and break the batch, which fails on `pinned.len() == 1` for a reason that has + /// nothing to do with the floor. Any time between the `is_pinned` round trip and + /// the add's 1.7s return proves the same thing. + #[sqlx::test] + async fn pin_add_with_spent_budget_still_records_row(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool.clone()); + db.run_migrations().await.expect("migrations"); + let tmp = tempfile::TempDir::new().unwrap(); + let repo_path = tmp.path().join("spent_budget.git"); + let oids = seed_loose_blobs(&repo_path, 1); + let sha = oids[0].clone(); + let endpoint = delaying_endpoint(vec![Duration::from_millis(1700)]).await; + // Install a log capture even though nothing here asserts on it: `tracing` + // caches a callsite's interest globally the first time it is hit, and a hit + // from a thread with no subscriber caches it as never-interested for the whole + // binary, which silently blinds the sibling tests that DO assert on the batch + // deadline warn. + let (_logs, _log_guard) = capture_logs(); + + let lock_pool = pool.clone(); + let locker = async move { + tokio::time::sleep(Duration::from_millis(500)).await; + let mut conn = lock_table(&lock_pool, "pinned_cids").await; + tokio::time::sleep(Duration::from_millis(1900)).await; + rollback(&mut conn).await; + }; + + let pin = tokio::time::timeout( + Duration::from_secs(15), + pin_new_objects( + &endpoint, + &repo_path, + "git", + Duration::from_secs(30), + oids.clone(), + &db, + "repo-spent-budget", + Duration::from_millis(2000), + ), + ); + let (pinned, ()) = tokio::join!(pin, locker); + let pinned = pinned.expect("the floored record must land well inside this wrap"); + + assert!( + db.is_pinned(&sha).await.unwrap(), + "a successful add whose batch deadline is spent must still land its \ + pinned_cids row: without the floor the bytes sit in Kubo with no row and \ + nothing can resolve the CID" ); - let text = logs.text(); assert_eq!( - text.lines() - .filter(|l| l.contains("pin batch deadline reached")) - .count(), + pinned.len(), 1, - "the truncation must be reported exactly once: {text}" + "the durably recorded pin must be returned: {pinned:?}" ); } - /// A store-wide fault breaks the batch instead of amplifying it. When the object - /// store itself cannot be read every remaining object fails identically, so - /// continuing would spawn one doomed bounded child per object and burn the budget - /// on reaping them. + /// Scenario 8, the other direction of the floor: the skip branch's DEFINITE-error + /// arm with the budget already spent must still write the incomplete marker. + /// Without it the source set is incomplete AND unmarked, which is exactly the + /// state the marker exists to prevent: the resolver reads a non-empty below-cap + /// set as complete and 404s an object this repo would serve. /// - /// The fixture looks wrong and is not: with the objects LOOSE and only - /// `objects/pack` unreadable, git still resolves each object, but it prints an - /// `error:` diagnostic that the probe routes to a fault before the present/missing - /// parse, so the read reaches `classify_store_fault` and (the store being - /// unreadable) returns `Transient`. - #[cfg(unix)] + /// The definite error is a dropped `pin_repo_sources`, not a timeout: the DROP + /// runs inside a transaction that commits at 1.5s, so the insert blocks on that + /// transaction's lock and then fails outright. With a 1.2s budget the retry + /// ladder therefore returns its definite error at ~1.6s, past the deadline, and + /// only the floor lets the marker write run at all. #[sqlx::test] - async fn pin_new_objects_breaks_the_batch_on_an_unreadable_store(pool: sqlx::PgPool) { - use std::os::unix::fs::PermissionsExt; - let db = crate::db::Db::for_testing(pool); + async fn pin_skip_branch_definite_error_with_spent_budget_marks_incomplete(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool.clone()); db.run_migrations().await.expect("migrations"); let tmp = tempfile::TempDir::new().unwrap(); - let repo_path = tmp.path().join("unreadable.git"); - let oids = seed_loose_blobs(&repo_path, 5); - let log = tmp.path().join("calls.log"); - let git_bin = counting_git(tmp.path(), &log); + let repo_path = tmp.path().join("definite_error.git"); + let oids = seed_loose_blobs(&repo_path, 1); + let sha = oids[0].clone(); + db.record_pinned_cid_with_source(&sha, &seed_cid(), "repo-seed") + .await + .unwrap(); let endpoint = delaying_endpoint(vec![Duration::ZERO]).await; + // Install a log capture even though nothing here asserts on it: `tracing` + // caches a callsite's interest globally the first time it is hit, and a hit + // from a thread with no subscriber caches it as never-interested for the whole + // binary, which silently blinds the sibling tests that DO assert on the batch + // deadline warn. + let (_logs, _log_guard) = capture_logs(); - let pack_dir = repo_path.join("objects").join("pack"); - let chmod = |mode: u32| { - let mut perms = std::fs::metadata(&pack_dir).unwrap().permissions(); - perms.set_mode(mode); - std::fs::set_permissions(&pack_dir, perms).unwrap(); + let mut dropper = pool.acquire().await.unwrap(); + sqlx::raw_sql("BEGIN; DROP TABLE pin_repo_sources;") + .execute(&mut *dropper) + .await + .unwrap(); + + let commit = async move { + tokio::time::sleep(Duration::from_millis(1500)).await; + sqlx::raw_sql("COMMIT") + .execute(&mut *dropper) + .await + .unwrap(); }; - chmod(0o000); - // Root bypasses permission bits, so witness the exact operation the probe - // performs and skip rather than falsely fail. - let genuinely_unreadable = std::fs::read_dir(&pack_dir).is_err(); - let pinned = tokio::time::timeout( - Duration::from_secs(30), + let pin = tokio::time::timeout( + Duration::from_secs(15), pin_new_objects( &endpoint, &repo_path, - &git_bin, - oids.clone(), + "git", + Duration::from_secs(30), + oids, &db, - Duration::from_secs(60), + "repo-definite-error", + Duration::from_millis(1200), ), - ) - .await - .expect("an immediately-faulting store cannot take 30s"); - let attempted = objects_attempted(&log); - chmod(0o755); // restore BEFORE any assertion that can panic, so TempDir cleans up + ); + let (pinned, ()) = tokio::join!(pin, commit); + pinned.expect("a definite DB error resolves inside the record floor, not the wrap"); - if genuinely_unreadable { - assert!( - pinned.is_empty(), - "nothing can be pinned through a store that cannot be read: {pinned:?}" - ); - assert_eq!( - attempted, 1, - "a store-wide fault must break the batch after the first object, not spawn \ - one doomed bounded child per object: {attempted} of 5 objects were read" - ); - } + assert!( + db.pin_sources_incomplete(&sha).await.unwrap(), + "the DEFINITE-error arm must still mark the source set incomplete with the \ + batch deadline spent: an incomplete-and-unmarked set is read as complete and \ + 404s an object this repo would serve" + ); } - /// The failure mode the store-wide re-check exists for. A `Transient` fault does not - /// prove the store is gone: the readability verdict is judged FOR one oid, so a single - /// unreadable `objects/` fan-out (1/256 of the store) taints only the objects that - /// live in it. Breaking there forfeits every remaining object over a fault that costs - /// at most a few of them, and permanently: the documented recovery re-derives the same - /// list and breaks at the same index. + /// The MARKER's own floor, which the two tests above leave unbound. /// - /// Exactly ONE fan-out dir is chmod'd, and the expected pin count is derived from the - /// oids that actually land in it rather than assumed to be one: two seeded blobs can - /// share a fan-out prefix, and hardcoding "four must pin" would make this test flap on - /// a collision instead of reporting it. - #[cfg(unix)] + /// Both of them assert the marker lands with the batch deadline spent, and both + /// pass with the marker's `db_record_deadline` replaced by the bare `deadline`. + /// The reason is timing, not coverage: the remainder there really is ~0, but + /// `tokio::time::timeout` polls the inner future before it checks the timer, and a + /// local Postgres UPDATE against an uncontended table round-trips inside that one + /// poll. So the unfloored write lands anyway and the floor is never load-bearing. + /// + /// This makes the marker write SLOW, so a zero bound cannot smuggle it through. + /// `mark_pin_sources_incomplete` is `UPDATE pinned_cids`, so `pinned_cids` is held + /// under `ACCESS EXCLUSIVE` from 300ms (after `is_pinned` and `provenance_for_oid` + /// have read it, both round trips inside the first few ms) until 3s. + /// + /// The schedule, with a 1.5s budget and `pin_repo_sources` locked for the whole + /// run so the source record stalls: + /// + /// - ~10ms: `record_pin_source` starts and blocks on the sources lock. Its own + /// floored bound is `now + 2s`, so it elapses at ~2.01s; + /// - ~2.01s: the elapsed arm runs the marker write, which blocks on the + /// `pinned_cids` lock. Floored, its bound is ~4.01s; unfloored it is the spent + /// 1.5s batch deadline, so the bound is ~0 and the blocked UPDATE is cancelled at + /// once, leaving no marker; + /// - ~3.0s: the lock lifts, a full second after the write started and a full second + /// before its floored bound expires, so the floored write lands. + /// + /// Both margins are a full second on purpose. The proof only needs the release to + /// fall strictly between zero and `DB_RECORD_GRACE`, so there is no reason to put + /// it near either end and make the test a race. #[sqlx::test] - async fn pin_new_objects_continues_past_one_unreadable_fanout_dir(pool: sqlx::PgPool) { - use std::os::unix::fs::PermissionsExt; - let db = crate::db::Db::for_testing(pool); + async fn pin_skip_branch_marker_write_needs_the_record_floor(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool.clone()); db.run_migrations().await.expect("migrations"); let tmp = tempfile::TempDir::new().unwrap(); - let repo_path = tmp.path().join("one_bad_fanout.git"); - let oids = seed_loose_blobs(&repo_path, 5); - let log = tmp.path().join("calls.log"); - let git_bin = counting_git(tmp.path(), &log); + let repo_path = tmp.path().join("marker_floor.git"); + let oids = seed_loose_blobs(&repo_path, 1); + let sha = oids[0].clone(); + db.record_pinned_cid_with_source(&sha, &seed_cid(), "repo-seed") + .await + .unwrap(); let endpoint = delaying_endpoint(vec![Duration::ZERO]).await; + // Asserted on here, and installed for the sibling reason too: `tracing` caches + // a callsite's interest globally on first hit, so a hit from a thread with no + // subscriber caches it as never-interested for the whole binary. + let (logs, _log_guard) = capture_logs(); - let prefix = oids[0][0..2].to_string(); - let tainted: Vec = oids.iter().filter(|o| o[0..2] == prefix).cloned().collect(); - assert!( - tainted.len() < oids.len(), - "the fixture must leave healthy objects outside the tainted fan-out, or this \ - test cannot tell an object-scoped fault from a store-wide one" - ); + let mut sources_lock = lock_table(&pool, "pin_repo_sources").await; - let fanout = repo_path.join("objects").join(&prefix); - let chmod = |mode: u32| { - let mut perms = std::fs::metadata(&fanout).unwrap().permissions(); - perms.set_mode(mode); - std::fs::set_permissions(&fanout, perms).unwrap(); + let lock_pool = pool.clone(); + let controller = async { + tokio::time::sleep(Duration::from_millis(300)).await; + let mut cids_lock = lock_table(&lock_pool, "pinned_cids").await; + tokio::time::sleep(Duration::from_millis(2700)).await; + rollback(&mut cids_lock).await; }; - chmod(0o000); - // Root bypasses permission bits, so witness the exact operation the probe performs - // (an open of this oid's loose path) and skip rather than falsely fail. - let genuinely_unreadable = std::fs::File::open(fanout.join(&oids[0][2..])).is_err(); - let pinned = tokio::time::timeout( - Duration::from_secs(30), + let started = std::time::Instant::now(); + let pin = tokio::time::timeout( + Duration::from_secs(20), pin_new_objects( &endpoint, &repo_path, - &git_bin, - oids.clone(), + "git", + Duration::from_secs(30), + oids, &db, - Duration::from_secs(60), + "repo-marker-floor", + Duration::from_millis(1500), ), - ) - .await - .expect("an immediate endpoint and four healthy objects cannot take 30s"); - let attempted = objects_attempted(&log); - chmod(0o755); // restore BEFORE any assertion that can panic, so TempDir cleans up + ); + let (pinned, ()) = tokio::join!(pin, controller); + let pinned = pinned.expect("the floored marker write must land well inside this wrap"); + let elapsed = started.elapsed(); - if genuinely_unreadable { - assert_eq!( - attempted, - oids.len(), - "one unreadable fan-out is 1/256 of the store, not the store: every object \ - must still be attempted, got {attempted} of {}", - oids.len() - ); - let pinned_shas: Vec<&String> = pinned.iter().map(|(sha, _)| sha).collect(); - let expected: Vec<&String> = oids.iter().filter(|o| !tainted.contains(o)).collect(); - assert_eq!( - pinned_shas, expected, - "every object outside the tainted fan-out must pin, and only those" - ); - } + rollback(&mut sources_lock).await; + drop(sources_lock); + + assert!( + pinned.is_empty(), + "an already-pinned object is skipped, never re-pinned: {pinned:?}" + ); + assert!( + logs.text() + .contains("did not complete inside the batch deadline"), + "the fixture only proves anything if the marker was reached from the ELAPSED \ + arm of the source record, not from the definite-error arm: {}", + logs.text() + ); + assert!( + elapsed < Duration::from_secs(8), + "the call must end at one budget plus the one chained record grace the \ + blocked marker write costs (~3s), never at the lock's lifetime; got \ + {elapsed:?}" + ); + assert!( + db.pin_sources_incomplete(&sha).await.unwrap(), + "the marker write must be given DB_RECORD_GRACE, not the spent batch \ + deadline: it starts here with ~0 of the budget left and needs ~1s to get \ + past the lock, so an unfloored bound cancels it and the source set is left \ + incomplete AND unmarked, which the resolver reads as complete and 404s a \ + copy this repo would serve" + ); } - /// The must-not direction of the arm above: an object-scoped fault must NOT break - /// the batch. One corrupt loose object among healthy ones is a `Deterministic` - /// fault (the store is readable, git still fails), and the documented recovery path - /// cannot repair it: a later full-scan push re-offers the same object and would - /// break at the same place, so breaking here stops the repo pinning permanently. + /// The TRANSITIVE site. `repair_legacy_provider_cid` runs on the skip branch under + /// the same permit, and its own `deadline` argument used to bound only the + /// `spawn_blocking` git read: the two DB awaits inside it (`cid_for_oid` and the + /// key rewrite) were bare, so a stall there parked the permit exactly the way the + /// loop-body awaits did. A grep over the loop bodies cannot see this site, which + /// is why it is driven here rather than argued. /// - /// Deliberately not the bad-config corruption, which is repo-wide: all five objects - /// would fault and the test would pin the store-wide case rather than the - /// object-scoped one this arm rests on. - #[cfg(unix)] + /// Fixture, ordered so the stall lands on `cid_for_oid` and nothing earlier: + /// `pin_repo_sources` is locked from the start so the skip branch's source record + /// blocks; at 1.5s `pinned_cids` is locked (nothing is reading it by then) and at + /// 1.6s the first lock is released. + /// + /// What the loop actually does with that, since the timing is easy to misread: when + /// the `pin_repo_sources` lock lifts at 1.6s the insert succeeds, and then + /// `record_pin_source`'s follow-up `UPDATE pinned_cids` immediately blocks on the + /// `pinned_cids` lock taken at 1.5s and eats the rest of the budget, elapsing at + /// 2.2s. The timeout arm then writes the incomplete marker, another `UPDATE + /// pinned_cids`, which blocks on the same lock and elapses against its own floor at + /// ~4.2s. So the repair is reached with its deadline long SPENT, not with ~600ms + /// left, and ~4.2s is the fixture's expected total: one budget plus one chained + /// record grace, which is the `db_record_deadline` re-flooring described on + /// `pin_new_objects`. + /// + /// The test is load-bearing either way, and the 10s wrap is what makes it so: the + /// `pinned_cids` lock is held until after the call returns, so an unbounded + /// `cid_for_oid` inside the repair hangs past the wrap instead of returning here. #[sqlx::test] - async fn pin_new_objects_continues_past_a_deterministic_fault(pool: sqlx::PgPool) { - use std::os::unix::fs::PermissionsExt; - let db = crate::db::Db::for_testing(pool); + async fn pin_new_objects_stalled_legacy_repair_lookup_returns_by_budget(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool.clone()); db.run_migrations().await.expect("migrations"); let tmp = tempfile::TempDir::new().unwrap(); - let repo_path = tmp.path().join("corrupt.git"); - let oids = seed_loose_blobs(&repo_path, 5); - let log = tmp.path().join("calls.log"); - let git_bin = counting_git(tmp.path(), &log); + let repo_path = tmp.path().join("repair_stalled.git"); + let oids = seed_loose_blobs(&repo_path, 1); + let sha = oids[0].clone(); + db.record_pinned_cid_with_source(&sha, &seed_cid(), "repo-seed") + .await + .unwrap(); let endpoint = delaying_endpoint(vec![Duration::ZERO]).await; + // Install a log capture even though nothing here asserts on it: `tracing` + // caches a callsite's interest globally the first time it is hit, and a hit + // from a thread with no subscriber caches it as never-interested for the whole + // binary, which silently blinds the sibling tests that DO assert on the batch + // deadline warn. + let (_logs, _log_guard) = capture_logs(); - // Overwrite exactly one loose object with non-zlib garbage (0o444 by default). - let victim = repo_path - .join("objects") - .join(&oids[0][0..2]) - .join(&oids[0][2..]); - assert!(victim.is_file(), "fixture must leave the blob loose"); - let mut perms = std::fs::metadata(&victim).unwrap().permissions(); - perms.set_mode(0o644); - std::fs::set_permissions(&victim, perms).unwrap(); - std::fs::write(&victim, b"garbage not a zlib stream").unwrap(); + let mut sources_lock = lock_table(&pool, "pin_repo_sources").await; - let pinned = tokio::time::timeout( - Duration::from_secs(30), + let lock_pool = pool.clone(); + let controller = async { + tokio::time::sleep(Duration::from_millis(1500)).await; + let cids_lock = lock_table(&lock_pool, "pinned_cids").await; + tokio::time::sleep(Duration::from_millis(100)).await; + rollback(&mut sources_lock).await; + cids_lock + }; + + let started = std::time::Instant::now(); + let pin = tokio::time::timeout( + Duration::from_secs(10), pin_new_objects( &endpoint, &repo_path, - &git_bin, - oids.clone(), + "git", + Duration::from_secs(30), + oids, &db, - Duration::from_secs(60), + "repo-repair-stalled", + Duration::from_millis(2200), ), - ) - .await - .expect("an immediate endpoint and four healthy objects cannot take 30s"); - - assert_eq!( - objects_attempted(&log), - 5, - "an object-scoped fault must not stop the batch: every object must be read" ); - assert_eq!( - pinned.len(), - 4, - "one corrupt object must cost only itself: the other four must still pin" + let (pinned, mut cids_lock) = tokio::join!(pin, controller); + pinned.expect( + "the repair's own DB lookup must be bounded by the batch deadline: the bare \ + await hangs past this wrap", + ); + let elapsed = started.elapsed(); + + assert!( + elapsed < Duration::from_secs(6), + "a stall inside repair_legacy_provider_cid must end the call at ~budget \ + (2.2s) plus the one chained record grace the marker write costs against the \ + same lock (~4.2s), never at the lock's lifetime; got {elapsed:?}" ); + + rollback(&mut cids_lock).await; } } diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 10aaca5b..66bfa096 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -59,6 +59,32 @@ struct DbStartupStatus { next_retry_secs: AtomicU64, } +/// Hard ceiling on the advisory-lock pool's `max_connections`. +/// +/// `max_concurrent_git_pushes` is validated all the way up to 1_048_576, and the lock +/// pool used to derive its size straight from that knob, so raising the push cap +/// silently raised the node's Postgres connection ceiling with no CLI error and no +/// relation to the server's own `max_connections` (#173 F4). The node's total budget is +/// now bounded: `db_max_connections` (default 48) + at most this. +const LOCK_POOL_MAX_CONNECTIONS: u32 = 64; + +/// Connections the lock pool keeps above the push cap. Covers the three non-push +/// `acquire_write` callers (`api/issues.rs` x2, `api/pulls.rs`), which hold no +/// concurrency permit, so a push never queues here for a connection where it did not +/// before. +const LOCK_POOL_PUSH_HEADROOM: u8 = 8; + +/// Size the advisory-lock pool for a given push cap: the cap plus +/// [`LOCK_POOL_PUSH_HEADROOM`], clamped to [`LOCK_POOL_MAX_CONNECTIONS`]. Past the +/// clamp a push may wait for a lock-pool connection, which is a bounded wait that sheds +/// a clean 503 (see `LockPoolBusy`), not an unbounded hang. +fn lock_pool_size(max_concurrent_git_pushes: usize) -> u32 { + u32::try_from(max_concurrent_git_pushes) + .unwrap_or(u32::MAX) + .saturating_add(u32::from(LOCK_POOL_PUSH_HEADROOM)) + .min(LOCK_POOL_MAX_CONNECTIONS) +} + #[tokio::main] async fn main() -> Result<()> { tracing_subscriber::fmt() @@ -90,6 +116,9 @@ async fn main() -> Result<()> { // Load or generate the node's identity keypair let keypair = load_or_create_keypair(&config)?; + // Sealing key for the legacy-scan continuation tokens, DERIVED from the identity + // just loaded so it is the same key after a restart (see `derive_scan_token_key`). + let scan_token_key = AppState::derive_scan_token_key(&keypair); let node_did = keypair.did(); // One-time metrics init. Must run before any handler that calls into @@ -287,8 +316,20 @@ async fn main() -> Result<()> { None }; - let repo_store = - git::repo_store::RepoStore::new(config.repos_dir.clone(), tigris, db.pool().clone()); + // Repo write locks run on their own pool, never the main query pool: each push + // holds its connection for the whole receive-pack, so a burst of concurrent + // pushes drawing from the main pool would park that many connections for the + // duration of their receive-packs and starve every other query. That holds + // whatever the two pools are sized at, which is why the separation is + // structural rather than a consequence of the defaults; config validate() + // separately requires db_max_connections >= max_concurrent_git_pushes + 8. See + // build_lock_pool for the cancellation semantics (#173). + let lock_pool = git::repo_store::build_lock_pool( + db.pool(), + lock_pool_size(config.max_concurrent_git_pushes), + std::time::Duration::from_secs(config.db_acquire_timeout_secs), + ); + let repo_store = git::repo_store::RepoStore::new(config.repos_dir.clone(), tigris, lock_pool); // Per-DID limiter for the creation endpoints. Keyed on the authenticated // DID (attacker-varied), so bound its key set to cap memory. @@ -382,6 +423,22 @@ async fn main() -> Result<()> { rate_limiter, create_ip_rate_limiter, push_rate_limiter, + ipfs_max_history_walks: crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, + // The legacy-probe budget is operator-tunable via GITLAWB_IPFS_MAX_LEGACY_PROBES + // (R5), which is what `ipfs_legacy_probe_budget` reads. NOT + // GITLAWB_IPFS_MAX_REPOS_WALKED, which this comment used to name: that is the + // separate cap on expensive visibility walks, so an operator following the old + // text tuned the walk cap and left this fan-out unchanged. The history-walk + // ceiling above stays constant (a smaller value false-503s a provenanced + // request). Default 256 preserves the shipped behaviour. + ipfs_max_legacy_probes: AppState::ipfs_legacy_probe_budget(&config), + ipfs_legacy_scan_page_rows: crate::api::ipfs::LEGACY_SCAN_PAGE_ROWS, + // Operator-tunable via GITLAWB_IPFS_MAX_LEGACY_SCAN_ROWS, read through the same + // helper shape as the probe budget so the knob cannot be a silent no-op. + ipfs_max_legacy_scan_rows: AppState::ipfs_legacy_scan_row_budget(&config), + ipfs_max_legacy_scan_rule_bytes: crate::api::ipfs::MAX_LEGACY_SCAN_RULE_BYTES_PER_REQUEST, + ipfs_scan_token_key: Arc::new(scan_token_key), + ipfs_max_served_object_bytes: crate::api::ipfs::MAX_SERVED_OBJECT_BYTES, push_limiter_trust, sync_trigger_rate_limiter, peer_write_rate_limiter, @@ -456,6 +513,15 @@ async fn main() -> Result<()> { std::time::Duration::from_secs(3600), 200_000, ), + // Separate WORK-budget bucket for the resolver's per-probe/per-walk charges (R6). + // Its capacity is DERIVED from the route limit (no new knob) and floored at the + // legacy-probe budget, so one full default-config legacy scan never self-throttles + // mid-request while the route brake above stays the pure once-per-request cap. + ipfs_work_rate_limiter: rate_limit::RateLimiter::new_bounded( + AppState::ipfs_work_budget(&config), + std::time::Duration::from_secs(3600), + 200_000, + ), git_bin: "git".to_string(), }; if config.ipfs_rate_limit == 0 { @@ -490,14 +556,16 @@ async fn main() -> Result<()> { // Periodic cleanup of expired rate limit entries + consumed-proof ledger { - let sweep_state = state.clone(); + let cleanup_state = state.clone(); let db = state.db.clone(); let mut shutdown_rx = state.subscribe_shutdown(); tokio::spawn(async move { loop { tokio::select! { _ = tokio::time::sleep(std::time::Duration::from_secs(300)) => { - sweep_rate_limiters(&sweep_state).await; + // Sweep every per-IP/DID limiter (incl. the ipfs walk brake) + // so bounded maps shed stale keys instead of sitting at cap. + cleanup_state.sweep_rate_limiters().await; let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map(|d| d.as_secs() as i64) @@ -516,6 +584,8 @@ async fn main() -> Result<()> { }); } + let _legacy_cid_sweep = spawn_legacy_cid_sweep(&state, &config); + let router = server::build_router(state.clone()); // Re-register the socket bound at startup — same fd, so there was never a // moment with the port closed between the degraded and full servers. @@ -638,6 +708,44 @@ async fn main() -> Result<()> { Ok(()) } +/// U4 (#173): spawn the periodic legacy provider-CID repair sweep. Releases before this +/// version stored the PROVIDER CID (Kubo dag-pb / Pinata CIDv0) in `pinned_cids.cid`, +/// and this version's `/ipfs/{cid}` resolver withholds any row whose stored key is not +/// the raw-content CID. The opportunistic repair on the pin path only fires when a push +/// re-carries the object, which normal git negotiation makes it not do, so those rows +/// need a walk. DETACHED, never on the boot path: the caller keeps serving while this +/// runs, and the sweep's own batch bound plus inter-batch delay keep it off the DB's +/// critical path. Its cursor is durable, so a restart mid-walk resumes instead of +/// rewinding. It re-arms after every run, including a failed one, so it never returns +/// and there is no awaited value to log here; the shutdown watcher below is what ends +/// it. +/// +/// A named function rather than an inline block in `main` so the WIRING has a seam a +/// test can call: that the task is spawned at all, that it reads its batch and delay +/// from the config knobs rather than some other field, that the caller is not blocked +/// on it, and that the shutdown watcher actually ends it mid-walk. The sweep's own +/// behavior is covered elsewhere; this is the boot-path half. +fn spawn_legacy_cid_sweep(state: &AppState, config: &Config) -> tokio::task::JoinHandle<()> { + let db = state.db.clone(); + let repos_dir = config.repos_dir.clone(); + let git_bin = state.git_bin.clone(); + let git_timeout = std::time::Duration::from_secs(config.git_service_timeout_secs); + let batch = config.pin_repair_sweep_batch; + let delay = std::time::Duration::from_secs(config.pin_repair_sweep_delay_secs); + let mut shutdown_rx = state.subscribe_shutdown(); + tokio::spawn(async move { + tokio::select! { + _ = ipfs_pin::run_sweep_rearmed( + &repos_dir, &git_bin, git_timeout, batch, delay, + ipfs_pin::SWEEP_REARM_DELAY, &db, + ) => {} + // Shutdown mid-walk simply drops the run; the persisted cursor means the + // next boot picks up where this one stopped. + _ = shutdown_rx.changed() => {} + } + }) +} + fn spawn_shutdown_signal(tx: watch::Sender) { tokio::spawn(async move { #[cfg(unix)] @@ -1082,6 +1190,7 @@ mod rate_limiter_sweep_tests { state.sync_trigger_rate_limiter = RateLimiter::new(10, window); state.peer_write_rate_limiter = RateLimiter::new(10, window); state.ipfs_rate_limiter = RateLimiter::new(10, window); + state.ipfs_work_rate_limiter = RateLimiter::new(10, window); let limiters = |s: &crate::state::AppState| { [ @@ -1091,6 +1200,7 @@ mod rate_limiter_sweep_tests { s.sync_trigger_rate_limiter.clone(), s.peer_write_rate_limiter.clone(), s.ipfs_rate_limiter.clone(), + s.ipfs_work_rate_limiter.clone(), ] }; for l in limiters(&state) { @@ -1099,7 +1209,7 @@ mod rate_limiter_sweep_tests { } tokio::time::sleep(window * 3).await; - super::sweep_rate_limiters(&state).await; + state.sweep_rate_limiters().await; for (i, l) in limiters(&state).into_iter().enumerate() { assert_eq!(l.tracked_keys().await, 0, "limiter {i} was not swept"); @@ -1107,21 +1217,6 @@ mod rate_limiter_sweep_tests { } } -/// Evict expired entries from every per-key rate limiter on the state. -/// -/// Named and driven off `AppState` so the periodic sweeper stays in step with -/// the limiters the router actually mounts: adding a limiter field and -/// forgetting it here leaves its keys pinned until the map hits `max_keys` and -/// the inline capacity sweep runs (the `/ipfs` limiter was missed this way). -async fn sweep_rate_limiters(state: &AppState) { - state.rate_limiter.cleanup().await; - state.create_ip_rate_limiter.cleanup().await; - state.push_rate_limiter.cleanup().await; - state.sync_trigger_rate_limiter.cleanup().await; - state.peer_write_rate_limiter.cleanup().await; - state.ipfs_rate_limiter.cleanup().await; -} - async fn gossip_ping_round( db: &Db, client: &reqwest::Client, @@ -1298,6 +1393,140 @@ fn load_or_create_keypair(config: &Config) -> Result { } } +#[cfg(test)] +mod legacy_cid_sweep_wiring_tests { + use super::spawn_legacy_cid_sweep; + use sqlx::PgPool; + use std::time::Duration; + + /// Seed `count` `pinned_cids` rows whose keys are already canonical raw CIDv1, in a + /// known `sha256_hex` order. The sweep's own cost gate skips a raw-CIDv1 row without + /// reading bytes or resolving a repo, so each row is SCANNED (it advances the cursor) + /// and nothing else. That is what makes the cursor a clean readout of how far the + /// walk got, with no dependency on repos on disk. + async fn seed_scannable_rows(pool: &PgPool, count: usize) -> Vec { + let mut shas = Vec::new(); + for i in 1..=count { + let sha = format!("wire{i:02}"); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(sha.as_bytes()).to_string(); + assert!( + gitlawb_core::cid::is_raw_cidv1(&cid), + "the seeded key must hit the sweep's raw-CIDv1 skip, not a repair attempt" + ); + sqlx::query("INSERT INTO pinned_cids (sha256_hex, cid, pinned_at) VALUES ($1, $2, $3)") + .bind(&sha) + .bind(&cid) + .bind("2020-01-01T00:00:00Z") + .execute(pool) + .await + .unwrap(); + shas.push(sha); + } + shas + } + + /// Poll the persisted sweep cursor until it reaches `want`, or give up. + async fn cursor_reaches(db: &crate::db::Db, want: &str, within: Duration) -> String { + let deadline = std::time::Instant::now() + within; + loop { + let c = db.pin_repair_cursor().await.unwrap(); + if c == want || std::time::Instant::now() >= deadline { + return c; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + } + + /// #173 U4, the BOOT-PATH half. The sweep's own logic (batching, cursor resumption, + /// terminal vs retryable skips) is covered in `test_support`; what this covers is the + /// wiring `main` performs, which nothing else executes: the task is spawned at all, + /// it takes its batch and delay from the two `pin_repair_sweep_*` knobs rather than + /// some other config field, the caller is not blocked on the walk, and the shutdown + /// watcher ends the run mid-walk. + /// + /// Six scannable rows, batch 2, delay 30s. One pass must land the cursor on exactly + /// the second row and the task must then still be alive in its inter-batch sleep, + /// which pins both knobs at once: a different batch stops at a different row, and a + /// delay that did not come from the knob either finishes the table or leaves the task + /// gone. Shutdown must then end it while four rows are still unwalked. + #[sqlx::test] + async fn the_boot_path_spawns_the_sweep_detached_with_its_configured_knobs(pool: PgPool) { + let state = crate::test_support::test_state(pool.clone()).await; + let shas = seed_scannable_rows(&pool, 6).await; + let repos_dir = tempfile::TempDir::new().unwrap(); + + let mut config = (*state.config).clone(); + config.repos_dir = repos_dir.path().to_path_buf(); + config.pin_repair_sweep_batch = 2; + // Far longer than this test runs, so a task still alive after the first pass can + // only be one that is honoring the configured inter-batch delay. + config.pin_repair_sweep_delay_secs = 30; + + let started = std::time::Instant::now(); + let handle = spawn_legacy_cid_sweep(&state, &config); + let spawn_cost = started.elapsed(); + + let cursor = cursor_reaches(&state.db, &shas[1], Duration::from_secs(10)).await; + assert_eq!( + cursor, shas[1], + "the spawned sweep must run and stop its first pass at the CONFIGURED batch \ + bound (2), leaving the cursor on the second row" + ); + assert!( + spawn_cost < Duration::from_secs(1), + "the sweep must be detached, not awaited on the boot path; the spawn took \ + {spawn_cost:?}" + ); + assert!( + !handle.is_finished(), + "with a 30s inter-batch delay the task must still be sleeping between passes, \ + not finished: a finished task means the delay was not the configured one" + ); + + state.shutdown(); + tokio::time::timeout(Duration::from_secs(10), handle) + .await + .expect("the shutdown watcher must end the sweep, and not after its 30s delay") + .expect("the sweep task must not panic"); + + assert_eq!( + state.db.pin_repair_cursor().await.unwrap(), + shas[1], + "shutdown must have ended the run MID-walk, with the remaining rows unwalked" + ); + } +} + +#[cfg(test)] +mod lock_pool_sizing_tests { + use super::{lock_pool_size, LOCK_POOL_MAX_CONNECTIONS, LOCK_POOL_PUSH_HEADROOM}; + + /// The default push cap gets its cap plus headroom, so no push ever queues for a + /// lock-pool connection where it did not before. + #[test] + fn default_push_cap_gets_headroom_over_the_cap() { + assert_eq!(lock_pool_size(32), 32 + u32::from(LOCK_POOL_PUSH_HEADROOM)); + assert_eq!(lock_pool_size(1), 1 + u32::from(LOCK_POOL_PUSH_HEADROOM)); + } + + /// #173 F4: `max_concurrent_git_pushes` is validated all the way to 1_048_576, so an + /// operator raising it used to raise the node's Postgres connection ceiling with it, + /// silently and without bound. The lock pool is CLAMPED instead. + #[test] + fn an_oversized_push_cap_is_clamped_not_propagated() { + assert_eq!(lock_pool_size(1_048_576), LOCK_POOL_MAX_CONNECTIONS); + assert_eq!(lock_pool_size(usize::MAX), LOCK_POOL_MAX_CONNECTIONS); + // The largest cap that still fits under the clamp keeps its full headroom. + let widest = (LOCK_POOL_MAX_CONNECTIONS - u32::from(LOCK_POOL_PUSH_HEADROOM)) as usize; + assert_eq!(lock_pool_size(widest), LOCK_POOL_MAX_CONNECTIONS); + assert_eq!( + lock_pool_size(widest - 1), + LOCK_POOL_MAX_CONNECTIONS - 1, + "values below the clamp must not be rounded up to it" + ); + } +} + #[cfg(test)] mod gossip_ssrf_tests { use super::{ diff --git a/crates/gitlawb-node/src/pinata.rs b/crates/gitlawb-node/src/pinata.rs index a3077191..14f1d582 100644 --- a/crates/gitlawb-node/src/pinata.rs +++ b/crates/gitlawb-node/src/pinata.rs @@ -76,42 +76,87 @@ pub async fn pin_object( /// still needed to read each object's bytes, and `git_bin` names the binary those /// reads run: the production caller passes the literal `"git"`, and a test passes a /// fake so the loop's own bound can be driven with a git that never answers. -/// Objects already recorded with a `pinata_cid` are skipped. Returns -/// `(sha_hex, cid)` pairs for each newly pinned object. +/// `git_timeout` is the per-object read bound, the same value and the same role it has +/// in the twin: it bounds both the pin read and the skip branch's opportunistic repair. +/// Objects already recorded with a `pinata_cid` are skipped, and `repo_id` records the +/// pin's provenance (#173). Returns `(sha_hex, provider_cid)` pairs for each newly +/// pinned object: the provider CID is the Pinata gateway CID (used for branch->CID +/// recording and ref-update gossip), NOT the raw resolver-key CID stored in +/// `pinned_cids.cid`. /// /// # What `batch_budget` does and does not bound /// /// The loop runs under a `pin_semaphore` permit and that pool defers rather than /// sheds, so the hold has to be bounded by something other than the pusher's object -/// count. Two things here are: +/// count. Three things here are: /// /// - this loop's own wall-clock: the deadline is taken once at loop start and /// checked at the top of every iteration, so no object's work begins with less /// than the read floor left. It is a gate, not a hard ceiling, since a started /// iteration still runs to completion; /// - the git read: `store::read_object_bounded` runs under `spawn_blocking` against the -/// ABSOLUTE batch deadline (not the loop-top remainder, which the `has_pinata_cid` -/// round-trip sitting between the two would push past it), with SIGTERM-then-SIGKILL -/// process-group teardown, so a hung `git cat-file` costs this batch its remaining -/// budget plus one watchdog teardown instead of holding the permit for the child's -/// whole lifetime and blocking a runtime worker while it does. +/// earlier of the ABSOLUTE batch deadline (not the loop-top remainder, which the +/// `has_pinata_cid` round-trip sitting between the two would push past it) and this +/// object's own `git_timeout`, with SIGTERM-then-SIGKILL process-group teardown, so a +/// hung `git cat-file` costs this batch one `git_timeout` plus one watchdog teardown +/// instead of holding the permit for the child's whole lifetime and blocking a runtime +/// worker while it does; +/// - the DB round-trips: every DB operation reachable from inside the region is +/// bounded by the same absolute deadline through `crate::ipfs_pin::db_bounded`, +/// including the two inside `repair_legacy_provider_cid`, which this loop body's own +/// call sites do not show. `retry_db_record` is wrapped as a whole so its ladder +/// cannot multiply one remainder, and the durability writes (the post-upload +/// `record_pinata_cid` and source record, the skip branch's source record, and both +/// incomplete markers) take the floored remainder `max(remaining, DB_RECORD_GRACE)` +/// so a spent budget delays the permit release rather than dropping a write. That +/// floor matters more here than on the twin: `pinned.push` is unconditional, so a +/// dropped record would leave `api::repos` advertising a CID the resolver 404s. A +/// bound is not a rollback, and what an elapsed bound MEANS is a property of the +/// operation, so each site maps that arm from the shape of the call it wrapped. Both +/// source-record sites wrap `record_pin_source`, an explicit transaction whose +/// `tx.commit()` a cancelled future never reaches, so a timeout there definitely did +/// not land and both write the incomplete marker exactly as the definite-error arm +/// does. `record_pinata_cid` is a single autocommit upsert, so ITS timeout is a +/// genuine unknown outcome and is never treated as a failed write. See +/// `crate::ipfs_pin::BoundedDbError::Elapsed`. /// /// So the LOOP's hold is bounded by roughly `batch_budget`, plus one watchdog -/// teardown and one upload (the shared client's whole-request timeout bounds the -/// upload; `pin_object` takes no per-request override). The PERMIT's hold is NOT -/// bounded by any of this: `api::repos` acquires the permit and then re-derives the -/// object list with `pinata_object_list_for_refs` BEFORE this function is entered, -/// and that walk carries no aggregate deadline. The DB round-trips -/// (`has_pinata_cid`, `record_pinata_cid`) are untimed inside the budgeted region -/// too. +/// teardown, one upload (the shared client's whole-request timeout bounds the +/// upload; `pin_object` takes no per-request override), and the record graces one +/// iteration can chain. `db_record_deadline` re-floors from `Instant::now()` at EVERY +/// call, so the graces inside a single iteration add up rather than sharing one floor: +/// the worst case here is the add path at `deadline + 6s` (`record_pinata_cid`, then +/// `record_pin_source`, then its incomplete marker), against the skip branch's +/// `deadline + 4s`. It does NOT stack per object, because the next iteration's first +/// statement is `batch_budget_gate`, which breaks the batch, so the overrun is one +/// iteration's worth however many objects the push carried. Against the 120s +/// `PIN_BATCH_BUDGET` that is roughly a 5% overrun for the batch, not an unbounded +/// hold. The PERMIT's hold is NOT bounded by any of this, and that stays out of +/// scope: `api::repos` acquires the permit (repos.rs ~2688) and only then re-derives +/// the object list with `pinata_object_list_for_refs` (~2697), BEFORE this function is +/// entered, and that walk carries no aggregate deadline. What is bounded is this +/// loop's own hold, not the permit's total hold and not the semaphore's worst-case +/// queue. /// -/// The twin in `ipfs_pin.rs` shares the budget gate and the bounded read with this -/// loop, so both are at parity again. Change them in lockstep: the skip-if-pinned -/// check, the fault arms, the returned pairs, and the budget handling. -// Eight arguments, one over clippy's threshold: the two the budget adds (`git_bin`, -// `batch_budget`) are what put the read under test injection and under a deadline, and -// grouping them into a struct would only move the same values behind a name the twin in -// `ipfs_pin.rs` does not use. Same allow as the sibling call sites in `api::repos`. +/// The twin in `ipfs_pin.rs` is at parity with this loop on everything that bounds or +/// repairs an object: the shared budget gate, the read bounded by the earlier of the +/// batch deadline and `git_timeout`, the skip branch's opportunistic legacy +/// provider-CID repair, and the DB bound above, which is the SAME helper and the same +/// floor on both sides rather than a copy. Change them in lockstep: the +/// skip-if-pinned check, the provenance and source recording, the fault arms, and the +/// budget handling. +/// +/// The RETURNED PAIRS are the one deliberate divergence, and it is not drift. This side +/// pushes a pin whose DB record exhausted its retries, because this return is a real +/// input: `api::repos` builds the sha-to-cid `cid_map` from it, which drives +/// `upsert_branch_cid` and the p2p `publish_ref_update` gossip CID. The twin's return is +/// log-only, so it omits a record-failed pin rather than logging a pin the resolver +/// cannot serve. Moving this side to match would need that consumer moved first. +// Ten arguments, over clippy's threshold: the three the budget and the git seam add +// (`git_bin`, `git_timeout`, `batch_budget`) plus #173's `repo_id` are what put the read +// under test injection and under a deadline, and grouping them into a struct would only +// move the same values behind a name the twin in `ipfs_pin.rs` does not use. Same allow +// as the sibling call sites in `api::repos`. #[allow(clippy::too_many_arguments)] pub async fn pin_new_objects( client: &reqwest::Client, @@ -119,8 +164,10 @@ pub async fn pin_new_objects( jwt: &str, repo_path: &std::path::Path, git_bin: &str, + git_timeout: Duration, object_list: Vec, db: &crate::db::Db, + repo_id: &str, batch_budget: Duration, ) -> Vec<(String, String)> { if jwt.is_empty() { @@ -144,8 +191,124 @@ pub async fn pin_new_objects( break; } - match db.has_pinata_cid(&sha).await { - Ok(true) => continue, + // Every DB call from here to the end of the iteration is bounded by the + // ABSOLUTE batch deadline (F3, #173), through the same `db_bounded` helper the + // ipfs_pin twin routes through: this loop runs under the same global pin permit + // and a bare await parked it for the whole stall. The elapsed arm is mapped per + // site below, never as a blanket "existing error arm": a timeout cancels the + // client future but not the statement Postgres is running, so it reports an + // UNKNOWN outcome, not a failed write. + match crate::ipfs_pin::db_bounded(deadline, db.has_pinata_cid(&sha)).await { + Ok(true) => { + // Backfill NULL first-pinner provenance from a known source, in lockstep + // with the ipfs_pin skip branch: a pinata-only node otherwise leaves + // pre-provenance rows' `pinned_cids.repo_id` NULL forever (grok P2-D). The + // resolver still finds the object via the pin_repo_sources union below, so + // this is a consistency backfill, not a correctness fix. + // + // Elapsed here is free to skip: the read costs nothing when it lands late, + // and the backfill's own `AND repo_id IS NULL` guard makes a late-landing + // write idempotent. + match crate::ipfs_pin::db_bounded(deadline, db.provenance_for_oid(&sha)).await { + Ok(None) => { + if let Err(e) = crate::ipfs_pin::db_bounded( + deadline, + db.backfill_pin_provenance(&sha, repo_id), + ) + .await + { + tracing::warn!(sha = %sha, err = %e, "failed to backfill pin provenance"); + } + } + Ok(Some(_)) => {} + Err(e) => { + tracing::warn!(sha = %sha, err = %e, "DB error reading pin provenance"); + } + } + // F1 (#173 round 8): record this repo as an additional source for the + // already-pinned object (mirrors the ipfs_pin skip-branch insert) so the + // resolver can serve a shared object from any pin-path source. U3 (#173): + // retried through the SHARED helper (this was a bare call, so a single + // transient error dropped the source outright) and, on exhaustion, marked + // durably so the resolver keeps the bounded scan fallback for the object. + // The retry ladder is bounded AS A WHOLE, not per attempt: three stalls + // plus their backoff otherwise multiply one remainder by three. Floored at + // DB_RECORD_GRACE because this is a durability write. + match crate::ipfs_pin::db_bounded( + crate::ipfs_pin::db_record_deadline(deadline), + crate::ipfs_pin::retry_db_record(|| db.record_pin_source(&sha, repo_id)), + ) + .await + { + Ok(()) => {} + // Elapsed here is a DEFINITE non-write, in lockstep with the twin, + // and for a reason that comes from the operation rather than the + // timeout: `record_pin_source` is an explicit transaction whose + // `tx.commit()` a cancelled future never reaches, so no COMMIT is + // sent and the row cannot have landed. Mark the set incomplete + // exactly as the definite-error arm does; an incomplete-and-unmarked + // set is read as COMPLETE and 404s a copy this repo would serve. The + // marker's cost is bounded (the fallback scan is capped at + // `ipfs_max_legacy_probes` and charges the per-IP work rate limiter + // per probe). The arm stays separate only so the warn tells an + // operator a stalled batch from a scattered per-object failure. + Err(e @ crate::ipfs_pin::BoundedDbError::Elapsed) => { + tracing::warn!( + sha = %sha, + err = %e, + "pin source record did not complete inside the batch deadline; \ + a cancelled multi-statement transaction never commits, so the \ + source is definitely missing and the set is marked incomplete" + ); + if let Err(e) = crate::ipfs_pin::db_bounded( + crate::ipfs_pin::db_record_deadline(deadline), + db.mark_pin_sources_incomplete(&sha, repo_id), + ) + .await + { + tracing::warn!(sha = %sha, err = %e, "failed to mark pin sources incomplete"); + } + } + // The retries are spent on REAL errors, so this repo is definitely NOT + // in the source set and the set is known incomplete. Floored for the + // same reason the record above is: a spent budget must not drop the + // compensation. The marker write is a single autocommit statement, so + // ITS own elapsed arm is a genuine unknown outcome; nothing branches + // on it, which is why warn-only is right there. + Err(e) => { + tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); + if let Err(e) = crate::ipfs_pin::db_bounded( + crate::ipfs_pin::db_record_deadline(deadline), + db.mark_pin_sources_incomplete(&sha, repo_id), + ) + .await + { + tracing::warn!(sha = %sha, err = %e, "failed to mark pin sources incomplete"); + } + } + } + // R8 (#173 round 10), in lockstep with the ipfs_pin skip branch: + // opportunistically repair a legacy provider-CID row (Kubo dag-pb / + // Pinata) to the raw-content resolver key on this re-push. Cost-gated on + // the stored key's codec, so a non-legacy row reads no bytes. Warn-only: + // a failure leaves the row as-is for a later re-push or the deferred + // one-shot sweep. + // Clamped to the batch deadline, in lockstep with the ipfs_pin twin: this + // runs with the pin permit held, so an unclamped `git_timeout` would let + // one wedged read hold a global pin slot for 600s against a 120s budget. + if let Err(e) = crate::ipfs_pin::repair_legacy_provider_cid( + repo_path, + git_bin, + std::cmp::min(deadline, std::time::Instant::now() + git_timeout), + &sha, + db, + ) + .await + { + tracing::warn!(sha = %sha, err = %e, "failed to repair legacy provider CID"); + } + continue; + } Ok(false) => {} Err(e) => { tracing::warn!(sha = %sha, err = %e, "DB error checking pinata_cid"); @@ -163,7 +326,16 @@ pub async fn pin_new_objects( // between the two, so `Instant::now() + budget_left` would land past `deadline` by // however long the DB took, and under a saturated pool that is the dominant term. // A slow DB check must not push the read's own bound out. - let read_deadline = deadline; + // + // Bounded by the EARLIER of the batch deadline (#174) and this object's own + // `git_timeout` (#173), the same pair the ipfs_pin twin uses. Both bounds are + // load-bearing and neither implies the other: the batch deadline alone would let + // ONE wedged `cat-file` hold the pin permit for the whole budget, while + // `git_timeout` alone would let a batch of merely-slow reads run past the budget. + // As on the twin, at SHIPPED DEFAULTS the batch deadline is the arm that binds + // (600s git timeout against a 120s budget); the `git_timeout` arm is for an + // operator who tightens that knob below the remaining budget. + let read_deadline = std::cmp::min(deadline, std::time::Instant::now() + git_timeout); let read_path = repo_path.to_path_buf(); let read_sha = sha.clone(); let read_git = git_bin.to_string(); @@ -224,9 +396,86 @@ pub async fn pin_new_objects( match pin_object(client, upload_url, jwt, &sha, &data).await { Ok(cid) if !cid.is_empty() => { - if let Err(e) = db.record_pinata_cid(&sha, &cid).await { + // The resolver key (`pinned_cids.cid`) must be the locally-computed + // raw-content CID, never the provider CID: Pinata wraps the bytes in + // dag-pb/UnixFS, so its returned CID does not hash the raw content and + // must not become an alias `/ipfs/{cid}` serves raw git bytes for (#173). + let raw_cid = gitlawb_core::cid::Cid::from_git_object_bytes(&data).to_string(); + // U3 (#173): both records go through the shared retry helper, at parity + // with the ipfs_pin twin. These were bare calls, so one transient DB error + // permanently dropped a pin source. + // + // Both bounds here are FLOORED at DB_RECORD_GRACE (F3, #173). The upload + // runs under the shared client's own ceiling, so a successful one can + // return with ~0 of the batch budget left, and an unfloored bound would + // fail a write that today completes in milliseconds. That costs more on + // this side than on the twin: `pinned.push` below is UNCONDITIONAL, so + // `api/repos.rs` builds its `cid_map` from the pair either way and drives + // `upsert_branch_cid` plus the p2p `publish_ref_update` gossip from it. A + // dropped record therefore makes the node ADVERTISE a CID whose `/ipfs` + // read 404s. If the floored bound still fires, THIS site's outcome really + // is unknown, and unlike the source record below that is a property of + // the operation: `record_pinata_cid` is a single autocommit upsert, so + // the statement Postgres already started can still land after the client + // future is cancelled. The warn names the arm through the error's own + // Display and the site keeps its existing behavior: the pair is still + // returned, and the row may or may not exist. + if let Err(e) = crate::ipfs_pin::db_bounded( + crate::ipfs_pin::db_record_deadline(deadline), + crate::ipfs_pin::retry_db_record(|| { + db.record_pinata_cid(&sha, &raw_cid, &cid, Some(repo_id)) + }), + ) + .await + { tracing::warn!(sha = %sha, err = %e, "failed to record pinata_cid in DB"); } + // F1 (#173 round 8): also record the first pinner in pin_repo_sources. + // U3: an exhausted retry marks the set incomplete so the resolver keeps + // the scan fallback rather than 404ing a copy it could serve. + match crate::ipfs_pin::db_bounded( + crate::ipfs_pin::db_record_deadline(deadline), + crate::ipfs_pin::retry_db_record(|| db.record_pin_source(&sha, repo_id)), + ) + .await + { + Ok(()) => {} + // Same rule as the skip branch above, and the same reason: this + // wraps `record_pin_source`, an explicit transaction, so a timed-out + // call definitely never committed and the source is definitely + // missing. Mark the set incomplete rather than leaving it incomplete + // and unmarked. Note the contrast with `record_pinata_cid` a few + // lines up: that one is a single autocommit statement, so its + // timeout genuinely is an unknown outcome and it is warn-only. + Err(e @ crate::ipfs_pin::BoundedDbError::Elapsed) => { + tracing::warn!( + sha = %sha, + err = %e, + "pin source record did not complete inside the batch deadline; \ + a cancelled multi-statement transaction never commits, so the \ + source is definitely missing and the set is marked incomplete" + ); + if let Err(e) = crate::ipfs_pin::db_bounded( + crate::ipfs_pin::db_record_deadline(deadline), + db.mark_pin_sources_incomplete(&sha, repo_id), + ) + .await + { + tracing::warn!(sha = %sha, err = %e, "failed to mark pin sources incomplete"); + } + } + Err(e) => { + tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); + if let Err(e) = crate::ipfs_pin::db_bounded( + crate::ipfs_pin::db_record_deadline(deadline), + db.mark_pin_sources_incomplete(&sha, repo_id), + ) + .await + { + tracing::warn!(sha = %sha, err = %e, "failed to mark pin sources incomplete"); + } + } + } pinned.push((sha, cid)); } Ok(_) => {} @@ -433,8 +682,10 @@ mod tests { "test-jwt", &repo_path, "git", + Duration::from_secs(60), oids, &db, + "repo-merge-test", Duration::from_millis(5500), ), ) @@ -522,8 +773,10 @@ mod tests { "test-jwt", &repo_path, fake.to_str().unwrap(), + Duration::from_secs(60), oids, &db, + "repo-merge-test", Duration::from_secs(2), ), ) @@ -574,6 +827,66 @@ mod tests { ); } + /// U3 scenario 5 (#173): the read is bounded by the EARLIER of the batch deadline and + /// this object's own `git_timeout`, the same pair the ipfs_pin twin uses. The batch + /// budget here is generous (60s) so the budget gate cannot be what ends the call: only + /// the 1s `git_timeout` can. A wedged `git cat-file` that traps SIGTERM and sleeps 30s + /// must therefore be reaped in the `git_timeout` order and the call must return, rather + /// than holding the pin permit for the whole budget. + /// + /// RED with `let read_deadline = deadline;` (the pre-U3 bare batch deadline): the read + /// waits out the wedged child, the call runs ~30s, and the outer 20s timeout fires. + #[cfg(unix)] + #[sqlx::test] + async fn pin_new_objects_bounds_the_read_by_git_timeout_not_the_batch_budget( + pool: sqlx::PgPool, + ) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.expect("migrations"); + let tmp = tempfile::TempDir::new().unwrap(); + let repo_path = tmp.path().join("git-timeout.git"); + let oids = seed_loose_blobs(&repo_path, 1); + let fake = tmp.path().join("hanging-git"); + write_script(&fake, "#!/bin/sh\ntrap '' TERM\necho $$ > pid\nsleep 30\n"); + + let (_logs, _guard) = capture_logs(); + let client = reqwest::Client::new(); + let started = std::time::Instant::now(); + let pinned = tokio::time::timeout( + Duration::from_secs(20), + pin_new_objects( + &client, + "http://127.0.0.1:9", + "test-jwt", + &repo_path, + fake.to_str().unwrap(), + // The bound under test. + Duration::from_secs(1), + oids, + &db, + "repo-git-timeout", + // Generous, so a call that ends on time ended on `git_timeout`. + Duration::from_secs(60), + ), + ) + .await + .expect( + "the read must be bounded by git_timeout, not by the batch budget: a wedged git \ + cannot hold the pin permit for the whole 60s budget", + ); + let elapsed = started.elapsed(); + + assert!( + pinned.is_empty(), + "a git that never answers cannot produce a pinned object: {pinned:?}" + ); + assert!( + elapsed < Duration::from_secs(15), + "elapsed {elapsed:?} must stay in the git_timeout order (1s plus one watchdog \ + teardown), not the 60s batch budget" + ); + } + /// A `git_bin` wrapper that records every invocation's arguments and then execs the /// real git, so a test can tell which objects the loop actually attempted. The returned /// pin list cannot: it is empty both when the loop broke after one object and when it @@ -650,8 +963,10 @@ mod tests { "test-jwt", &repo_path, &git_bin, + Duration::from_secs(60), oids, &db, + "repo-merge-test", Duration::from_secs(60), ), ) @@ -722,8 +1037,10 @@ mod tests { "test-jwt", &repo_path, &git_bin, + Duration::from_secs(60), oids, &db, + "repo-merge-test", Duration::from_secs(60), ), ) @@ -771,8 +1088,10 @@ mod tests { "test-jwt", &repo_path, "git", + Duration::from_secs(60), oids.clone(), &db, + "repo-merge-test", Duration::from_secs(60), ), ) @@ -798,8 +1117,10 @@ mod tests { "test-jwt", &repo_path, "git", + Duration::from_secs(60), oids, &db, + "repo-merge-test", Duration::from_secs(60), ), ) @@ -849,8 +1170,10 @@ mod tests { "", &repo_path, fake.to_str().unwrap(), + Duration::from_secs(60), oids, &db, + "repo-merge-test", Duration::from_secs(60), ), ) @@ -865,6 +1188,406 @@ mod tests { assert_eq!(requests.load(std::sync::atomic::Ordering::SeqCst), 0); } + // ── Stalled-DB bound (F3, #173) ─────────────────────────────────────── + // + // `api/repos.rs` acquires the GLOBAL `pin_semaphore` for the Pinata + // replication task and holds it across this whole function, the same permit + // the IPFS lane takes. `batch_budget_gate` only gates BETWEEN objects and the + // git read is already clamped, so a bare DB await in the region parked that + // permit for as long as the query was stuck. The tests below drive the stall + // with a `LOCK TABLE .. IN ACCESS EXCLUSIVE MODE` held on a dedicated pooled + // connection, the same technique the ipfs_pin twin's stall tests use, and copy + // their tolerances (a 1.5s budget, an `elapsed < 3s` assertion, a 10s outer + // wrap). The budget is above `PIN_READ_FLOOR` on purpose: below it + // `batch_budget_gate` breaks the batch as the loop body's FIRST statement, so a + // ~1s budget would never reach a DB call and the test would pass with the bound + // deleted. + // --------------------------------------------------------------------- + + /// Take an `ACCESS EXCLUSIVE` lock on `table` on a dedicated pooled connection. + /// Every SELECT needs `ACCESS SHARE`, which conflicts, so the next statement + /// touching the table blocks at lock acquisition regardless of row count. + /// Copied from `ipfs_pin.rs`'s test mod rather than shared, since test mods are + /// private, the same way `seed_loose_blobs` and `capture_logs` are. + async fn lock_table( + pool: &sqlx::PgPool, + table: &str, + ) -> sqlx::pool::PoolConnection { + let mut conn = pool.acquire().await.unwrap(); + sqlx::raw_sql(&format!( + "BEGIN; LOCK TABLE {table} IN ACCESS EXCLUSIVE MODE;" + )) + .execute(&mut *conn) + .await + .unwrap(); + conn + } + + async fn rollback(conn: &mut sqlx::pool::PoolConnection) { + sqlx::raw_sql("ROLLBACK") + .execute(&mut **conn) + .await + .unwrap(); + } + + /// Scenario 3: the FIRST DB call in this lane's budgeted region + /// (`has_pinata_cid`) stalls. With the batch deadline bounding it the loop + /// abandons the object, the budget gate then breaks the batch, and the call + /// returns at ~budget having uploaded nothing. Pre-fix the bare await blocks for + /// the lock's whole lifetime, holding the caller's global pin permit with it. + /// + /// The upload mock is at `.expect(0)`: a stalled pinned-status check must never + /// fall through to an upload, since that would re-send bytes Pinata may already + /// hold and, worse, return a CID this node then advertises. + #[sqlx::test] + async fn pinata_pin_new_objects_stalled_db_returns_by_budget(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool.clone()); + db.run_migrations().await.expect("migrations"); + let tmp = tempfile::TempDir::new().unwrap(); + let repo_path = tmp.path().join("pinata_stalled.git"); + let oids = seed_loose_blobs(&repo_path, 1); + + let mut server = mockito::Server::new_async().await; + let upload = server + .mock("POST", mockito::Matcher::Any) + .with_status(200) + .with_body(r#"{"data":{"cid":"QmShouldNotHappen"}}"#) + .expect(0) + .create_async() + .await; + + // Install a log capture even though nothing here asserts on it: `tracing` + // caches a callsite's interest globally the first time it is hit, and a hit + // from a thread with no subscriber caches it as never-interested for the whole + // binary, which silently blinds the sibling tests that DO assert on the batch + // deadline warn. + let (_logs, _log_guard) = capture_logs(); + + let mut lock = lock_table(&pool, "pinned_cids").await; + + let client = reqwest::Client::new(); + let started = std::time::Instant::now(); + let pinned = tokio::time::timeout( + Duration::from_secs(10), + pin_new_objects( + &client, + &server.url(), + "test-jwt", + &repo_path, + "git", + Duration::from_secs(30), + oids, + &db, + "repo-pinata-stalled", + Duration::from_millis(1500), + ), + ) + .await + .expect( + "a stalled DB must cost the batch its budget, not the lock's lifetime: the \ + bare await hangs past this wrap", + ); + let elapsed = started.elapsed(); + + assert!( + pinned.is_empty(), + "a stalled pinata-status check cannot produce a pinned object: {pinned:?}" + ); + assert!( + elapsed < Duration::from_secs(3), + "the batch deadline must end the call at ~budget (1.5s); got {elapsed:?}" + ); + upload.assert_async().await; + + rollback(&mut lock).await; + } + + /// The Pinata half of the timed-out source record, driven rather than argued: the + /// twin's `pin_new_objects_skip_branch_stalled_record_returns_by_budget` covers the + /// Kubo lane and this covers the site that has to change in lockstep with it. + /// + /// The object already has a `pinata_cid`, so the loop takes the skip branch and + /// tries to record this repo as an additional source; `pin_repo_sources` is locked + /// for the whole run, so that insert stalls inside `retry_db_record` and the whole + /// ladder elapses against one floored remainder. Two properties: + /// + /// - the call still returns promptly, at the record floor rather than the lock's + /// lifetime; + /// - on the TIMEOUT arm the incomplete marker IS written. `record_pin_source` is an + /// explicit transaction, so the cancelled future never reaches `tx.commit()`, no + /// COMMIT is ever sent, and the source definitely did not land. Withholding the + /// marker there would leave the set incomplete AND unmarked, which the resolver + /// reads as complete and 404s a copy this repo would serve. + /// + /// `pinned_cids` is deliberately NOT locked, so the marker write itself is free to + /// land and the assertion below is about the branch, not about lock contention. + #[sqlx::test] + async fn pinata_skip_branch_stalled_record_marks_incomplete(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool.clone()); + db.run_migrations().await.expect("migrations"); + let tmp = tempfile::TempDir::new().unwrap(); + let repo_path = tmp.path().join("pinata_skip_stalled.git"); + let oids = seed_loose_blobs(&repo_path, 1); + let sha = oids[0].clone(); + // Seed the object as already Pinata-pinned, by a DIFFERENT repo, so the skip + // branch is taken and the source record below is a genuine additional-source + // insert rather than a no-op on the conflict. The resolver key is a canonical + // raw CIDv1 so the opportunistic legacy repair takes its cost gate and reads no + // bytes. + let raw_cid = + gitlawb_core::cid::Cid::from_git_object_bytes(b"pinata skip seed").to_string(); + db.record_pinata_cid(&sha, &raw_cid, "QmSeedProviderCid", Some("repo-seed")) + .await + .unwrap(); + db.record_pin_source(&sha, "repo-seed").await.unwrap(); + let requests = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let endpoint = delaying_pinata_endpoint( + vec![Duration::from_millis(0)], + std::sync::Arc::clone(&requests), + ) + .await; + // Install a log capture even though nothing here asserts on it: `tracing` + // caches a callsite's interest globally the first time it is hit, and a hit + // from a thread with no subscriber caches it as never-interested for the whole + // binary, which silently blinds the sibling tests that DO assert on the batch + // deadline warn. + let (_logs, _log_guard) = capture_logs(); + + let mut lock = lock_table(&pool, "pin_repo_sources").await; + + let client = reqwest::Client::new(); + let started = std::time::Instant::now(); + let pinned = tokio::time::timeout( + Duration::from_secs(15), + pin_new_objects( + &client, + &endpoint, + "test-jwt", + &repo_path, + "git", + Duration::from_secs(30), + oids, + &db, + "repo-pinata-skip-stalled", + Duration::from_millis(1500), + ), + ) + .await + .expect( + "the wrapped retry ladder must fit inside one floored remainder: the bare \ + retry_db_record hangs past this wrap", + ); + let elapsed = started.elapsed(); + + assert!( + pinned.is_empty(), + "an already-pinned object is skipped, never re-uploaded: {pinned:?}" + ); + assert_eq!( + requests.load(std::sync::atomic::Ordering::SeqCst), + 0, + "the skip branch must not reach the upload at all" + ); + assert!( + elapsed < Duration::from_secs(5), + "the record's floored remainder must end the call promptly; got {elapsed:?}" + ); + + rollback(&mut lock).await; + drop(lock); + + assert!( + db.pin_sources_incomplete(&sha).await.unwrap(), + "a TIMED-OUT `record_pin_source` definitively did not land: it is an explicit \ + multi-statement transaction, and the cancelled future never reaches \ + `tx.commit()`, so no COMMIT is ever sent and the row cannot exist. The set is \ + therefore incomplete, and leaving it UNMARKED is the exact state the marker \ + exists to prevent: the resolver reads a non-empty below-cap set as complete \ + and 404s a copy this repo would serve" + ); + } + + /// Scenario 8, the Pinata half of the durability floor. `batch_budget_gate` only + /// guarantees `PIN_READ_FLOOR` before an object STARTS and the upload runs under + /// the shared client's own ceiling, so a successful upload can return with ~0 of + /// the batch budget left. Without the floor the post-upload `record_pinata_cid` + /// would then be failed by a spent deadline, and this lane's `pinned.push` is + /// UNCONDITIONAL: `api/repos.rs` builds `cid_map` from the return and drives + /// `upsert_branch_cid` plus the p2p `publish_ref_update` gossip from it, so a + /// dropped record makes the node advertise a CID whose `/ipfs` read 404s. + /// + /// Fixture: a 2s budget, a 1.7s upload, and `pinned_cids` locked from 500ms (well + /// after `has_pinata_cid` has read it, and still well before the upload returns) + /// until 2.4s. The record therefore starts at ~1.72s with ~280ms of budget left + /// and needs ~680ms of lock wait to land, which only the `DB_RECORD_GRACE` floor + /// buys it. + /// + /// The lock time is a MARGIN, not a boundary: taking it at 100ms left + /// `has_pinata_cid` racing it on a loaded box, and losing that race makes the read + /// block, time out, and break the batch, which fails on `pinned.len() == 1` for a + /// reason that has nothing to do with the floor. Any time between the + /// `has_pinata_cid` round trip and the upload's 1.7s return proves the same thing. + #[sqlx::test] + async fn pinata_pin_add_with_spent_budget_still_records_cid(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool.clone()); + db.run_migrations().await.expect("migrations"); + let tmp = tempfile::TempDir::new().unwrap(); + let repo_path = tmp.path().join("pinata_spent_budget.git"); + let oids = seed_loose_blobs(&repo_path, 1); + let sha = oids[0].clone(); + let requests = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let endpoint = delaying_pinata_endpoint( + vec![Duration::from_millis(1700)], + std::sync::Arc::clone(&requests), + ) + .await; + // Install a log capture even though nothing here asserts on it: `tracing` + // caches a callsite's interest globally the first time it is hit, and a hit + // from a thread with no subscriber caches it as never-interested for the whole + // binary, which silently blinds the sibling tests that DO assert on the batch + // deadline warn. + let (_logs, _log_guard) = capture_logs(); + + let lock_pool = pool.clone(); + let locker = async move { + tokio::time::sleep(Duration::from_millis(500)).await; + let mut conn = lock_table(&lock_pool, "pinned_cids").await; + tokio::time::sleep(Duration::from_millis(1900)).await; + rollback(&mut conn).await; + }; + + let client = reqwest::Client::new(); + let pin = tokio::time::timeout( + Duration::from_secs(20), + pin_new_objects( + &client, + &endpoint, + "test-jwt", + &repo_path, + "git", + Duration::from_secs(30), + oids.clone(), + &db, + "repo-pinata-spent-budget", + Duration::from_millis(2000), + ), + ); + let (pinned, ()) = tokio::join!(pin, locker); + let pinned = pinned.expect("the floored record must land well inside this wrap"); + + assert_eq!( + requests.load(std::sync::atomic::Ordering::SeqCst), + 1, + "the fixture only proves anything if the upload actually ran" + ); + assert!( + db.has_pinata_cid(&sha).await.unwrap(), + "a successful upload whose batch deadline is spent must still land its \ + pinned_cids row: this lane pushes the pair unconditionally, so a dropped \ + record makes api/repos.rs advertise a CID the resolver cannot serve" + ); + assert_eq!( + pinned.len(), + 1, + "the uploaded pin must still be returned: {pinned:?}" + ); + } + + /// The POST-UPLOAD source record's timeout arm, the one site of the three that + /// nothing else executes. The skip-branch twin above and the ipfs_pin lane cover + /// the other two; this arm sits after a SUCCESSFUL `pin_object`, so no skip-branch + /// fixture can reach it. + /// + /// Why it has to write the marker at all: `record_pin_source` is an explicit + /// transaction ending in `tx.commit()`, and a cancelled future never gets there, so + /// no COMMIT is sent and the row definitely does not exist. The set is therefore + /// incomplete, and leaving it unmarked is the state the marker exists to prevent. + /// + /// Fixture: the object is NOT seeded as Pinata-pinned, so `has_pinata_cid` is false + /// and the run takes the upload path. The mock answers the upload at once, then + /// `pin_repo_sources` is held under `ACCESS EXCLUSIVE` for the whole run, so the + /// post-upload `record_pin_source` blocks and elapses against its floored bound at + /// ~2s. `pinned_cids` is deliberately left UNLOCKED, so both `record_pinata_cid` + /// and the marker write itself are free to land and the assertion is about the arm + /// rather than about my own lock. + /// + /// The upload assertion is what keeps this from being vacuous: without it the test + /// would pass just as well if the run never reached the post-upload path at all. + #[sqlx::test] + async fn pinata_post_upload_stalled_record_marks_incomplete(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool.clone()); + db.run_migrations().await.expect("migrations"); + let tmp = tempfile::TempDir::new().unwrap(); + let repo_path = tmp.path().join("pinata_post_upload.git"); + let oids = seed_loose_blobs(&repo_path, 1); + let sha = oids[0].clone(); + + let mut server = mockito::Server::new_async().await; + let upload = server + .mock("POST", mockito::Matcher::Any) + .with_status(200) + .with_body(r#"{"data":{"cid":"QmPostUploadProviderCid"}}"#) + .expect(1) + .create_async() + .await; + + // Install a log capture even though nothing here asserts on it: `tracing` + // caches a callsite's interest globally the first time it is hit, and a hit + // from a thread with no subscriber caches it as never-interested for the whole + // binary, which silently blinds the sibling tests that DO assert on the batch + // deadline warn. + let (_logs, _log_guard) = capture_logs(); + + let mut lock = lock_table(&pool, "pin_repo_sources").await; + + let client = reqwest::Client::new(); + let started = std::time::Instant::now(); + let pinned = tokio::time::timeout( + Duration::from_secs(20), + pin_new_objects( + &client, + &server.url(), + "test-jwt", + &repo_path, + "git", + Duration::from_secs(30), + oids, + &db, + "repo-pinata-post-upload", + Duration::from_millis(1500), + ), + ) + .await + .expect( + "the wrapped retry ladder must fit inside one floored remainder: the bare \ + retry_db_record hangs past this wrap", + ); + let elapsed = started.elapsed(); + + rollback(&mut lock).await; + drop(lock); + + upload.assert_async().await; + assert_eq!( + pinned.len(), + 1, + "the upload succeeded, so this lane still returns the pair: {pinned:?}" + ); + assert!( + elapsed < Duration::from_secs(8), + "the record's floored remainder must end the call promptly, never at the \ + lock's lifetime; got {elapsed:?}" + ); + assert!( + db.pin_sources_incomplete(&sha).await.unwrap(), + "the POST-UPLOAD arm must mark the source set incomplete when its record \ + times out: `record_pin_source` is an explicit transaction whose cancelled \ + future never reaches `tx.commit()`, so the row definitely did not land, and \ + an incomplete-and-unmarked set is read as complete and 404s a copy this \ + repo would serve" + ); + } + #[tokio::test] async fn test_pin_skipped_when_jwt_empty() { let client = reqwest::Client::new(); diff --git a/crates/gitlawb-node/src/rate_limit.rs b/crates/gitlawb-node/src/rate_limit.rs index d203f69f..c3626089 100644 --- a/crates/gitlawb-node/src/rate_limit.rs +++ b/crates/gitlawb-node/src/rate_limit.rs @@ -121,11 +121,26 @@ impl RateLimiter { true } - /// Number of keys currently tracked. Tests use it to observe what a sweep - /// reclaimed; there is no production reader. - #[cfg(test)] - pub async fn tracked_keys(&self) -> usize { - self.state.lock().await.len() + /// Non-consuming check: is this key ALREADY at its limit for the current window? + /// Unlike [`check`], it records nothing and never inserts a new key — used to shed + /// expensive preparatory work (e.g. the `/ipfs/{cid}` legacy scan's O(repos) DB + /// preload) BEFORE it runs, without perturbing the per-unit budget the consuming + /// `check` maintains (#173, F3). An unknown key or a disabled limiter is not + /// throttled. Prunes the key's expired timestamps as a side effect (keeps state + /// tidy) but adds none, so it cannot itself fill or grow the map. + pub(crate) async fn is_throttled(&self, key: &str) -> bool { + if self.max_requests == 0 { + return false; + } + let now = Instant::now(); + let mut state = self.state.lock().await; + if let Some(window) = state.get_mut(key) { + window + .timestamps + .retain(|t| now.duration_since(*t) < self.window); + return window.timestamps.len() >= self.max_requests; + } + false } pub async fn cleanup(&self) { @@ -137,6 +152,14 @@ impl RateLimiter { !w.timestamps.is_empty() }); } + + /// Number of distinct keys currently tracked. Test-only introspection so a + /// cross-module test can assert that a sweep actually evicted expired entries + /// and observe what it reclaimed. There is no production reader. + #[cfg(test)] + pub(crate) async fn tracked_keys(&self) -> usize { + self.state.lock().await.len() + } } /// Per-source concurrency cap derived from the write-pool size: one resolved client diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 9d23572b..24607e5a 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -8,6 +8,19 @@ use crate::git::repo_store::RepoStore; use crate::p2p::P2pHandle; use crate::rate_limit::RateLimiter; +/// HKDF salt for [`AppState::derive_scan_token_key`]. A constant, not a secret: HKDF's +/// salt is a domain qualifier, and the confidentiality of the derived key rests entirely +/// on the node's private seed being the input keying material. +const SCAN_TOKEN_KEY_SALT: &[u8] = b"gitlawb/hkdf-salt/ipfs-scan-token"; + +/// HKDF `info` for [`AppState::derive_scan_token_key`]: the domain separation AND the +/// rotation handle in one string. Nothing else in the node derives from this label, so +/// the token key is unrelated to the signing key it shares an input with; bumping the +/// trailing version rotates every node's token key on its next boot, which invalidates +/// outstanding continuations (they simply fail to open and the caller restarts at the +/// front) and needs no migration, no config, and no change to this function. +const SCAN_TOKEN_KEY_INFO: &[u8] = b"gitlawb/ipfs-scan-token/v1"; + #[derive(Clone, Debug)] pub struct RefUpdateBroadcast { pub repo: String, @@ -66,6 +79,98 @@ pub struct AppState { /// brake a push flood from a DID farm (one throwaway DID per repo), so the /// push path throttles on the resolved client IP instead. pub push_rate_limiter: RateLimiter, + /// Per-client-IP ROUTE brake for `GET /ipfs/{cid}`: charged ONCE per request by the + /// `rate_limit_by_ip` middleware (server.rs), never inside the handler. It bounds + /// request RATE (the "requests per hour" contract of `GITLAWB_IPFS_RATE_LIMIT`) on + /// the non-farmable source IP, so an anonymous flood of the public route is capped. + /// The per-probe/per-walk WORK accounting the resolver does WITHIN a request draws + /// from the SEPARATE `ipfs_work_rate_limiter` below — the two cannot share one bucket + /// or a single request that spends a route token and then its own probe token off the + /// same bucket is admitted at the route and falsely shed mid-request (#173 round-10, + /// R6). Keyed by `push_limiter_trust`. + pub ipfs_rate_limiter: RateLimiter, + /// Per-client-IP WORK-budget limiter for the `GET /ipfs/{cid}` resolver's internal + /// fan-out: charged per legacy (NULL-provenance) PROBE (`acquire` + `cat-file`) and + /// per provenance-path WALK, and peeked non-consuming before the O(repos) legacy + /// preload. A legacy CID from the public pins index otherwise lets one request drive + /// O(repos) subprocess spawns and cold Tigris fetches, and repeat requests amplify + /// that across requests with zero limiter contact (INV-10, F3). Charging the work to + /// the non-farmable source IP bounds it. A bucket DISTINCT from the route brake above: + /// one request legitimately spends many work tokens (a full legacy scan is up to + /// `ipfs_max_legacy_probes` probes), so it must not double as the once-per-request + /// route bucket. Capacity is DERIVED from the route limit (`AppState::ipfs_work_budget`, + /// no separate operator knob), floored at the legacy-probe budget so a single default- + /// config deep search never self-throttles mid-scan. Keyed by `push_limiter_trust`. + pub ipfs_work_rate_limiter: RateLimiter, + /// Per-request ceiling on full-history reachability walks the CID resolver + /// may spawn (default `api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST`). A field, + /// not a bare const, so tests can shrink it to exercise the cap cheaply; + /// production keeps the const default. + pub ipfs_max_history_walks: u32, + /// Per-request ceiling on legacy (NULL-provenance) repo probes in the CID + /// resolver's scan fallback (default `api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST`). + /// Bounds the anonymous `acquire` + `cat-file` fan-out across the node (#173, + /// INV-10); a field for the same test-seam reason as `ipfs_max_history_walks`. + pub ipfs_max_legacy_probes: u32, + /// How many repo rows the CID resolver's legacy scan pulls per keyset page + /// (default `api::ipfs::LEGACY_SCAN_PAGE_ROWS`). Bounds the DATABASE-facing half + /// of the same fan-out `ipfs_max_legacy_probes` bounds on the probe side: without + /// it the scan materialized every repo row and every matching visibility rule + /// before spending a single probe (#173, INV-10). A field for the same test-seam + /// reason as the sibling caps. + pub ipfs_legacy_scan_page_rows: usize, + /// Per-request ceiling on how many repo ROWS the CID resolver's legacy scan may + /// fetch (default `api::ipfs::MAX_LEGACY_SCAN_ROWS_PER_REQUEST`, operator-tunable via + /// `GITLAWB_IPFS_MAX_LEGACY_SCAN_ROWS`). `ipfs_max_legacy_probes` bounds the PROBE + /// fan-out but only starts counting once a probe runs, and quarantine plus a + /// root-scope visibility deny both return before a probe or a visit is spent, so an + /// all-denying inventory paged the whole repo table at zero probes (#173 round 13, F2). + /// Truncating here sheds a retryable 503 carrying a sealed continuation token. + pub ipfs_max_legacy_scan_rows: usize, + /// Per-request ceiling on the BYTES of visibility rules the CID resolver's legacy + /// scan may retain (default `api::ipfs::MAX_LEGACY_SCAN_RULE_BYTES_PER_REQUEST`). The + /// row ceiling above bounds the row count but not the memory each row drags in: the + /// pager keeps every fetched page's rules for the whole request, and neither the + /// number of rules per repo nor the length of a rule's reader list is capped, so a + /// rule COUNT would be the wrong unit. Enforced by the rules QUERY + /// (`Db::list_visibility_rules_for_repos_bounded`) rather than by a sum taken once the + /// page has landed, so the oversized page is never materialized at all. Deliberately + /// NOT an operator knob (it is a + /// memory guard, not a reach tradeoff); a field only for the same test-seam reason as + /// the sibling caps. + pub ipfs_max_legacy_scan_rule_bytes: usize, + /// Key sealing the legacy scan's continuation tokens (INV-13), derived from the + /// node's persistent identity by [`AppState::derive_scan_token_key`]. + /// + /// The token is minted from a FETCHED row on a scan that served nothing, so by + /// construction that row is a private or quarantined repo the caller may not read: + /// its `created_at` and its `id` (which carries the owner's DID) are withheld + /// fields. The token is therefore AEAD-SEALED, never signed plaintext and never + /// base64-of-plaintext, since integrity is not confidentiality. + /// + /// DERIVED, not random per boot. This reverses an earlier revision of this design, + /// which argued that derivation "would make old tokens valid across restarts for no + /// benefit and tie a throwaway transport secret to a long-lived signing key." Both + /// halves were wrong. Surviving a restart IS the benefit: the token is the ONLY way + /// a caller resumes a ladder, an unopenable one is treated as absent, and a caller + /// treated as absent silently restarts at the front of the scan. A node whose + /// inventory needs several ladder steps and which deploys more often than a caller + /// can climb therefore keeps that caller from ever reaching a holder, which + /// contradicts the reach bound the README states. And the tie to the signing key is + /// what the HKDF domain separation removes: the derived key is a one-way function of + /// the seed under an `info` string nothing else uses, so it is not the signing key + /// and cannot be worked back into one. + /// + /// Persisting a random key instead would need a schema change for a value the node + /// already has on disk, so derivation is also the smaller mechanism. Rotation is a + /// bump of the version in [`SCAN_TOKEN_KEY_INFO`]. + pub ipfs_scan_token_key: Arc<[u8; 32]>, + /// Hard ceiling on the byte size of an object `GET /ipfs/{cid}` will buffer and + /// serve (default `api::ipfs::MAX_SERVED_OBJECT_BYTES`). The serve reads via a + /// blocking `git cat-file` and buffers the whole object; without a bound a large + /// public blob could exhaust memory or block a runtime worker (#173, F6, INV-10). + /// A field for the same test-seam reason as the sibling caps. + pub ipfs_max_served_object_bytes: u64, /// Which forwarded header (if any) the edge is trusted to set, for /// resolving the push limiter's client-IP key. See `GITLAWB_TRUSTED_PROXY`. /// Node-wide; also keys the two peer-sync limiters below. @@ -214,13 +319,6 @@ pub struct AppState { /// (`with_default_max_keys`, reject-before-insert) so a source-key farm cannot grow /// it (INV-15). pub git_ipfs_walk_per_caller: crate::rate_limit::PerCallerConcurrency, - /// Per-client-IP rate limiter for `GET /ipfs/{cid}`. The route is publicly - /// reachable and each request can drive a full-history git walk, so it carries a - /// per-IP flood brake in addition to the concurrency cap above — a rate limit - /// bounds request *rate*, the semaphore bounds concurrent slow holds (different - /// axes). Keyed on the resolved client IP via `push_limiter_trust`. Layered on the - /// `/ipfs` route via `rate_limit_by_ip`. - pub ipfs_rate_limiter: RateLimiter, /// The `git` executable the served-git withheld-blob walk spawns. Production is /// `"git"` (resolved via PATH); injectable so a fake `git` can drive the walk's /// process-group teardown in handler tests without mutating the process-global @@ -235,6 +333,21 @@ impl AppState { self.shutdown_tx.subscribe() } + /// Sweep expired entries from every per-IP/DID rate limiter. Driven by the + /// periodic cleanup task so a bounded limiter's key map sheds stale entries + /// instead of sitting near its cap until an inline capacity sweep reclaims + /// them. Every limiter on the state is swept here; adding a new limiter means + /// adding it to this list. + pub(crate) async fn sweep_rate_limiters(&self) { + self.rate_limiter.cleanup().await; + self.create_ip_rate_limiter.cleanup().await; + self.push_rate_limiter.cleanup().await; + self.ipfs_rate_limiter.cleanup().await; + self.ipfs_work_rate_limiter.cleanup().await; + self.sync_trigger_rate_limiter.cleanup().await; + self.peer_write_rate_limiter.cleanup().await; + } + /// Trigger graceful shutdown. Idempotent — calling more than once /// has no effect. Returns `true` if this call was the one that /// flipped the signal. @@ -255,6 +368,130 @@ impl AppState { pub fn is_shutting_down(&self) -> bool { *self.shutdown_tx.borrow() } + + /// Legacy-probe budget wired from the `GITLAWB_IPFS_MAX_LEGACY_PROBES` operator + /// knob. The knob seeds `ipfs_max_legacy_probes` at construction so it controls the + /// per-request legacy (NULL-provenance) probe fan-out it advertises. It deliberately + /// does NOT feed the history-walk ceiling: that is governed by + /// `ipfs_max_repos_walked` under a `MAX_PIN_SOURCES + 1` floor, because a value + /// below the floor truncates a provenanced request with a full source set into a + /// false 503. The knob is `usize`, the field `u32`; the range cap (1_048_576) keeps + /// the cast lossless. + pub(crate) fn ipfs_legacy_probe_budget(config: &crate::config::Config) -> u32 { + config.ipfs_max_legacy_probes as u32 + } + + /// Legacy-scan ROW budget wired from the `GITLAWB_IPFS_MAX_LEGACY_SCAN_ROWS` knob, + /// the same helper shape as the probe budget above so the knob cannot become a + /// silent no-op if a struct literal drifts back to the bare constant. + pub(crate) fn ipfs_legacy_scan_row_budget(config: &crate::config::Config) -> usize { + config.ipfs_max_legacy_scan_rows + } + + /// The key sealing legacy-scan continuation tokens (INV-13), DERIVED from the node's + /// persistent identity rather than minted per boot. + /// + /// HKDF-SHA256 over the node's Ed25519 seed, the same key material + /// `load_or_create_keypair` reads back from its PKCS#8 PEM on every boot, so the + /// derived key is byte-identical after a restart or a rolling deploy. + /// + /// Two properties the derivation has to carry, both load-bearing: + /// + /// * DOMAIN SEPARATION. The `info` string below is unique to this use, so the + /// derived key is not the signing seed and is not any other secret derived from + /// it. HKDF is one-way, so a leaked token key yields nothing about the signing + /// key and cannot be turned against a signature. + /// * A VERSION component, carried in the same `info` string + /// ([`SCAN_TOKEN_KEY_INFO`]) alongside a fixed [`SCAN_TOKEN_KEY_SALT`]. Bumping + /// the version rotates every token key on the next boot without touching the + /// identity, the token format, or any caller, so rotation is a constant change + /// rather than a fork of this derivation. + pub(crate) fn derive_scan_token_key(keypair: &Keypair) -> [u8; 32] { + use hmac::{Hmac, Mac}; + type HmacSha256 = Hmac; + + let seed = keypair.to_seed(); + // HKDF-Extract: PRK = HMAC(salt, ikm). + let mut extract = + HmacSha256::new_from_slice(SCAN_TOKEN_KEY_SALT).expect("HMAC takes a key of any size"); + extract.update(seed.as_slice()); + let prk = extract.finalize().into_bytes(); + // HKDF-Expand, one block: T(1) = HMAC(PRK, info || 0x01). One 32-byte output + // needs exactly one block of SHA-256, so there is no counter loop to get wrong. + let mut expand = + HmacSha256::new_from_slice(prk.as_slice()).expect("HMAC takes a key of any size"); + expand.update(SCAN_TOKEN_KEY_INFO); + expand.update(&[0x01]); + expand.finalize().into_bytes().into() + } + + /// Work-budget capacity for [`ipfs_work_rate_limiter`](Self#structfield.ipfs_work_rate_limiter) + /// (R6, KTD6), DERIVED from the route limit rather than a new operator knob. The route + /// limiter (`ipfs_rate_limiter`) charges once per request; this separate bucket absorbs + /// the resolver's per-probe/per-walk work charges so both the route "requests per hour" + /// contract and the amplification bound hold. Floor: at least one complete COMBINED + /// resolution per window, the provenance phase's walks plus a full legacy search + /// (probes plus page tolls), so a single default-config resolution cannot + /// self-throttle part-way and recreate the admit-then-429 for a legitimate caller. + /// `GITLAWB_IPFS_RATE_LIMIT=0` + /// disables the route brake and this derived bucket alike (a 0-capacity limiter admits + /// everything). + /// + /// The floor carries a WALK term (#173 round 15, F2). The provenance visibility walk + /// in `gate_and_serve` debits this SAME bucket (its `!legacy_scan` charge), once per + /// path-scoped source, before the legacy fallback runs at all, and the walk cap is + /// charged per phase. A floor counting only the search therefore under-sizes the + /// window by up to that cap exactly when the floor binds (a route limit set below + /// it): the provenance phase spends from the budget the floor reserved for the + /// search, the fallback 429s short of its configured reach, and the retry re-pays the + /// same provenance charges. The term is + /// `min(MAX_HISTORY_WALKS_PER_REQUEST, ipfs_max_repos_walked)` because that is what + /// the resolver's own `walk_cap` in `gate_and_serve` evaluates to. The two + /// expressions are separate, not one shared value: this one reads the CONSTANT + /// `crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST`, while `walk_cap` reads the + /// `AppState` seam `state.ipfs_max_history_walks`. They agree only because every + /// construction seeds that seam from that constant: the production one in `main.rs`, + /// and the two test ones in `auth`'s test module and `test_support`. Nothing + /// mechanically ties the two expressions beyond this comment and its counterpart at + /// `walk_cap` (no shared helper, no type, no assertion outside the one fixture that + /// pins the seam as a precondition), so the two `min()`s have to be moved together + /// by hand, and a construction that seeded the seam from anything else would size + /// the floor for a cap the resolver does not enforce. It reads the + /// CONSTANT and the CONFIG knob for the same reason the page term below reads the + /// constant page size: `ipfs_max_history_walks` is an `AppState` test seam, and + /// sizing a production floor from a seam would inflate the budget by whatever a test + /// chose. + /// + /// The PROBE term is the LEGACY-PROBE knob, not `ipfs_max_repos_walked`. Those were + /// one field before the walk cap and the probe budget were split apart, and sizing + /// the probe term off the walk cap would silently read 64 instead of 256. That is a + /// statement about which knob sizes the PROBE term; it is not a claim that the walk + /// cap has no place in the floor, since the walk term above is added to this one + /// rather than substituted for it. + /// + /// The floor also carries the scan's PAGE toll (#173 round 13, F2): every page the + /// legacy scan buys is charged to this same bucket, so a deep scan spends + /// `ceil(ipfs_max_legacy_scan_rows / LEGACY_SCAN_PAGE_ROWS)` tokens on pages on top + /// of its probes. Leaving those out would 429 an honest caller part-way down their + /// own continuation-token ladder, which is the F6 admit-then-429 shape in a new + /// place. The page term uses the CONSTANT page size, not `AppState`'s field: the + /// field is a test seam that shrinks pages to make paging observable, and sizing a + /// production floor from it would inflate the budget by whatever a test chose. + pub(crate) fn ipfs_work_budget(config: &crate::config::Config) -> usize { + if config.ipfs_rate_limit == 0 { + return 0; + } + let pages = config + .ipfs_max_legacy_scan_rows + .div_ceil(crate::api::ipfs::LEGACY_SCAN_PAGE_ROWS); + let walks = std::cmp::min( + crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST as usize, + config.ipfs_max_repos_walked, + ); + config + .ipfs_rate_limit + .max(config.ipfs_max_legacy_probes + pages + walks) + } } /// Bounds the OUTSTANDING post-push encryption-task set by per-repo coalescing @@ -1226,3 +1463,80 @@ mod repo_write_lease_tests { } } } + +#[cfg(test)] +mod scan_token_key_tests { + use super::AppState; + use gitlawb_core::identity::Keypair; + use gitlawb_core::scan_token::{open_scan_token, seal_scan_token, ScanPosition}; + + const CID: &str = "bafkreiscantokenkeyfixtureaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + + fn pos() -> ScanPosition { + ScanPosition { + created_at_key: "2020-01-01T12:00:00+00:00".to_string(), + id: "did:key:z6MkScanTokenOwner/repo".to_string(), + // 40 hex: the production shape, since the node's own repos are sha1. + sha256_hex: "a1b2c3d4e5f60718293a4b5c6d7e8f9012345678".to_string(), + } + } + + /// The RESTART case. A token minted before a restart must still open after it, or a + /// caller laddering a deep inventory is silently returned to the front of the scan on + /// every rolling deploy and can never reach a holder buried past one window. + /// + /// The second key is derived from the identity RELOADED THROUGH ITS ON-DISK PEM, + /// which is exactly what `load_or_create_keypair` does on boot, so this exercises the + /// real restart path rather than a clone of the in-memory keypair. + #[test] + fn scan_token_key_survives_a_restart_of_the_same_identity() { + let kp = Keypair::generate(); + let pem = kp.to_pem().expect("the identity serializes"); + let reloaded = Keypair::from_pem(&pem).expect("the identity reloads"); + + let before = AppState::derive_scan_token_key(&kp); + let after = AppState::derive_scan_token_key(&reloaded); + + let token = seal_scan_token(&before, CID, &pos(), i64::MAX - 1).expect("seal"); + assert_eq!( + open_scan_token(&after, CID, &token, 0), + Some(pos()), + "a continuation minted before a restart must open after it: the node's \ + identity is the same, so the derived sealing key must be too" + ); + } + + /// The must-not: a DIFFERENT node identity must derive a DIFFERENT key, so a token is + /// no more portable between nodes than it was when the key was random per boot. + #[test] + fn scan_token_key_does_not_open_under_a_different_identity() { + let mine = Keypair::generate(); + let theirs = Keypair::generate(); + + let token = seal_scan_token( + &AppState::derive_scan_token_key(&mine), + CID, + &pos(), + i64::MAX - 1, + ) + .expect("seal"); + assert_eq!( + open_scan_token(&AppState::derive_scan_token_key(&theirs), CID, &token, 0), + None, + "a continuation sealed by one node must not open under another node's identity" + ); + } + + /// Domain separation, executed rather than asserted in prose: the derived token key + /// must not be the signing seed itself. Compromising a token key must not hand an + /// attacker the material that signs. + #[test] + fn scan_token_key_is_not_the_signing_seed() { + let kp = Keypair::generate(); + assert_ne!( + AppState::derive_scan_token_key(&kp), + *kp.to_seed(), + "the token key must be a DERIVED secret, never the Ed25519 signing seed" + ); + } +} diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 2b5aef95..109e94d6 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -55,6 +55,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { use clap::Parser; let keypair = Keypair::generate(); + let scan_token_key = crate::state::AppState::derive_scan_token_key(&keypair); let node_did = keypair.did(); let (ref_tx, _) = tokio::sync::broadcast::channel(1); let (task_tx, _) = tokio::sync::broadcast::channel(1); @@ -78,6 +79,15 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { rate_limiter: RateLimiter::new(100, Duration::from_secs(60)), create_ip_rate_limiter: RateLimiter::new(1000, Duration::from_secs(3600)), push_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + ipfs_work_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + ipfs_max_history_walks: crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, + ipfs_max_legacy_probes: crate::api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST, + ipfs_legacy_scan_page_rows: crate::api::ipfs::LEGACY_SCAN_PAGE_ROWS, + ipfs_max_legacy_scan_rows: crate::api::ipfs::MAX_LEGACY_SCAN_ROWS_PER_REQUEST, + ipfs_max_legacy_scan_rule_bytes: crate::api::ipfs::MAX_LEGACY_SCAN_RULE_BYTES_PER_REQUEST, + ipfs_scan_token_key: Arc::new(scan_token_key), + ipfs_max_served_object_bytes: crate::api::ipfs::MAX_SERVED_OBJECT_BYTES, push_limiter_trust: crate::rate_limit::TrustedProxy::None, sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)), peer_write_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), @@ -100,7 +110,6 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { git_ipfs_walk_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( 16, ), - ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), git_bin: "git".to_string(), } } @@ -1953,6 +1962,127 @@ mod tests { ); } + /// A signed request body reused by the request-target trio below. Non-empty so + /// the content-digest the signature covers is a real hash rather than the + /// empty-body constant, which keeps `@path` the only component under test. + const TARGET_PIN_BODY: &[u8] = br#"{"task_type":"noop","payload":{}}"#; + + /// Send `body` to `uri` carrying a signature made over `signed_over`, through + /// the PRODUCTION router (`app`, which goes through `server::build_router`, where + /// `add_auth_layers` installs `require_signature` on the write routes). Returns + /// the status and the parsed JSON body (`Null` when the response is not JSON, as + /// a handler response past the middleware may be). Going through `app` rather + /// than a hand-mounted `Router::new().route(...)` probe is the point: a bare + /// router answers whether the middleware rejects the request, not whether that + /// is how a caller is actually gated. + async fn signed_over_then_sent( + pool: PgPool, + signed_over: &str, + uri: &str, + ) -> (StatusCode, serde_json::Value) { + use gitlawb_core::http_sig::sign_request; + use gitlawb_core::identity::Keypair; + + let kp = Keypair::generate(); + let signed = sign_request(&kp, "POST", signed_over, TARGET_PIN_BODY); + let req = Request::builder() + .method(Method::POST) + .uri(uri) + .header(axum::http::header::CONTENT_TYPE, "application/json") + .header("content-digest", signed.content_digest) + .header("signature-input", signed.signature_input) + .header("signature", signed.signature) + .body(Body::from(TARGET_PIN_BODY)) + .unwrap(); + + let resp = app(pool).await.oneshot(req).await.unwrap(); + let status = resp.status(); + let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .unwrap(); + let json = serde_json::from_slice(&bytes).unwrap_or(serde_json::Value::Null); + (status, json) + } + + /// The server half of the redirect finding: `require_signature` rebuilds `@path` + /// from the URI of the request it actually received, so a signature minted over + /// `/api/v1/repos` and presented at `POST /api/v1/tasks` verifies against the + /// wrong request-target and is refused 401 `invalid_signature`. Both routes sit + /// behind `add_auth_layers` in `build_router`, so the request really does reach + /// the middleware instead of 404ing at the fallback. This is the node-side proof + /// that a client which lets a redirect rewrite the target gets a 401, which is + /// what the production report showed. + /// + /// No pre-fix RED is obtainable here: the verifier already gates on `@path` (that + /// is precisely why the client bug surfaced as a 401 rather than as a silently + /// accepted request), so there is no broken state to observe first. The test is a + /// must-not guard, green by design, and its RED proof is by mutation of the + /// reconstruction it pins. + #[sqlx::test] + async fn require_signature_refuses_a_stale_request_target_path(pool: PgPool) { + let (status, json) = signed_over_then_sent(pool, "/api/v1/repos", "/api/v1/tasks").await; + assert_eq!( + status, + StatusCode::UNAUTHORIZED, + "a signature minted over one route path must not verify when replayed on another" + ); + assert_eq!( + json["error"], "invalid_signature", + "the refusal must come from the signature check, not from a handler or a later gate" + ); + } + + /// The query half of the same reconstruction: `@path` is path-and-query, not path + /// alone, so a signature minted over `/api/v1/tasks` and sent to + /// `/api/v1/tasks?x=1` is refused 401 `invalid_signature` too. Without this case a + /// reconstruction narrowed to `parts.uri.path()` would keep the sibling test above + /// green while admitting every query rewrite, so both components of the received + /// target are pinned rather than just the first. + /// + /// No pre-fix RED is obtainable here either, for the reason given on the sibling + /// above: the verifier already covers the query, so this is a green-by-design + /// must-not guard whose RED proof is by mutation. + #[sqlx::test] + async fn require_signature_refuses_a_stale_request_target_query(pool: PgPool) { + let (status, json) = + signed_over_then_sent(pool, "/api/v1/tasks", "/api/v1/tasks?x=1").await; + assert_eq!( + status, + StatusCode::UNAUTHORIZED, + "a signature minted over the bare task path \ + must not verify on a request that carries a query" + ); + assert_eq!( + json["error"], "invalid_signature", + "the query refusal must come from the signature check, \ + not from a handler rejecting the unknown parameter" + ); + } + + /// The paired positive control the two refusals need to mean anything: signed and + /// sent over the identical target `/api/v1/tasks?x=1`, the request clears + /// `require_signature`. Without it a reconstruction that produced garbage for + /// every request would satisfy both refusals above and look like coverage. The + /// request carries no `x-ucan` header, so `require_ucan_chain` passes it through + /// and whatever status arrives past the auth pair is the handler's own; the + /// assertion is therefore that the response is NOT the 401 `invalid_signature` the + /// mismatch cases get, not a pin on some particular handler outcome. + /// + /// Green by design like its siblings, and for the same reason: the verifier + /// already reconstructs the received target, so there is no pre-fix RED to + /// observe and the proof that this assertion is load-bearing comes from degrading + /// the reconstruction under mutation. + #[sqlx::test] + async fn require_signature_admits_the_exact_request_target(pool: PgPool) { + let (status, json) = + signed_over_then_sent(pool, "/api/v1/tasks?x=1", "/api/v1/tasks?x=1").await; + assert!( + !(status == StatusCode::UNAUTHORIZED && json["error"] == "invalid_signature"), + "an identically signed and sent request-target must clear require_signature, \ + so this control must not draw the same refusal as the mismatch cases; got {status}" + ); + } + /// Issue #6 / jatmn finding 2: `/api/v1/stats` counts logical repos, not raw /// rows. With a mirror+canonical pair and a standalone repo present, the /// `repos` count is 2. @@ -3136,13 +3266,19 @@ mod tests { /// Seed a SHA-256 source repo (public/a.txt + secret/b.txt), bare-clone it /// into each `/tmp//.git` path, and return guards + oids. - /// SHA-256 object format is required: `get_by_cid` resolves a CID whose - /// multihash digest IS the git object id, which only matches in sha256 repos. + /// SHA-256 object format matches production (`--object-format=sha256`) so the + /// oids are 64-hex. A real CID digests the raw object CONTENT (not the git + /// oid), so tests build the request CID with `pin_cid_for` — mirroring the pin + /// path — and `get_by_cid` maps it back to the oid via `pinned_cids` (#173). struct CidFixture { _guards: Vec, secret_oid: String, public_oid: String, secret_tree_oid: String, + public_tree_oid: String, + root_tree_oid: String, + commit_oid: String, + tag_oid: String, } impl Drop for CidFixture { fn drop(&mut self) { @@ -3176,6 +3312,8 @@ mod tests { run(&["config", "user.name", "t"], &src); run(&["add", "."], &src); run(&["commit", "-qm", "seed"], &src); + // Annotated tag of the commit — exercises the "tags stay served" guard. + run(&["tag", "-a", "-m", "annotated", "v1", "HEAD"], &src); let oid = |rev: &str| { let out = Command::new("git") .args(["rev-parse", rev]) @@ -3188,6 +3326,10 @@ mod tests { let secret_oid = oid("HEAD:secret/b.txt"); let public_oid = oid("HEAD:public/a.txt"); let secret_tree_oid = oid("HEAD:secret"); + let public_tree_oid = oid("HEAD:public"); + let root_tree_oid = oid("HEAD^{tree}"); + let commit_oid = oid("HEAD"); + let tag_oid = oid("refs/tags/v1"); let mut guards = vec![src.clone()]; for name in bare_names { let bare = std::path::PathBuf::from("/tmp") @@ -3205,6 +3347,12 @@ mod tests { ], &src, ); + // `git clone --bare` does NOT copy the source repo's local identity, so + // fixtures that create objects directly in the bare repo (`commit-tree`, + // `git tag -a`) abort with "identity unknown" on a CI runner that has no + // ambient/global git identity. Set it explicitly so the suite is portable. + run(&["config", "user.email", "t@t"], &bare); + run(&["config", "user.name", "t"], &bare); } // One guard for the whole /tmp/ tree covers every bare clone. guards.push(std::path::PathBuf::from("/tmp").join(slug)); @@ -3213,245 +3361,9664 @@ mod tests { secret_oid, public_oid, secret_tree_oid, + public_tree_oid, + root_tree_oid, + commit_oid, + tag_oid, } } - /// CID whose sha2-256 multihash digest equals the given 64-hex git oid, so - /// `get_by_cid` decodes it back to that oid and `git cat-file`s it. - fn cid_for_oid(oid_hex: &str) -> String { - use gitlawb_core::cid::Cid; - let bytes = hex::decode(oid_hex).expect("hex oid"); - let arr: [u8; 32] = bytes.as_slice().try_into().expect("32-byte sha256 oid"); - Cid::from_sha256_bytes(&arr).to_string() + /// Record a pin exactly as the production pin path does — read the object's + /// raw bytes (`git cat-file `, no framing), CID them with + /// `Cid::from_git_object_bytes`, and store the `(oid, cid)` row — then return + /// the CID string the node advertises (`gl ipfs list`) and a client sends to + /// `GET /ipfs/{cid}`. Building the CID from the oid instead (the old + /// `cid_for_oid`) produced an identifier that never occurs in production and + /// made the gate assertions vacuous: a real pin CID digests the raw content, + /// not the git oid, so `get_by_cid` resolves it through `pinned_cids` (#173). + async fn pin_cid_for(bare_repo: &std::path::Path, oid: &str, db: &crate::db::Db) -> String { + let (_ty, raw) = crate::git::store::read_object(bare_repo, oid) + .expect("read object bytes") + .expect("object exists in repo"); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(&raw).to_string(); + // Legacy-style pin (no provenance) so existing CID tests exercise the + // resolver's scan fallback; provenance-path tests pin via `pin_cid_for_repo`. + db.record_pinned_cid(oid, &cid, None) + .await + .expect("record pinned cid"); + cid } - fn cid_router(state: &AppState) -> Router { - Router::new() - .route( - "/ipfs/{cid}", - axum::routing::get(crate::api::ipfs::get_by_cid), - ) - .layer(axum::middleware::from_fn(crate::auth::optional_signature)) - .with_state(state.clone()) + /// Like [`pin_cid_for`] but records the pin's provenance (`repo_id`), so the + /// resolver resolves the CID straight to `repo_id` instead of scanning (#173). + #[allow(dead_code)] // used by the provenance-path resolver tests (P-U3) + async fn pin_cid_for_repo( + bare_repo: &std::path::Path, + oid: &str, + db: &crate::db::Db, + repo_id: &str, + ) -> String { + let (_ty, raw) = crate::git::store::read_object(bare_repo, oid) + .expect("read object bytes") + .expect("object exists in repo"); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(&raw).to_string(); + db.record_pinned_cid(oid, &cid, Some(repo_id)) + .await + .expect("record pinned cid with provenance"); + cid } - async fn cid_parts(resp: axum::response::Response) -> (StatusCode, String) { - let st = resp.status(); - let b = axum::body::to_bytes(resp.into_body(), usize::MAX) + + /// INV-7 upgrade path for the pin-provenance column (#173, jatmn round 2): a node + /// already past v11 gets `pinned_cids.repo_id` from the NEW v19 migration, and a + /// legacy pin recorded before the column existed survives with NULL provenance (so + /// it falls back to the repo scan). Simulate the pre-v19 node by dropping the + /// column and un-applying v12, seed a legacy row, then re-migrate. RED before the + /// v19 migration exists (the column is never re-added → the SELECT errors); GREEN + /// after. + #[sqlx::test] + async fn pinned_cids_repo_provenance_upgrade_path(pool: PgPool) { + let state = test_state(pool.clone()).await; + + // Pre-v19 shape: drop the provenance column and forget v19 was applied. + sqlx::query("ALTER TABLE pinned_cids DROP COLUMN IF EXISTS repo_id") + .execute(&pool) .await .unwrap(); - (st, String::from_utf8_lossy(&b).to_string()) - } - fn cid_anon(cid: &str) -> Request { - Request::builder() - .method(Method::GET) - .uri(format!("/ipfs/{cid}")) - .body(Body::empty()) - .unwrap() - } - fn cid_signed(kp: &gitlawb_core::identity::Keypair, cid: &str) -> Request { - let path = format!("/ipfs/{cid}"); - let s = gitlawb_core::http_sig::sign_request(kp, "GET", &path, b""); - Request::builder() - .method(Method::GET) - .uri(&path) - .header("content-digest", s.content_digest) - .header("signature-input", s.signature_input) - .header("signature", s.signature) - .body(Body::empty()) - .unwrap() + sqlx::query("DELETE FROM schema_migrations WHERE version = 19") + .execute(&pool) + .await + .unwrap(); + + // A legacy pin recorded before provenance existed. + sqlx::query("INSERT INTO pinned_cids (sha256_hex, cid, pinned_at) VALUES ($1, $2, $3)") + .bind("legacyoid") + .bind("legacycid") + .bind("2020-01-01T00:00:00Z") + .execute(&pool) + .await + .unwrap(); + + // Upgrade: re-run migrations → v19 re-adds the column. + state.db.run_migrations().await.expect("migrate to v12"); + + // The legacy pin survives with NULL provenance. + let legacy: Option = + sqlx::query_scalar("SELECT repo_id FROM pinned_cids WHERE sha256_hex = 'legacyoid'") + .fetch_one(&pool) + .await + .expect("legacy pin row survives the upgrade"); + assert!( + legacy.is_none(), + "a pin recorded before v19 must keep NULL provenance (it falls back to the scan)" + ); + + // A new pin can carry provenance. + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) VALUES ($1, $2, $3, $4)", + ) + .bind("newoid") + .bind("newcid") + .bind("2026-01-01T00:00:00Z") + .bind("repo-abc") + .execute(&pool) + .await + .unwrap(); + let prov: Option = + sqlx::query_scalar("SELECT repo_id FROM pinned_cids WHERE sha256_hex = 'newoid'") + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!( + prov.as_deref(), + Some("repo-abc"), + "a pin recorded after v19 carries its source repo_id" + ); } - /// #110: `GET /ipfs/{cid}` must gate a withheld blob by per-caller visibility. - /// RED before U2 (the current handler serves the secret to anon). + /// #173: a pin records the repository it came from; `provenance_for_oid` reads it + /// back; a legacy pin (no repo) reads back None; and first-pinner-owns holds — a + /// second push of the same oid does NOT rewrite provenance (ON CONFLICT DO + /// NOTHING). This is what lets the resolver gate a CID against its ONE source repo. #[sqlx::test] - async fn ipfs_cid_gate_withholds_blob_from_unauthorized(pool: PgPool) { - use crate::db::VisibilityMode; - use gitlawb_core::identity::Keypair; - - let owner = Keypair::generate(); - let owner_did = owner.did().to_string(); - let reader = Keypair::generate(); - let reader_did = reader.did().to_string(); - let stranger = Keypair::generate(); - let slug = owner_did.replace([':', '/'], "_"); - let short = owner_did.split(':').next_back().unwrap().to_string(); + async fn record_pinned_cid_stores_and_reads_provenance(pool: PgPool) { let state = test_state(pool).await; - let fx = seed_cid_repos(&slug, &short, &["withhold"]); - let secret_cid = cid_for_oid(&fx.secret_oid); - let tree_cid = cid_for_oid(&fx.secret_tree_oid); - let public_cid = cid_for_oid(&fx.public_oid); - state .db - .create_repo(&seed_repo(&owner_did, "withhold")) + .record_pinned_cid("oidA", "cidA", Some("repo-xyz")) .await - .expect("seed repo"); - let rec = state + .unwrap(); + assert_eq!( + state + .db + .provenance_for_oid("oidA") + .await + .unwrap() + .as_deref(), + Some("repo-xyz"), + "a provenanced pin reads back its source repo_id" + ); + + state .db - .get_repo(&owner_did, "withhold") + .record_pinned_cid("oidB", "cidB", None) .await - .unwrap() .unwrap(); + assert_eq!( + state.db.provenance_for_oid("oidB").await.unwrap(), + None, + "a legacy pin (no repo) has NULL provenance" + ); + + // First-pinner-owns: a later push of the same oid must not rewrite provenance. state .db - .set_visibility_rule( - &rec.id, - "/secret/**", - VisibilityMode::B, - std::slice::from_ref(&reader_did), - &owner_did, - ) + .record_pinned_cid("oidA", "cidA", Some("repo-OTHER")) .await - .expect("deny rule"); - - // anon → withheld blob: must 404, must not leak content. (RED on current handler.) - let (st, body) = cid_parts( - cid_router(&state) - .oneshot(cid_anon(&secret_cid)) - .await - .unwrap(), - ) - .await; + .unwrap(); assert_eq!( - st, - StatusCode::NOT_FOUND, - "anon must not read the withheld blob" - ); - assert!( - !body.contains("TOP SECRET"), - "404 body must not leak the secret" + state + .db + .provenance_for_oid("oidA") + .await + .unwrap() + .as_deref(), + Some("repo-xyz"), + "ON CONFLICT DO NOTHING keeps the first repo's provenance" ); - // signed non-reader → 404. - let (st, body) = cid_parts( - cid_router(&state) - .oneshot(cid_signed(&stranger, &secret_cid)) - .await - .unwrap(), - ) - .await; + // An unpinned oid has no provenance. assert_eq!( - st, - StatusCode::NOT_FOUND, - "non-reader must not read the withheld blob" + state.db.provenance_for_oid("never-pinned").await.unwrap(), + None ); - assert!(!body.contains("TOP SECRET")); - - // owner (signed) → 200 + secret bytes. - let (st, body) = cid_parts( - cid_router(&state) - .oneshot(cid_signed(&owner, &secret_cid)) - .await - .unwrap(), - ) - .await; - assert_eq!(st, StatusCode::OK, "owner reads the withheld blob"); - assert!(body.contains("TOP SECRET"), "owner gets the content"); + } - // listed reader (signed) → 200. - let (st, body) = cid_parts( - cid_router(&state) - .oneshot(cid_signed(&reader, &secret_cid)) - .await - .unwrap(), - ) - .await; - assert_eq!(st, StatusCode::OK, "listed reader reads the blob"); - assert!(body.contains("TOP SECRET")); + /// #173 (provenance, happy path): a CID pinned with provenance resolves straight + /// to its ONE source repo and serves an authorized reader — no repo scan. + #[sqlx::test] + async fn ipfs_cid_provenance_serves_from_pinning_repo(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; - // KTD3: anon tree CID under /secret → 200 (trees/commits are not withheld). - let (st, _) = cid_parts( - cid_router(&state) - .oneshot(cid_anon(&tree_cid)) - .await - .unwrap(), - ) - .await; - assert_eq!(st, StatusCode::OK, "tree object is served to anon (KTD3)"); + let _fx = seed_cid_repos(&slug, &short, &["provserve"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("provserve.git"); + let fx = &_fx; - // R3: public blob anon → 200 (non-withheld content not affected). - let (st, _) = cid_parts( - cid_router(&state) - .oneshot(cid_anon(&public_cid)) - .await - .unwrap(), - ) - .await; - assert_eq!(st, StatusCode::OK, "public blob stays served"); + // Build the repo FIRST so the pin can carry its id as provenance. + let repo = seed_repo(&owner_did, "provserve"); // public + state.db.create_repo(&repo).await.expect("seed repo"); + let cid = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, &repo.id).await; - // R5: a genuine unknown CID also 404, uniform with the withheld 404. - let absent_cid = cid_for_oid(&"ab".repeat(32)); - let (st, _) = cid_parts( - cid_router(&state) - .oneshot(cid_anon(&absent_cid)) - .await - .unwrap(), - ) - .await; + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; assert_eq!( st, - StatusCode::NOT_FOUND, - "absent CID 404 (uniform with withheld)" + StatusCode::OK, + "a provenanced public CID serves its content" + ); + assert!( + body.contains("public bytes"), + "the pinning repo's object is served" ); - - // malformed CID → 400 (unchanged). - let (st, _) = cid_parts( - cid_router(&state) - .oneshot(cid_anon("not-a-cid")) - .await - .unwrap(), - ) - .await; - assert_eq!(st, StatusCode::BAD_REQUEST, "malformed CID still 400"); } - /// R4: the same object withheld in one repo but public in another is still - /// served from the public copy; the withholding repo is iterated first. + /// #173 (provenance, THE load-bearing one — #124 flip + bounded fan-out): a CID + /// pinned from a PRIVATE repo must gate against that pinning repo (404), NOT serve + /// from a byte-identical PUBLIC copy in another repo. Provenance is strictly more + /// restrictive than the old scan (which served the public copy). RED before the + /// rework (the scan serves the public copy → 200 + leaks the secret bytes); GREEN + /// after (provenance → the private repo → 404, no leak). #[sqlx::test] - async fn ipfs_cid_served_from_public_copy_when_withheld_elsewhere(pool: PgPool) { - use crate::db::VisibilityMode; - use chrono::Utc; + async fn ipfs_cid_provenance_private_denies_despite_public_copy(pool: PgPool) { use gitlawb_core::identity::Keypair; - let owner = Keypair::generate(); let owner_did = owner.did().to_string(); let slug = owner_did.replace([':', '/'], "_"); let short = owner_did.split(':').next_back().unwrap().to_string(); let state = test_state(pool).await; - let fx = seed_cid_repos(&slug, &short, &["withhold", "pubcopy"]); - let secret_cid = cid_for_oid(&fx.secret_oid); + let fx = seed_cid_repos(&slug, &short, &["privsrc", "pubcopy"]); + let priv_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("privsrc.git"); - // Withholding repo, iterated FIRST (later updated_at; list_all_repos is DESC). - let mut withhold = seed_repo(&owner_did, "withhold"); - withhold.updated_at = Utc::now(); + // Private source repo, built first so the pin carries its id as provenance. + let mut priv_repo = seed_repo(&owner_did, "privsrc"); + priv_repo.is_public = false; state .db - .create_repo(&withhold) + .create_repo(&priv_repo) + .await + .expect("seed private repo"); + let cid = pin_cid_for_repo(&priv_bare, &fx.secret_oid, &state.db, &priv_repo.id).await; + + // A PUBLIC repo holds the SAME object (the old scan would serve it). + let pub_repo = seed_repo(&owner_did, "pubcopy"); // public, no rule + state + .db + .create_repo(&pub_repo) + .await + .expect("seed public copy"); + + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "a provenanced private CID must 404, not serve from a public copy elsewhere (#124 flip)" + ); + assert!( + !body.contains("TOP SECRET"), + "the 404 body must not leak the withheld object" + ); + } + + /// #173 (jatmn round 8, F1 — load-bearing): a shared object first pinned from a + /// PRIVATE repo, then pushed again from a PUBLIC repo through the real pin path, + /// must serve by CID to an anonymous caller from the public source. First-pinner- + /// only provenance 404s it (only the private source is known); recording EVERY + /// pin-path source fixes it. The second push hits the already-pinned skip branch, + /// so this proves the skip-branch source insert fires (and does NOT re-pin: /add + /// expect(0)). RED before U1 (anon 404); GREEN after. + #[sqlx::test] + async fn ipfs_cid_multi_source_serves_from_later_public_pinner(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["privfirst", "pubsecond"]); + let priv_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("privfirst.git"); + let pub_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("pubsecond.git"); + + // Private repo pins the object FIRST — it owns the first-pinner provenance. + let mut priv_repo = seed_repo(&owner_did, "privfirst"); + priv_repo.is_public = false; + state + .db + .create_repo(&priv_repo) + .await + .expect("seed private first-pinner"); + let cid = pin_cid_for_repo(&priv_bare, &fx.public_oid, &state.db, &priv_repo.id).await; + + // A PUBLIC repo pushes the SAME object through the real pin path. The object is + // already pinned, so this hits the already-pinned skip branch, which must record + // the public repo as an additional source without re-pinning (/add expect 0). + let pub_repo = seed_repo(&owner_did, "pubsecond"); // public, no rule + state + .db + .create_repo(&pub_repo) + .await + .expect("seed public second-pinner"); + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyshouldnothappen"}"#) + .expect(0) + .create_async() + .await; + crate::ipfs_pin::pin_new_objects( + &server.url(), + &pub_bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + vec![fx.public_oid.clone()], + &state.db, + &pub_repo.id, + crate::ipfs_pin::PIN_BATCH_BUDGET, + ) + .await; + m.assert_async().await; // asserts /add was NOT called (already pinned) + + // Anonymous CID fetch: the private first source denies, the public second + // source serves → 200. Before F1 only the private source is known → 404. + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::OK, + "a shared object must serve by CID from a later public pin-path source (F1)" + ); + assert!( + body.contains("public bytes"), + "the served body is the public object's bytes" + ); + } + + /// U1 (grok round-4 P1): `pin_sources_at_cap` flips exactly at `MAX_PIN_SOURCES`. + /// It is the signal `get_by_cid` uses to decide a provenance miss may be hiding a + /// dropped servable source and must fall back to the bounded scan. + #[sqlx::test] + async fn pin_sources_at_cap_flips_at_max(pool: PgPool) { + let state = test_state(pool).await; + let cap = crate::db::MAX_PIN_SOURCES; + assert!( + !state.db.pin_sources_at_cap("atcapoid").await.unwrap(), + "an oid with no pin_repo_sources rows is not at cap" + ); + for i in 0..(cap - 1) { + state + .db + .record_pin_source("atcapoid", &format!("r-{i:02}")) + .await + .unwrap(); + } + assert!( + !state.db.pin_sources_at_cap("atcapoid").await.unwrap(), + "one below MAX_PIN_SOURCES is not at cap" + ); + state + .db + .record_pin_source("atcapoid", "r-last") + .await + .unwrap(); + assert!( + state.db.pin_sources_at_cap("atcapoid").await.unwrap(), + "exactly MAX_PIN_SOURCES rows is at cap" + ); + } + + /// U2 (grok round-4 P1, load-bearing): the pin-source GRIEFING hole. A private + /// first-pinner denies anon; an attacker fills the whole `MAX_PIN_SOURCES` source + /// window with deny-anon sources BEFORE a legitimate public repo pins the same + /// object, so the public repo's `record_pin_source` no-ops (cap full) and it is + /// buried — present in NO provenance record. The resolver's provenance set is then + /// {private + 16 attacker}, all deny anon. Because the set is at_cap (may hide a + /// dropped source), the handler falls back to the bounded legacy scan, which gates + /// every repo through the real gate and finds the buried PUBLIC copy → 200. + /// MUTATION (RED): remove the `at_cap` fallback edge in `get_by_cid` and the buried + /// public object 404s forever. + #[sqlx::test] + async fn ipfs_cid_buried_public_source_still_serves_via_scan_fallback(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["privfirst", "pubburied"]); + let priv_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("privfirst.git"); + let pub_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("pubburied.git"); + + // Private repo pins FIRST — owns the first-pinner provenance, denies anon. + let mut priv_repo = seed_repo(&owner_did, "privfirst"); + priv_repo.is_public = false; + state + .db + .create_repo(&priv_repo) + .await + .expect("seed private first-pinner"); + let cid = pin_cid_for_repo(&priv_bare, &fx.public_oid, &state.db, &priv_repo.id).await; + + // Attacker fills the ENTIRE MAX_PIN_SOURCES window with deny-anon (non-existent) + // sources BEFORE the public repo registers, so the cap is full. + let cap = crate::db::MAX_PIN_SOURCES; + for i in 0..cap { + state + .db + .record_pin_source(&fx.public_oid, &format!("00-attacker-{i:02}")) + .await + .expect("attacker source"); + } + + // A PUBLIC repo pushes the SAME object through the real pin path. Already pinned + // (skip branch), so it only tries record_pin_source — which NO-OPS because the + // cap is full. The public repo is thus buried: not the first-pinner, not in + // pin_repo_sources. + let pub_repo = seed_repo(&owner_did, "pubburied"); // public, no rule + state + .db + .create_repo(&pub_repo) + .await + .expect("seed public buried source"); + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyshouldnothappen"}"#) + .expect(0) + .create_async() + .await; + crate::ipfs_pin::pin_new_objects( + &server.url(), + &pub_bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + vec![fx.public_oid.clone()], + &state.db, + &pub_repo.id, + crate::ipfs_pin::PIN_BATCH_BUDGET, + ) + .await; + m.assert_async().await; // /add NOT called (already pinned) + + // The buried public object must STILL serve: the provenance set is at_cap and + // all-deny, so the handler falls back to the bounded scan, which finds pubburied. + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::OK, + "a public source buried by a full attacker source window must still serve via the bounded scan fallback (F1)" + ); + assert!( + body.contains("public bytes"), + "the served body is the buried public object's bytes" + ); + } + + /// #173 (jatmn round 8, F1 — bound, R2): the per-object source set is capped at + /// `MAX_PIN_SOURCES` so an adversary pushing one object from many repos cannot make + /// resolution O(repos). Recording the same oid from `MAX_PIN_SOURCES + 3` distinct + /// repos leaves exactly `MAX_PIN_SOURCES` rows. + #[sqlx::test] + async fn ipfs_cid_pin_sources_capped_at_max(pool: PgPool) { + let state = test_state(pool).await; + let cap = crate::db::MAX_PIN_SOURCES; + for i in 0..(cap + 3) { + state + .db + .record_pin_source("capoid", &format!("repo-{i}")) + .await + .expect("record source"); + } + let sources = state.db.pin_sources_for_oid("capoid").await.unwrap(); + assert_eq!( + sources.len() as i64, + cap, + "the per-object source set is capped at MAX_PIN_SOURCES" + ); + } + + /// #173 (jatmn round 8, F1 — availability, grok-4.5 adversarial catch): the resolver's + /// per-object source cap must NEVER evict the first-pinner. A legacy public pin keeps + /// its source in `pinned_cids.repo_id` but not in `pin_repo_sources` (pre-v20 pins, or + /// a pin whose best-effort `record_pin_source` missed). If the cap `LIMIT` were applied + /// to the whole union with a lexicographic order, an attacker could push the same + /// object from `MAX_PIN_SOURCES` repos whose grindable ids sort before the public + /// source and evict it from the window — turning a public CID that served 200 into a + /// 404. This drives exactly that: a legacy public first-pinner plus `MAX_PIN_SOURCES` + /// lower-sorting attacker sources must STILL serve the public object. RED with a + /// whole-union LIMIT (the first-pinner is dropped → 404); GREEN once the first-pinner + /// is always included and the LIMIT caps only the additional sources. + #[sqlx::test] + async fn ipfs_cid_first_pinner_never_evicted_by_lower_sorting_sources(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["pubfirst"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("pubfirst.git"); + // Public repo whose id sorts AFTER every attacker id below. Legacy shape: the + // source lives in pinned_cids.repo_id only (pin_cid_for_repo records no + // pin_repo_sources row), exactly like a pin from before v13. + let mut pub_repo = seed_repo(&owner_did, "pubfirst"); // public, no rule + pub_repo.id = "zzzzzzzz-pubfirst".to_string(); + state + .db + .create_repo(&pub_repo) + .await + .expect("seed public first-pinner"); + let cid = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, &pub_repo.id).await; + + // Attacker fills the whole MAX_PIN_SOURCES window with lower-sorting source ids + // (non-existent repos — their mere presence would evict the first-pinner under a + // whole-union LIMIT). + let cap = crate::db::MAX_PIN_SOURCES; + for i in 0..cap { + state + .db + .record_pin_source(&fx.public_oid, &format!("00-attacker-{i:02}")) + .await + .expect("attacker source"); + } + + // The public first-pinner must still serve — never evicted by the cap window. + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::OK, + "the first-pinner public source must never be evicted by lower-sorting attacker sources (F1 availability)" + ); + assert!( + body.contains("public bytes"), + "the public object is served from the first-pinner" + ); + } + + /// INV-7 upgrade path for the F1 `pin_repo_sources` table (#173, jatmn round 8): a + /// node already past v19 gets the table from the NEW v20 migration. Simulate the + /// pre-v20 node by dropping the table and un-applying v13, then re-migrate and + /// assert a source row round-trips. RED before the v20 migration exists. + #[sqlx::test] + async fn pin_repo_sources_upgrade_path(pool: PgPool) { + let state = test_state(pool.clone()).await; + sqlx::query("DROP TABLE IF EXISTS pin_repo_sources") + .execute(&pool) + .await + .unwrap(); + sqlx::query("DELETE FROM schema_migrations WHERE version = 20") + .execute(&pool) + .await + .unwrap(); + state.db.run_migrations().await.expect("re-migrate"); + state + .db + .record_pin_source("upgradeoid", "repo-upg") + .await + .expect("record after re-migrate"); + assert_eq!( + state.db.pin_sources_for_oid("upgradeoid").await.unwrap(), + vec!["repo-upg".to_string()], + "the v20 pin_repo_sources table is present after upgrade" + ); + } + + // ── U3 (#173): durable pin-source incompleteness marker ────────────────── + // + // `record_pin_source` is best effort at every call site, so a non-empty, + // below-cap source set is NOT proof of completeness: an object first pinned + // from a PRIVATE repo and later pushed from a PUBLIC repo whose record failed + // has a set that names only the private source. The resolver used to treat + // that set as complete and 404 an object the public repo would serve. The + // pinned_cids.pin_sources_incomplete marker records the miss durably so the + // bounded scan fallback still runs. These tests drive both arms: the marker + // set (fallback runs, object serves, denial still denies) and the marker + // clear (ordinary denials stay off the O(repos) path, INV-10). + + /// Make `record_pin_source` fail for the duration of `body` by moving the + /// `pin_repo_sources` table out from under it, the closest honest stand-in for + /// the transient DB error the retry wrapper is there to absorb. Every other + /// pin-path query keeps working, so only the source record (and its retries) + /// fails, which is exactly the partial-record shape the finding turns on. + async fn with_pin_sources_broken(pool: &PgPool, body: F) -> T + where + F: FnOnce() -> Fut, + Fut: std::future::Future, + { + sqlx::query("ALTER TABLE pin_repo_sources RENAME TO pin_repo_sources_hidden") + .execute(pool) + .await + .expect("hide pin_repo_sources"); + let out = body().await; + sqlx::query("ALTER TABLE pin_repo_sources_hidden RENAME TO pin_repo_sources") + .execute(pool) + .await + .expect("restore pin_repo_sources"); + out + } + + /// Pin `oid` from `repo_id` through the real ipfs_pin path with a mock Kubo that + /// must NOT be called (the object is already pinned, so this drives the + /// skip-branch `record_pin_source` and nothing else). + async fn repin_via_skip_branch( + state: &AppState, + bare: &std::path::Path, + oid: &str, + repo_id: &str, + ) { + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyshouldnothappen"}"#) + .expect(0) + .create_async() + .await; + crate::ipfs_pin::pin_new_objects( + &server.url(), + bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + vec![oid.to_string()], + &state.db, + repo_id, + crate::ipfs_pin::PIN_BATCH_BUDGET, + ) + .await; + m.assert_async().await; + } + + /// U3 scenario 1 (#173, the finding's exact case): an object first pinned from a + /// PRIVATE repo, then pushed from a PUBLIC repo whose `record_pin_source` + /// exhausts its retries. The source set is non-empty and below cap, so the old + /// gate called it COMPLETE and 404'd an object the public repo would happily + /// serve. With the durable marker the bounded scan fallback still runs and the + /// public copy serves. RED before the marker (404); GREEN after (200). + #[sqlx::test] + async fn ipfs_cid_incomplete_source_set_falls_back_to_scan(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["u3priv", "u3pub"]); + let priv_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3priv.git"); + let pub_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3pub.git"); + + // Private first-pinner owns the only recorded source. + let mut priv_repo = seed_repo(&owner_did, "u3priv"); + priv_repo.is_public = false; + state + .db + .create_repo(&priv_repo) + .await + .expect("seed private"); + let cid = pin_cid_for_repo(&priv_bare, &fx.public_oid, &state.db, &priv_repo.id).await; + + // The PUBLIC repo holds the same object, but its source record never lands. + let pub_repo = seed_repo(&owner_did, "u3pub"); // public, no rule + state.db.create_repo(&pub_repo).await.expect("seed public"); + with_pin_sources_broken(&pool, || { + repin_via_skip_branch(&state, &pub_bare, &fx.public_oid, &pub_repo.id) + }) + .await; + + // The recorded set still names only the private repo, and it is below cap. + assert_eq!( + state.db.pin_sources_for_oid(&fx.public_oid).await.unwrap(), + vec![priv_repo.id.clone()], + "the public source really did fail to record" + ); + assert!( + !state.db.pin_sources_at_cap(&fx.public_oid).await.unwrap(), + "the set is below cap, so at_cap cannot be what triggers the fallback" + ); + + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::OK, + "a KNOWN-incomplete source set must keep the scan fallback so the public copy serves" + ); + assert!( + body.contains("public bytes"), + "the served body is the public object's bytes" + ); + } + + /// U3 scenario 2 (#173, INV-10 guard): the marker must not turn ORDINARY denials + /// into an O(repos) fan-out. With the marker false, a non-empty below-cap source + /// set and a provenance miss, the request must 404 WITHOUT the scan preload ever + /// running. The preload counter is the both-ways proof: forcing the marker true + /// unconditionally turns this red (count 1), which is what keeps the assertion + /// from being vacuous. + #[sqlx::test] + async fn ipfs_cid_complete_source_set_never_preloads(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["u3only"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3only.git"); + + // One PRIVATE source, recorded cleanly: the set is complete and below cap. + let mut priv_repo = seed_repo(&owner_did, "u3only"); + priv_repo.is_public = false; + state + .db + .create_repo(&priv_repo) + .await + .expect("seed private"); + let cid = pin_cid_for_repo(&bare, &fx.secret_oid, &state.db, &priv_repo.id).await; + state + .db + .record_pin_source(&fx.secret_oid, &priv_repo.id) + .await + .expect("record source"); + assert!( + !state + .db + .pin_sources_incomplete(&fx.secret_oid) + .await + .unwrap(), + "a clean record leaves the set marked complete" + ); + assert!( + !state.db.pin_sources_at_cap(&fx.secret_oid).await.unwrap(), + "the set is below cap, so at_cap cannot be what drives the gate" + ); + + crate::api::ipfs::reset_preload_queries(); + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "an anonymous caller denied by the only recorded source gets the opaque 404" + ); + assert!( + !body.contains("TOP SECRET"), + "the 404 body must not leak the withheld object" + ); + assert_eq!( + crate::api::ipfs::preload_queries(), + 0, + "an ordinary denial against a COMPLETE source set must never run the O(repos) preload (INV-10)" + ); + } + + /// U3 scenario 3 (#173): the marker is not permanent. Once a later + /// `record_pin_source` for the object succeeds, nothing is missing, so the marker + /// clears and the scan stops being triggered. BOTH sources here are private, so the + /// provenance walk MISSES and the request actually reaches the `needs_scan` gate: + /// with a marker left stuck the gate arms the O(repos) preload for an ordinary + /// denial forever. Drop the clear and both halves go red (marker still true, preload + /// 1). A public second source would make the preload half vacuous, because the + /// provenance path serves and returns before the gate is ever evaluated. + #[sqlx::test] + async fn ipfs_cid_marker_clears_on_a_later_successful_record(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["u3cfirst", "u3csecond"]); + let first_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3cfirst.git"); + let second_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3csecond.git"); + + let mut first_repo = seed_repo(&owner_did, "u3cfirst"); + first_repo.is_public = false; + state.db.create_repo(&first_repo).await.expect("seed first"); + let cid = pin_cid_for_repo(&first_bare, &fx.secret_oid, &state.db, &first_repo.id).await; + let mut second_repo = seed_repo(&owner_did, "u3csecond"); + second_repo.is_public = false; + state + .db + .create_repo(&second_repo) + .await + .expect("seed second"); + + // First push from the second repo: the source record fails, so the set is marked. + with_pin_sources_broken(&pool, || { + repin_via_skip_branch(&state, &second_bare, &fx.secret_oid, &second_repo.id) + }) + .await; + assert!( + state + .db + .pin_sources_incomplete(&fx.secret_oid) + .await + .unwrap(), + "the exhausted record marked the set incomplete" + ); + + // A later push from the same repo records cleanly, so nothing is missing. + repin_via_skip_branch(&state, &second_bare, &fx.secret_oid, &second_repo.id).await; + assert!( + !state + .db + .pin_sources_incomplete(&fx.secret_oid) + .await + .unwrap(), + "a successful record clears the marker" + ); + assert_eq!( + state + .db + .pin_sources_for_oid(&fx.secret_oid) + .await + .unwrap() + .len(), + 2, + "the repaired set really does name both sources" + ); + + crate::api::ipfs::reset_preload_queries(); + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "both sources are private, so the anonymous caller is denied" + ); + assert!( + !body.contains("TOP SECRET"), + "the 404 body must not leak the withheld object" + ); + assert_eq!( + crate::api::ipfs::preload_queries(), + 0, + "a repaired source set stops triggering the scan: the denial is back off the O(repos) path" + ); + } + + /// F5 (#173 round 11): the work-budget peek sheds an already-throttled caller BEFORE + /// the two marker queries, so a spent-budget source stops paying two lookups per + /// request for a scan it will never be allowed to run. The source set here is + /// non-empty and complete, which is the case that used to reach the queries anyway. + /// The counter is the both-ways guard: moving the peek back below the pair reads 1. + #[sqlx::test] + async fn ipfs_cid_throttled_caller_sheds_before_the_marker_queries(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + // One PRIVATE source, recorded cleanly: the set is non-empty, below cap and + // unmarked, so nothing but the peek can keep the request off the queries. + let fx = seed_cid_repos(&slug, &short, &["f5only"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("f5only.git"); + let mut repo = seed_repo(&owner_did, "f5only"); + repo.is_public = false; + state.db.create_repo(&repo).await.expect("seed private"); + let cid = pin_cid_for_repo(&bare, &fx.secret_oid, &state.db, &repo.id).await; + state + .db + .record_pin_source(&fx.secret_oid, &repo.id) + .await + .expect("record source"); + + // Spend the caller's whole work budget before the request. + assert!( + state.ipfs_work_rate_limiter.check("9.9.9.9").await, + "the budget starts with room" + ); + + crate::api::ipfs::reset_marker_queries(); + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon_xff(&cid, "9.9.9.9")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::TOO_MANY_REQUESTS, + "a spent-budget caller is shed at the peek" + ); + assert!( + !body.contains("TOP SECRET"), + "the shed response must not leak the withheld object" + ); + assert_eq!( + crate::api::ipfs::marker_queries(), + 0, + "a shed caller pays neither marker query" + ); + } + + /// U3 scenario 6 (#173, regression): a record that inserts NOTHING must not clear + /// the marker. `record_pin_source` is called for EVERY already-pinned object on the + /// skip path, and on a requeue pass that is the whole-repo enumeration, so the next + /// coalesced push from a repo ALREADY in the source set re-runs the insert as a + /// no-op. Clearing on that no-op re-hides the hole a different repo's failed record + /// recorded: the public copy stops being scanned for and 404s again. The assertion + /// is the SERVE outcome, not the column, so it still bites if the resolver ever + /// stops consulting the marker. RED before the rows_affected gate (404); GREEN after. + #[sqlx::test] + async fn ipfs_cid_noop_record_must_not_clear_the_marker(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["u3npriv", "u3npub"]); + let priv_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3npriv.git"); + let pub_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3npub.git"); + + // Repo A (private) is the first pinner AND is already recorded as a source, so a + // later record from A is a pure no-op insert. + let mut priv_repo = seed_repo(&owner_did, "u3npriv"); + priv_repo.is_public = false; + state + .db + .create_repo(&priv_repo) + .await + .expect("seed private"); + let cid = pin_cid_for_repo(&priv_bare, &fx.public_oid, &state.db, &priv_repo.id).await; + state + .db + .record_pin_source(&fx.public_oid, &priv_repo.id) + .await + .expect("record the first pinner as a source"); + + // Repo B (public) holds the same object, but its source record never lands, so + // the node marks the set known-incomplete. + let pub_repo = seed_repo(&owner_did, "u3npub"); + state.db.create_repo(&pub_repo).await.expect("seed public"); + with_pin_sources_broken(&pool, || { + repin_via_skip_branch(&state, &pub_bare, &fx.public_oid, &pub_repo.id) + }) + .await; + assert!( + state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "the exhausted record marked the set incomplete" + ); + + // A pushes again. The insert affects zero rows (A is already a source), so it + // recorded nothing and must not claim the set is complete. + repin_via_skip_branch(&state, &priv_bare, &fx.public_oid, &priv_repo.id).await; + assert_eq!( + state.db.pin_sources_for_oid(&fx.public_oid).await.unwrap(), + vec![priv_repo.id.clone()], + "the re-push really did add no source" + ); + + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::OK, + "a no-op record must not clear the marker: the public copy still has to serve" + ); + assert!( + body.contains("public bytes"), + "the served body is the public object's bytes" + ); + } + + /// U3 residual, CLOSED (#173 round 12): the incompleteness marker is per + /// `(object, repo)`, so a GENUINE record from a third repo C no longer clears the + /// marker repo B's FAILED record set, and the resolver keeps the scan fallback that + /// finds B's unrecorded public copy. + /// + /// This test asserted the opposite until the marker moved to `pin_source_failures`. + /// It was written as a deliberate pin on an accepted cost of the single boolean, with + /// a note saying that implementing the per-(oid, repo) marker should turn it red and + /// that it should then be updated rather than deleted. That is what happened, so the + /// assertions are inverted here and the fixture is unchanged. + /// + /// The BEFORE request is what makes the AFTER assertion mean anything: it proves the + /// unrecorded public holder IS reachable while the marker stands, so an AFTER 404 + /// would be caused by the clear and by nothing else in the fixture. The window this + /// covers is exactly `1 <= sources < MAX_PIN_SOURCES`: an empty set always scans, and + /// at cap the insert is a no-op so the marker survives regardless. + #[sqlx::test] + async fn ipfs_cid_third_repo_record_keeps_the_marker_and_still_serves_an_unrecorded_holder( + pool: PgPool, + ) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["u3ta", "u3tb", "u3tc"]); + let a_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3ta.git"); + let b_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3tb.git"); + let c_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3tc.git"); + + // Repo A (private) is the first pinner and the only recorded source. + let mut a_repo = seed_repo(&owner_did, "u3ta"); + a_repo.is_public = false; + state.db.create_repo(&a_repo).await.expect("seed A private"); + let cid = pin_cid_for_repo(&a_bare, &fx.public_oid, &state.db, &a_repo.id).await; + state + .db + .record_pin_source(&fx.public_oid, &a_repo.id) + .await + .expect("record the first pinner as a source"); + + // Repo B (public) genuinely holds the object, but its source record never lands, + // so the node marks the set known-incomplete. + let b_repo = seed_repo(&owner_did, "u3tb"); // public, no rule + state.db.create_repo(&b_repo).await.expect("seed B public"); + with_pin_sources_broken(&pool, || { + repin_via_skip_branch(&state, &b_bare, &fx.public_oid, &b_repo.id) + }) + .await; + assert!( + state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "B's exhausted record marked the set incomplete" + ); + + // BEFORE: while the marker stands, the scan fallback finds B and serves. + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::OK, + "with the marker set, the unrecorded public holder is reachable" + ); + assert!( + body.contains("public bytes"), + "the served body is the public object's bytes" + ); + + // Repo C (private) pushes the same object. Its record is a GENUINE insert (C is + // not yet a source), and it must NOT clear the marker B set, because B is still + // missing and C's record says nothing about B. + let mut c_repo = seed_repo(&owner_did, "u3tc"); + c_repo.is_public = false; + state.db.create_repo(&c_repo).await.expect("seed C private"); + repin_via_skip_branch(&state, &c_bare, &fx.public_oid, &c_repo.id).await; + + assert!( + state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "a third repo's record clears only its own pair, so B's marker survives" + ); + assert!( + !state.db.pin_sources_at_cap(&fx.public_oid).await.unwrap(), + "the set is below cap, so at_cap is not what drives the gate here" + ); + let sources = state.db.pin_sources_for_oid(&fx.public_oid).await.unwrap(); + assert_eq!(sources.len(), 2, "the set names A and C only: {sources:?}"); + assert!( + !sources.contains(&b_repo.id), + "B is still missing from the set it was marked for: {sources:?}" + ); + + // AFTER: the identical request still serves B's copy, because the surviving + // marker keeps the fallback armed. The scan is asserted to have RUN, so the 200 + // is the fallback finding B and not the provenance loop reaching it some other way. + crate::api::ipfs::reset_preload_queries(); + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::OK, + "B's marker survived C's record, so the unrecorded public holder is still reachable" + ); + assert!( + body.contains("public bytes"), + "the served body is the public object's bytes" + ); + assert!( + crate::api::ipfs::preload_queries() > 0, + "the fallback scan ran, so the 200 came from the armed fallback" + ); + } + + /// U3 scenario 4 (#173): the marker tracks the record's OUTCOME, not the attempt. + /// An exhausted retry sets it; a first-attempt success never does. Without the + /// second arm the first could be satisfied by marking unconditionally. + #[sqlx::test] + async fn pin_sources_incomplete_marks_only_exhausted_records(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["u3mark"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3mark.git"); + let repo = seed_repo(&owner_did, "u3mark"); + state.db.create_repo(&repo).await.expect("seed repo"); + let _ = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, &repo.id).await; + + // Arm A: a first-attempt success must leave the marker alone. + repin_via_skip_branch(&state, &bare, &fx.public_oid, &repo.id).await; + assert!( + !state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "a record that lands on the first attempt never marks the set incomplete" + ); + + // Arm B: an exhausted retry marks it. + with_pin_sources_broken(&pool, || { + repin_via_skip_branch(&state, &bare, &fx.public_oid, &repo.id) + }) + .await; + assert!( + state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "an exhausted record marks the set incomplete" + ); + + // An unpinned oid has no row and must read as complete, never as missing. + assert!( + !state + .db + .pin_sources_incomplete(&"f".repeat(64)) + .await + .unwrap(), + "an unpinned oid reads complete, so an unknown CID cannot arm the fallback" + ); + } + + /// #173 round 12 (jatmn): the incompleteness marker is per `(object, repo)`, so a + /// record from an UNRELATED repo does not clear a marker a different repo's failed + /// record set. It was one boolean per object, and the resolver reads a cleared marker + /// as "every source is recorded", drops the scan fallback, and 404s an anonymous + /// caller whose only servable copy is the unrecorded public one. + /// + /// Both directions, because the precision is the point: an unrelated repo must NOT + /// clear, and the repo that actually failed MUST clear, or every transient DB blip + /// would strand an object on the scan path forever. + #[sqlx::test] + async fn pin_source_failure_is_cleared_only_by_the_repo_that_failed(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["u3perrepo"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3perrepo.git"); + let repo_a = seed_repo(&owner_did, "u3perrepo"); + state.db.create_repo(&repo_a).await.expect("seed repo A"); + let repo_b = seed_repo(&owner_did, "u3perrepo-b"); + state.db.create_repo(&repo_b).await.expect("seed repo B"); + let repo_c = seed_repo(&owner_did, "u3perrepo-c"); + state.db.create_repo(&repo_c).await.expect("seed repo C"); + let _ = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, &repo_a.id).await; + + // Repo B's record fails: the object is now known to be missing B as a source. + state + .db + .mark_pin_sources_incomplete(&fx.public_oid, &repo_b.id) + .await + .expect("mark B's failure"); + assert!( + state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "B's failed record marks the set incomplete" + ); + + // A genuine record from an UNRELATED repo C. B is still missing. + state + .db + .record_pin_source(&fx.public_oid, &repo_c.id) + .await + .expect("record C"); + assert!( + state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "a record from an unrelated repo must not clear a marker another repo set: \ + the resolver would drop the scan fallback while B's copy is still unrecorded" + ); + + // The repo that actually failed lands its record: now the set is complete. + state + .db + .record_pin_source(&fx.public_oid, &repo_b.id) + .await + .expect("record B"); + assert!( + !state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "the repo whose record failed clears its own marker once it lands" + ); + } + + /// U3 scenario 5 (#173): the Pinata pin path had BARE `record_pin_source` calls, so + /// one transient DB error dropped a source permanently. It now shares the ipfs_pin + /// retry helper and marks/clears the same marker. The elapsed-time assertion is the + /// retry proof: a bare call returns immediately, whereas the wrapper sleeps + /// `PIN_RECORD_BACKOFF` between each of `PIN_RECORD_ATTEMPTS` tries. + #[sqlx::test] + async fn pinata_pin_path_retries_and_marks_incomplete(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["u3pinata"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3pinata.git"); + let repo = seed_repo(&owner_did, "u3pinata"); + state.db.create_repo(&repo).await.expect("seed repo"); + + // Already carries a pinata_cid, so pin_new_objects takes the skip branch and the + // only DB write under test is the source record. + let (_ty, raw) = crate::git::store::read_object(&bare, &fx.public_oid) + .unwrap() + .expect("object readable"); + let raw_cid = gitlawb_core::cid::Cid::from_git_object_bytes(&raw).to_string(); + state + .db + .record_pinata_cid(&fx.public_oid, &raw_cid, "QmProvider", Some(&repo.id)) + .await + .expect("seed pinata pin"); + + let client = reqwest::Client::new(); + let run = |db_broken: bool| { + let client = client.clone(); + let bare = bare.clone(); + let oid = fx.public_oid.clone(); + let repo_id = repo.id.clone(); + let state = &state; + async move { + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Any) + .with_status(200) + .with_body(r#"{"data":{"cid":"QmShouldNotHappen"}}"#) + .expect(0) + .create_async() + .await; + let started = std::time::Instant::now(); + crate::pinata::pin_new_objects( + &client, + &server.url(), + "test-jwt", + &bare, + "git", + // Generous: this test measures the record retry backoff, not the + // read bound, so the git_timeout must never be what fires. + std::time::Duration::from_secs(60), + vec![oid], + &state.db, + &repo_id, + // Far above the ~150ms the retry ladder spends, so the batch gate + // never truncates the one object under test: what is being measured + // is the retry backoff, not the budget. + std::time::Duration::from_secs(60), + ) + .await; + m.assert_async().await; // the upload is skipped: DB-only path + let _ = db_broken; + started.elapsed() + } + }; + + // Failing arm: retried (so it sleeps the full backoff horizon) and marked. + let elapsed = with_pin_sources_broken(&pool, || run(true)).await; + assert!( + elapsed >= std::time::Duration::from_millis(100), + "the pinata source record now RETRIES (bare call returns at once, got {elapsed:?})" + ); + assert!( + state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "an exhausted pinata record marks the set incomplete, same as the ipfs_pin path" + ); + + // Recovery arm: a later successful pinata record clears it, same as ipfs_pin. + run(false).await; + assert!( + !state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "a successful pinata record clears the marker" + ); + assert_eq!( + state.db.pin_sources_for_oid(&fx.public_oid).await.unwrap(), + vec![repo.id.clone()], + "the recovered record actually landed the source row" + ); + } + + /// A Pinata upload mock that must never fire, for the skip-branch tests below: an + /// object already carrying a `pinata_cid` is skipped before the upload, so a call + /// here means the branch under test was not the one taken. + async fn pinata_upload_mock_never(server: &mut mockito::ServerGuard) -> mockito::Mock { + server + .mock("POST", mockito::Matcher::Any) + .with_status(200) + .with_body(r#"{"data":{"cid":"QmShouldNotHappen"}}"#) + .expect(0) + .create_async() + .await + } + + /// The raw-content resolver key for an object in a bare repo, computed the way the + /// pin path computes it. + fn raw_key_for(bare: &std::path::Path, oid: &str) -> String { + gitlawb_core::cid::Cid::from_git_object_bytes( + &crate::git::store::read_object(bare, oid) + .expect("read object bytes") + .expect("object exists") + .1, + ) + .to_string() + } + + /// Seed a `pinned_cids` row by hand: the production helpers always store the raw + /// key, so a legacy provider-CID row can only be written with raw SQL. + async fn seed_pinned_row( + pool: &PgPool, + oid: &str, + cid: &str, + pinata_cid: Option<&str>, + repo_id: &str, + ) { + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, pinata_cid, repo_id) + VALUES ($1, $2, $3, $4, $5)", + ) + .bind(oid) + .bind(cid) + .bind("2020-01-01T00:00:00Z") + .bind(pinata_cid) + .bind(repo_id) + .execute(pool) + .await + .expect("seed pinned_cids row"); + } + + /// The stored resolver key and the stashed old provider value for a pinned object. + async fn stored_key_and_stash(pool: &PgPool, oid: &str) -> (String, Option) { + sqlx::query_as("SELECT cid, legacy_provider_cid FROM pinned_cids WHERE sha256_hex = $1") + .bind(oid) + .fetch_one(pool) + .await + .expect("the pinned row exists") + } + + /// The skip-branch repair runs while the caller holds a `pin_semaphore` permit, so it + /// must be bounded by the BATCH deadline and not by `git_timeout` alone. + /// `repair_legacy_provider_cid` builds its own deadline, and at shipped defaults that + /// is `git_service_timeout_secs` (600s) against a `PIN_BATCH_BUDGET` of 120s: one + /// legacy row whose `cat-file` wedges would hold a GLOBAL pin slot for five times the + /// budget the batch is supposed to cost, starving every other repo's pin work. The + /// loop's own budget gate cannot help, since it only runs at the top of the NEXT + /// iteration and cannot preempt a call already in flight. + /// + /// A wedged `cat-file`, a generous 60s `git_timeout`, and a 2s batch budget: the call + /// must return on the batch order. Both pin-permit-holding callers of the repair share + /// this clamp; the boot sweep keeps the plain `git_timeout`, since it holds no permit + /// and has no batch to overrun. + /// + /// REVERT PROOF (RED): pass `Instant::now() + git_timeout` to the repair instead of the + /// batch-clamped deadline and the wedged child runs the full 60s, blowing the outer + /// timeout below. + #[cfg(unix)] + #[sqlx::test] + async fn pinata_skip_branch_repair_is_bounded_by_the_batch_deadline(pool: PgPool) { + let state = test_state(pool.clone()).await; + let fx = seed_cid_repos("pinatabound", "pb", &["pinsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join("pinatabound") + .join("pinsrc.git"); + + // Resolve the real key with the real git BEFORE the fake is wired in, so the row + // is genuinely legacy-shaped and the repair has real work to attempt. + let raw_cid = raw_key_for(&bare, &fx.public_oid); + let provider_cid = legacy_dagpb_cid(&raw_cid); + seed_pinned_row( + &pool, + &fx.public_oid, + &provider_cid, + Some("QmPinataProvider"), + "repoPinataBound", + ) + .await; + + // `cat-file` never answers and ignores SIGTERM, so only the watchdog's group + // SIGKILL at the deadline can end it. Which deadline that is, is the whole test. + let tmp = tempfile::TempDir::new().unwrap(); + let fake = tmp.path().join("wedged-git"); + std::fs::write( + &fake, + "#!/bin/sh\ntrap '' TERM\ncase \"$1\" in\n cat-file) sleep 60 ;;\n *) : ;;\nesac\nexit 0\n", + ) + .unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&fake).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&fake, perm).unwrap(); + } + + let mut server = mockito::Server::new_async().await; + let m = pinata_upload_mock_never(&mut server).await; + let client = reqwest::Client::new(); + let started = std::time::Instant::now(); + tokio::time::timeout( + std::time::Duration::from_secs(25), + crate::pinata::pin_new_objects( + &client, + &server.url(), + "test-jwt", + &bare, + fake.to_str().unwrap(), + // Generous: if the call ends on time it ended on the batch deadline. + std::time::Duration::from_secs(60), + vec![fx.public_oid.clone()], + &state.db, + "repoPinataBound", + // The bound under test. + std::time::Duration::from_secs(2), + ), + ) + .await + .expect( + "a wedged skip-branch repair must be reaped on the batch deadline, not held for \ + the whole git_timeout while it pins a global pin permit", + ); + m.assert_async().await; + + let elapsed = started.elapsed(); + assert!( + elapsed < std::time::Duration::from_secs(20), + "elapsed {elapsed:?} must stay in the 2s batch-budget order (plus one watchdog \ + teardown), not the 60s git_timeout order" + ); + } + + /// The ipfs_pin twin of the clamp above, and the one that has been shipping: the Kubo + /// skip branch has always called the repair with a bare `git_timeout` while holding the + /// pin permit. Same wedged `cat-file`, same 60s `git_timeout` against a 2s batch + /// budget, same requirement that the call return on the batch order. + /// + /// REVERT PROOF (RED): drop the `min(deadline, ...)` clamp at the ipfs_pin skip-branch + /// call and this blows its outer timeout. + #[cfg(unix)] + #[sqlx::test] + async fn kubo_skip_branch_repair_is_bounded_by_the_batch_deadline(pool: PgPool) { + let state = test_state(pool.clone()).await; + let fx = seed_cid_repos("kubobound", "kb", &["pinsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join("kubobound") + .join("pinsrc.git"); + + let raw_cid = raw_key_for(&bare, &fx.public_oid); + let provider_cid = legacy_dagpb_cid(&raw_cid); + // A row in pinned_cids makes `is_pinned` true, so the Kubo loop takes the skip + // branch and reaches the repair without ever attempting an add. + seed_pinned_row(&pool, &fx.public_oid, &provider_cid, None, "repoKuboBound").await; + + let tmp = tempfile::TempDir::new().unwrap(); + let fake = tmp.path().join("wedged-git"); + std::fs::write( + &fake, + "#!/bin/sh\ntrap '' TERM\ncase \"$1\" in\n cat-file) sleep 60 ;;\n *) : ;;\nesac\nexit 0\n", + ) + .unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&fake).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&fake, perm).unwrap(); + } + + let started = std::time::Instant::now(); + tokio::time::timeout( + std::time::Duration::from_secs(25), + crate::ipfs_pin::pin_new_objects( + // Empty endpoint would return before the loop, so point at a closed port: + // the skip branch is reached and no add is ever attempted anyway. + "http://127.0.0.1:9", + &bare, + fake.to_str().unwrap(), + std::time::Duration::from_secs(60), + vec![fx.public_oid.clone()], + &state.db, + "repoKuboBound", + std::time::Duration::from_secs(2), + ), + ) + .await + .expect( + "a wedged skip-branch repair must be reaped on the batch deadline, not held for \ + the whole git_timeout while it pins a global pin permit", + ); + + let elapsed = started.elapsed(); + assert!( + elapsed < std::time::Duration::from_secs(20), + "elapsed {elapsed:?} must stay in the 2s batch-budget order, not the 60s \ + git_timeout order" + ); + } + + /// U3 scenario 1 (#173, Finding 2 lockstep): the PINATA skip branch runs the same + /// opportunistic legacy provider-CID repair the ipfs_pin skip branch runs. A row keyed + /// on a legacy provider CID that already carries a `pinata_cid` (so `has_pinata_cid` + /// answers true and the skip branch is taken) is rewritten to the raw-content resolver + /// key, stashing the old provider value in `legacy_provider_cid`. + /// + /// This drives `pinata::pin_new_objects`, never the ipfs_pin twin: the repair call + /// being PRESENT in `pinata.rs` proves nothing, only its execution through this lane + /// does. RED before the skip-branch call lands (the key stays the provider CID). + #[sqlx::test] + async fn pinata_skip_branch_repairs_legacy_provider_cid(pool: PgPool) { + let state = test_state(pool.clone()).await; + let fx = seed_cid_repos("pinatarepair", "pr", &["pinsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join("pinatarepair") + .join("pinsrc.git"); + + let raw_cid = raw_key_for(&bare, &fx.public_oid); + let provider_cid = legacy_dagpb_cid(&raw_cid); + assert_ne!( + provider_cid, raw_cid, + "the provider CID differs from the raw resolver key" + ); + seed_pinned_row( + &pool, + &fx.public_oid, + &provider_cid, + Some("QmPinataProvider"), + "repoPinataRepair", + ) + .await; + + let mut server = mockito::Server::new_async().await; + let m = pinata_upload_mock_never(&mut server).await; + let client = reqwest::Client::new(); + crate::pinata::pin_new_objects( + &client, + &server.url(), + "test-jwt", + &bare, + "git", + std::time::Duration::from_secs(60), + vec![fx.public_oid.clone()], + &state.db, + "repoPinataRepair", + crate::ipfs_pin::PIN_BATCH_BUDGET, + ) + .await; + m.assert_async().await; + + let (stored_cid, stashed) = stored_key_and_stash(&pool, &fx.public_oid).await; + assert_eq!( + stored_cid, raw_cid, + "the pinata skip branch repairs the key to the raw-content CID" + ); + assert_eq!( + stashed.as_deref(), + Some(provider_cid.as_str()), + "the old provider CID is stashed in legacy_provider_cid" + ); + } + + /// U3 scenario 2 (#173, cost gate): a canonical raw-CIDv1 row on the pinata skip + /// branch reads NO object bytes. Candidacy is decided from the stored key's codec + /// alone, so the steady-state skip cost stays DB-only on this lane too. The counter + /// lives inside `repair_legacy_provider_cid`, so it counts for whichever lane calls + /// it; this is the both-ways guard (removing the codec gate reads the raw row). + #[sqlx::test] + async fn pinata_skip_branch_repair_codec_gate_skips_raw_row(pool: PgPool) { + let state = test_state(pool.clone()).await; + let fx = seed_cid_repos("pinatagate", "pg", &["pinsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join("pinatagate") + .join("pinsrc.git"); + + let raw_cid = raw_key_for(&bare, &fx.public_oid); + assert!( + gitlawb_core::cid::is_raw_cidv1(&raw_cid), + "the steady-state key is a CIDv1/raw key" + ); + seed_pinned_row( + &pool, + &fx.public_oid, + &raw_cid, + Some("QmPinataProvider"), + "repoPinataGate", + ) + .await; + + let mut server = mockito::Server::new_async().await; + let m = pinata_upload_mock_never(&mut server).await; + let client = reqwest::Client::new(); + crate::ipfs_pin::reset_legacy_repair_reads(); + crate::pinata::pin_new_objects( + &client, + &server.url(), + "test-jwt", + &bare, + "git", + std::time::Duration::from_secs(60), + vec![fx.public_oid.clone()], + &state.db, + "repoPinataGate", + crate::ipfs_pin::PIN_BATCH_BUDGET, + ) + .await; + m.assert_async().await; + + assert_eq!( + crate::ipfs_pin::legacy_repair_reads(), + 0, + "a CIDv1/raw row triggers no object read on the pinata skip path (cost gate)" + ); + assert_eq!( + state + .db + .cid_for_oid(&fx.public_oid) + .await + .unwrap() + .as_deref(), + Some(raw_cid.as_str()), + "the raw row is left as-is" + ); + } + + /// U3 scenario 3 (#173): a repair that cannot complete is warn-only. It neither + /// aborts the batch nor loses the pin. The first object is a legacy row whose bytes + /// are NOT in the repo, so the read verifies an absence and the row stays withheld + /// rather than being destructively rewritten; the skip branch's own source record + /// still lands for it, and the SECOND object's legacy row is still repaired, which is + /// what proves the batch ran past the failure. + #[sqlx::test] + async fn pinata_skip_branch_repair_failure_is_warn_only(pool: PgPool) { + let state = test_state(pool.clone()).await; + let fx = seed_cid_repos("pinatawarn", "pw", &["pinsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join("pinatawarn") + .join("pinsrc.git"); + + // Object A: bytes absent from this repo, so its repair cannot complete. + let absent_oid = "a".repeat(64); + let absent_provider = legacy_dagpb_cid(&raw_key_for(&bare, &fx.secret_oid)); + seed_pinned_row( + &pool, + &absent_oid, + &absent_provider, + Some("QmPinataProviderA"), + "repoPinataWarn", + ) + .await; + // Object B: a repairable legacy row, queued behind A. + let raw_cid = raw_key_for(&bare, &fx.public_oid); + let provider_cid = legacy_dagpb_cid(&raw_cid); + seed_pinned_row( + &pool, + &fx.public_oid, + &provider_cid, + Some("QmPinataProviderB"), + "repoPinataWarn", + ) + .await; + + let mut server = mockito::Server::new_async().await; + let m = pinata_upload_mock_never(&mut server).await; + let client = reqwest::Client::new(); + crate::pinata::pin_new_objects( + &client, + &server.url(), + "test-jwt", + &bare, + "git", + std::time::Duration::from_secs(60), + vec![absent_oid.clone(), fx.public_oid.clone()], + &state.db, + "repoPinataWarn", + crate::ipfs_pin::PIN_BATCH_BUDGET, + ) + .await; + m.assert_async().await; + + let (a_cid, a_stash) = stored_key_and_stash(&pool, &absent_oid).await; + assert_eq!( + a_cid, absent_provider, + "an unrepairable row is never destructively rewritten" + ); + assert_eq!(a_stash, None, "nothing is stashed for an unrepaired row"); + assert_eq!( + state.db.pin_sources_for_oid(&absent_oid).await.unwrap(), + vec!["repoPinataWarn".to_string()], + "the skip branch's source record still lands: a failed repair loses no pin" + ); + + let (b_cid, b_stash) = stored_key_and_stash(&pool, &fx.public_oid).await; + assert_eq!( + b_cid, raw_cid, + "the batch ran past the failed repair and repaired the later object" + ); + assert_eq!(b_stash.as_deref(), Some(provider_cid.as_str())); + } + + /// U3 scenario 4 (#173, the must-not case): the repair is inside the `has_pinata_cid` + /// skip branch and nowhere else. An object with NO `pinata_cid` takes the upload path, + /// so it must run NO repair read and its (legacy-shaped) key must be left exactly as + /// stored, even though the row would be a repair candidate on the skip branch. Moving + /// the call out of the `Ok(true)` arm reads bytes here and trips the counter. + #[sqlx::test] + async fn pinata_upload_path_never_runs_the_legacy_repair(pool: PgPool) { + let state = test_state(pool.clone()).await; + let fx = seed_cid_repos("pinatanoskip", "pn", &["pinsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join("pinatanoskip") + .join("pinsrc.git"); + + // Legacy-shaped row with NO pinata_cid: `has_pinata_cid` is false, so the skip + // branch is not taken and the object goes to the upload path. + let raw_cid = raw_key_for(&bare, &fx.public_oid); + let provider_cid = legacy_dagpb_cid(&raw_cid); + seed_pinned_row( + &pool, + &fx.public_oid, + &provider_cid, + None, + "repoPinataNoSkip", + ) + .await; + + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Any) + .with_status(200) + .with_body(r#"{"data":{"cid":"QmPinataUploaded"}}"#) + .expect(1) + .create_async() + .await; + let client = reqwest::Client::new(); + crate::ipfs_pin::reset_legacy_repair_reads(); + let pinned = crate::pinata::pin_new_objects( + &client, + &server.url(), + "test-jwt", + &bare, + "git", + std::time::Duration::from_secs(60), + vec![fx.public_oid.clone()], + &state.db, + "repoPinataNoSkip", + crate::ipfs_pin::PIN_BATCH_BUDGET, + ) + .await; + m.assert_async().await; + + assert_eq!( + crate::ipfs_pin::legacy_repair_reads(), + 0, + "the upload path must never run the skip-branch repair" + ); + let (stored_cid, stashed) = stored_key_and_stash(&pool, &fx.public_oid).await; + assert_eq!( + stored_cid, provider_cid, + "an object that never reached the skip branch keeps its stored key untouched" + ); + assert_eq!(stashed, None, "and nothing is stashed for it"); + assert_eq!( + pinned, + vec![(fx.public_oid.clone(), "QmPinataUploaded".to_string())], + "the pinata return still carries the provider CID for the announcement cid_map" + ); + } + + /// U3 scenario 6 (#173, authorization): the marker arms a FALLBACK, never a bypass. + /// With the set marked incomplete and the object living only in a repo the caller + /// may not read, the scan gates every repo through the same per-caller gate, so the + /// caller is still denied and no bytes leak. + #[sqlx::test] + async fn ipfs_cid_marked_incomplete_still_denies_unauthorized_caller(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["u3deny"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3deny.git"); + let mut priv_repo = seed_repo(&owner_did, "u3deny"); + priv_repo.is_public = false; + state + .db + .create_repo(&priv_repo) + .await + .expect("seed private"); + let cid = pin_cid_for_repo(&bare, &fx.secret_oid, &state.db, &priv_repo.id).await; + + // The set is marked incomplete, so the fallback scan definitely runs. + with_pin_sources_broken(&pool, || { + repin_via_skip_branch(&state, &bare, &fx.secret_oid, &priv_repo.id) + }) + .await; + assert!( + state + .db + .pin_sources_incomplete(&fx.secret_oid) + .await + .unwrap(), + "the marker is set, so the scan fallback is armed for this object" + ); + + crate::api::ipfs::reset_preload_queries(); + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + crate::api::ipfs::preload_queries(), + 1, + "the fallback really did run (otherwise the denial below proves nothing)" + ); + assert_eq!( + st, + StatusCode::NOT_FOUND, + "the fallback scan gates every repo, so an unauthorized caller is still denied" + ); + assert!( + !body.contains("TOP SECRET"), + "the denial must not leak the withheld object's bytes" + ); + } + + /// U3 scenario 7 (#173, INV-7 upgrade path): a node already past v21 gets + /// `pinned_cids.pin_sources_incomplete` from the NEW v22 migration, re-running the + /// migrations is idempotent, and a row written before the column existed reads as + /// COMPLETE (so an upgrade cannot arm the O(repos) fallback for every legacy pin). + #[sqlx::test] + async fn pinned_cids_sources_incomplete_upgrade_path(pool: PgPool) { + let state = test_state(pool.clone()).await; + + // Pre-v22 shape: drop the column and forget v22 was applied. + sqlx::query("ALTER TABLE pinned_cids DROP COLUMN IF EXISTS pin_sources_incomplete") + .execute(&pool) + .await + .unwrap(); + sqlx::query("DELETE FROM schema_migrations WHERE version = 22") + .execute(&pool) + .await + .unwrap(); + sqlx::query("INSERT INTO pinned_cids (sha256_hex, cid, pinned_at) VALUES ($1, $2, $3)") + .bind("preu3oid") + .bind("preu3cid") + .bind(chrono::Utc::now().to_rfc3339()) + .execute(&pool) + .await + .unwrap(); + + state.db.run_migrations().await.expect("re-migrate"); + state + .db + .run_migrations() + .await + .expect("migrations are idempotent: a second run succeeds"); + + assert!( + !state.db.pin_sources_incomplete("preu3oid").await.unwrap(), + "a row predating the column reads COMPLETE, so the upgrade arms no fallback" + ); + state + .db + .mark_pin_sources_incomplete("preu3oid", "somerepo") + .await + .expect("mark after upgrade"); + assert!( + state.db.pin_sources_incomplete("preu3oid").await.unwrap(), + "the marker store is present and writable after the upgrade" + ); + } + + /// #173 round 12 (INV-7 upgrade path for v24): a node already carrying v22 markers + /// keeps them across the move to per-`(oid, repo)` state. Which repo failed was never + /// recorded, so a carried marker takes the empty sentinel and no real record clears + /// it, which is strictly safer than the v22 behavior it replaces (there, the next + /// unrelated record cleared it). Also asserts the re-migration is idempotent and that + /// an object with no marker still reads complete. + #[sqlx::test] + async fn pin_source_failures_upgrade_path(pool: PgPool) { + let state = test_state(pool.clone()).await; + + // Pre-v24 shape: drop the new table, forget v24, and leave a v22-style marker. + sqlx::query("DROP TABLE IF EXISTS pin_source_failures") + .execute(&pool) + .await + .unwrap(); + sqlx::query("DELETE FROM schema_migrations WHERE version = 24") + .execute(&pool) + .await + .unwrap(); + for (oid, marked) in [("carriedoid", true), ("cleanoid", false)] { + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, pin_sources_incomplete) \ + VALUES ($1, $2, $3, $4)", + ) + .bind(oid) + .bind(format!("{oid}cid")) + .bind(chrono::Utc::now().to_rfc3339()) + .bind(marked) + .execute(&pool) + .await + .unwrap(); + } + + state.db.run_migrations().await.expect("re-migrate"); + state + .db + .run_migrations() + .await + .expect("migrations are idempotent: a second run succeeds"); + + assert!( + state.db.pin_sources_incomplete("carriedoid").await.unwrap(), + "a v22 marker survives the upgrade instead of being silently dropped" + ); + assert!( + !state.db.pin_sources_incomplete("cleanoid").await.unwrap(), + "an unmarked row stays complete, so the upgrade arms no new fallback" + ); + + // A real record cannot clear a carried marker: the failing repo is unknown, so + // the sentinel it carries matches no repo id. + state + .db + .record_pin_source("carriedoid", "anyrepo") + .await + .expect("record a source"); + assert!( + state.db.pin_sources_incomplete("carriedoid").await.unwrap(), + "a carried marker names no repo, so nothing clears it by accident" + ); + } + + /// #173 (jatmn round 8, F2 — load-bearing): a legacy `pinned_cids` row keyed on a + /// PROVIDER CID (Pinata/Kubo dag-pb — every release before this branch stored the + /// provider CID as the resolver key, not the raw-content CID) must NOT serve raw git + /// bytes that do not hash to the requested CID. `get_by_cid` recomputes the CID over + /// the served bytes and refuses to serve on mismatch. Seeded with a RAW SQL INSERT + /// because the current helpers store the raw CID, so a helper-seeded row is already + /// correct-shape and the RED assertion would be vacuous (INV-21). RED before U2 + /// (serves the git bytes → 200); GREEN after (not served, no bytes egress). + #[sqlx::test] + async fn ipfs_cid_legacy_provider_cid_row_not_served(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["provsrc"]); + let repo = seed_repo(&owner_did, "provsrc"); // public, no rule + state.db.create_repo(&repo).await.expect("seed repo"); + + // A valid sha2-256 CID whose digest is NOT the object's raw-content digest — + // stands in for a Pinata/Kubo dag-pb provider CID (the legacy resolver key). + let provider_cid = gitlawb_core::cid::Cid::from_git_object_bytes( + b"a decoy object whose CID is not the served object's CID", + ) + .to_string(); + + // Legacy-shape row: cid = the PROVIDER CID (raw SQL — the helpers now store the + // raw CID and cannot reproduce this shape). The object itself is public+servable. + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) VALUES ($1, $2, $3, $4)", + ) + .bind(&fx.public_oid) + .bind(&provider_cid) + .bind("2020-01-01T00:00:00Z") + .bind(&repo.id) + .execute(&pool) + .await + .unwrap(); + + // Requesting the provider CID resolves the row and passes the repo gate, but the + // served bytes hash to a DIFFERENT CID, so the integrity check must withhold them. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&provider_cid)) + .await + .unwrap(), + ) + .await; + assert_ne!( + st, + StatusCode::OK, + "a provider-CID legacy row must not serve raw git bytes (F2)" + ); + assert!( + !body.contains("public bytes"), + "the mismatched bytes must not egress" + ); + } + + /// #173 (jatmn round 8, F6 — INV-10 cost guard): the serve path buffers the object via + /// a blocking `cat-file`; an object larger than `ipfs_max_served_object_bytes` must be + /// WITHHELD (rejected by the size precheck, never buffered), with zero body bytes + /// egressed. Under the cap it serves unchanged. The oversize-reject counter guards it + /// both ways: a removed size precheck serves the object and leaves the counter at 0. + #[sqlx::test] + async fn ipfs_cid_f6_oversized_object_withheld(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["big"]); + let bare = std::path::PathBuf::from("/tmp").join(&slug).join("big.git"); + let repo = seed_repo(&owner_did, "big"); // public, no rule + state.db.create_repo(&repo).await.expect("seed repo"); + let cid = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, &repo.id).await; + + // Cap below the object size ("public bytes\n" = 13 bytes) → withheld. + state.ipfs_max_served_object_bytes = 5; + crate::api::ipfs::reset_oversize_rejects(); + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_ne!( + st, + StatusCode::OK, + "an object over the size cap must not serve (F6)" + ); + assert!( + !body.contains("public bytes"), + "no object bytes egress for an over-cap object" + ); + assert_eq!( + crate::api::ipfs::oversize_rejects(), + 1, + "the oversized object was rejected by the size precheck" + ); + + // Control: raise the cap above the object size → serves unchanged. + state.ipfs_max_served_object_bytes = crate::api::ipfs::MAX_SERVED_OBJECT_BYTES; + crate::api::ipfs::reset_oversize_rejects(); + let (st2, body2) = + cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st2, + StatusCode::OK, + "under the cap the object serves normally" + ); + assert!( + body2.contains("public bytes"), + "the served body is the object's bytes" + ); + assert_eq!( + crate::api::ipfs::oversize_rejects(), + 0, + "no oversize reject under the cap" + ); + } + + /// #173 (provenance, INV-11): a quarantined pinning repo must 404 by CID even for + /// its own owner — quarantine hard-drops before the visibility gate on the + /// provenance path too. The owner-signed 404 is the load-bearing negative (a + /// visibility-only gate would Allow the owner). + #[sqlx::test] + async fn ipfs_cid_provenance_quarantined_repo_404_even_owner(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["quarsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("quarsrc.git"); + let repo = seed_repo(&owner_did, "quarsrc"); // public + state.db.create_repo(&repo).await.expect("seed repo"); + let cid = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, &repo.id).await; + + // Baseline: before quarantine the provenanced CID serves (proves the path works). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_signed(&owner, &cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "provenanced CID serves before quarantine" + ); + + state + .db + .set_repo_quarantine(&repo.id, true) + .await + .expect("quarantine"); + + for req in [cid_anon(&cid), cid_signed(&owner, &cid)] { + let (st, body) = cid_parts(cid_router(&state).oneshot(req).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "a quarantined pinning repo must 404 by CID (anon + owner)" + ); + assert!( + !body.contains("public bytes"), + "the 404 body must not leak quarantined content" + ); + } + } + + /// #173 (provenance, bounded — must NOT fall back to the scan): a CID whose + /// provenance points at a repo that no longer exists must 404 rather than scan + /// every repo and serve a byte-identical public copy. Falling back to the scan + /// would reopen the O(repos) anonymous fan-out the provenance rework closes. RED + /// before the rework (the scan serves the public copy → 200); GREEN after. + #[sqlx::test] + async fn ipfs_cid_provenance_missing_repo_404_no_scan_fallback(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["gonesrc", "pubcopy2"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("gonesrc.git"); + + // Pin with provenance = a repo_id that is never created (deleted/absent). + let cid = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, "nonexistent-repo-id").await; + + // A public repo holds the SAME object (the old scan would serve it). + let pub_repo = seed_repo(&owner_did, "pubcopy2"); + state + .db + .create_repo(&pub_repo) + .await + .expect("seed public copy"); + + let (st, _) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "a provenance pointing at a missing repo must 404, not fall back to the scan" + ); + } + + /// #173 (provenance, path-scoped WALK gate): the #135/#173 per-object gates must + /// run on the NEW provenance path, not only the legacy scan. A provenanced pin from + /// a repo under a `/secret/**` rule runs `allowed_blob_set_for_caller` via the shared + /// gate: a withheld secret blob 404s to anon (no byte leak); the allowed reader gets + /// it. Exercises the walk gate on the provenance path in BOTH directions. + #[sqlx::test] + async fn ipfs_cid_provenance_path_scoped_walk_gates_withheld_blob(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let reader = Keypair::generate(); + let reader_did = reader.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["provwalk"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("provwalk.git"); + let repo = seed_repo(&owner_did, "provwalk"); // public at "/" + state.db.create_repo(&repo).await.expect("seed repo"); + // /secret/** Mode B with the reader allowed → the secret blob walk gates by caller. + state + .db + .set_visibility_rule( + &repo.id, + "/secret/**", + VisibilityMode::B, + std::slice::from_ref(&reader_did), + &owner_did, + ) + .await + .expect("path rule"); + let cid = pin_cid_for_repo(&bare, &fx.secret_oid, &state.db, &repo.id).await; + + // Anon: the walk denies the secret blob → 404, no leak. + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "a withheld secret blob 404s to anon on the provenance path (walk gate runs)" + ); + assert!( + !body.contains("TOP SECRET"), + "the 404 body must not leak the withheld blob" + ); + + // Allowed reader: the walk includes the secret blob → 200 with content. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_signed(&reader, &cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "an allowed reader gets the secret blob via the provenance walk gate" + ); + assert!( + body.contains("TOP SECRET"), + "the allowed reader receives the content" + ); + } + + /// #173: the pinata pin path stores the locally-computed raw CID in the + /// resolver-key `cid` column and the provider CID in `pinata_cid`, and its ON + /// CONFLICT COALESCE fills a NULL provenance without overwriting an existing one + /// (first-pinner-owns). On conflict `cid` is left untouched so a prior local pin's + /// raw CID is never clobbered by a provider CID. + #[sqlx::test] + async fn record_pinata_cid_stores_and_coalesces_provenance(pool: PgPool) { + let state = test_state(pool).await; + + // Real raw-CIDv1 resolver keys, as the pin paths write them: `list_pinned_cids` + // withholds any row keyed on a non-raw (legacy provider) value (U4, #173), so a + // placeholder string here would be filtered out and make the assertions vacuous. + let raw1 = gitlawb_core::cid::Cid::from_git_object_bytes(b"pinata raw 1").to_string(); + let local2 = gitlawb_core::cid::Cid::from_git_object_bytes(b"local raw 2").to_string(); + + // A new row created via the pinata path carries provenance, and stores the + // raw CID in `cid` with the provider CID in `pinata_cid`. + state + .db + .record_pinata_cid("po1", &raw1, "pcid1", Some("repoA")) + .await + .unwrap(); + assert_eq!( + state.db.provenance_for_oid("po1").await.unwrap().as_deref(), + Some("repoA") + ); + let po1 = state + .db + .list_pinned_cids() + .await + .unwrap() + .into_iter() + .find(|r| r.sha256_hex == "po1") + .expect("po1 row exists"); + assert_eq!(po1.cid, raw1, "resolver-key cid is the raw CID"); + assert_eq!( + po1.pinata_cid.as_deref(), + Some("pcid1"), + "the provider CID is kept in pinata_cid" + ); + + // An existing NULL-provenance row: the pinata COALESCE fills it, and the + // prior local pin's `cid` is left untouched (not overwritten by the raw arg). + state + .db + .record_pinned_cid("po2", &local2, None) + .await + .unwrap(); + state + .db + .record_pinata_cid("po2", "rawcid2", "pcid2", Some("repoB")) + .await + .unwrap(); + assert_eq!( + state.db.provenance_for_oid("po2").await.unwrap().as_deref(), + Some("repoB"), + "pinata fills a NULL provenance" + ); + let po2 = state + .db + .list_pinned_cids() + .await + .unwrap() + .into_iter() + .find(|r| r.sha256_hex == "po2") + .expect("po2 row exists"); + assert_eq!( + po2.cid, local2, + "on conflict the prior local pin's cid is left untouched" + ); + + // An existing provenance: the pinata COALESCE must NOT overwrite it. + state + .db + .record_pinned_cid("po3", "cid3", Some("repoX")) + .await + .unwrap(); + state + .db + .record_pinata_cid("po3", "rawcid3", "pcid3", Some("repoY")) + .await + .unwrap(); + assert_eq!( + state.db.provenance_for_oid("po3").await.unwrap().as_deref(), + Some("repoX"), + "pinata COALESCE keeps the first-pinner's provenance" + ); + } + + /// #173 (jatmn, F4, load-bearing security): a Pinata-first pin (no prior local pin) + /// must make the resolver key (`pinned_cids.cid`) the locally-computed raw CID, NOT + /// the provider CID. Pinata wraps the bytes in dag-pb/UnixFS, so its returned CID + /// does not hash the raw content; if it became the resolver key, `/ipfs/{provider_cid}` + /// would serve raw git bytes that do not hash to it, breaking raw content-addressing. + /// Assert `oids_for_cid(raw_cid)` finds the sha AND `oids_for_cid(provider_cid)` does NOT. + #[sqlx::test] + async fn record_pinata_cid_resolver_key_is_raw_not_provider(pool: PgPool) { + let state = test_state(pool).await; + + let bytes = b"raw git object content for pinata-first pin"; + let raw_cid = gitlawb_core::cid::Cid::from_git_object_bytes(bytes).to_string(); + // A distinct provider CID (a dag-pb wrapper CID Pinata would return). + let provider_cid = "QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG"; + assert_ne!( + raw_cid, provider_cid, + "the provider CID must differ from the raw CID for this test to be meaningful" + ); + + // Pinata-first: no prior local pin, so this INSERT creates the row. + state + .db + .record_pinata_cid("pfsha", &raw_cid, provider_cid, Some("repoP")) + .await + .unwrap(); + + // The raw CID resolves to the sha. + assert_eq!( + state.db.oids_for_cid(&raw_cid).await.unwrap(), + vec!["pfsha".to_string()], + "the locally-computed raw CID is the resolver key" + ); + // The provider (dag-pb) CID must NOT resolve raw bytes. + assert!( + state + .db + .oids_for_cid(provider_cid) + .await + .unwrap() + .is_empty(), + "the provider dag-pb CID must never resolve raw git bytes" + ); + } + + /// #173 (end-to-end pin wiring): `pin_new_objects` records the repo_id it is given + /// as the pin's provenance. Drives the real pin path against a mocked IPFS `/add` + /// endpoint (so `pin_git_object` succeeds) and asserts `provenance_for_oid` returns + /// the repo — closing the gap between the push handler's threading and the DB write. + #[sqlx::test] + async fn pin_new_objects_records_provenance(pool: PgPool) { + let state = test_state(pool).await; + + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovtest"}"#) + .expect_at_least(1) + .create_async() + .await; + + let fx = seed_cid_repos("provpin_e2e", "ppe2e", &["pinsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join("provpin_e2e") + .join("pinsrc.git"); + + let pinned = crate::ipfs_pin::pin_new_objects( + &server.url(), + &bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + vec![fx.public_oid.clone()], + &state.db, + "repoZ", + crate::ipfs_pin::PIN_BATCH_BUDGET, + ) + .await; + assert!( + !pinned.is_empty(), + "the object was pinned via the real pin path" + ); + m.assert_async().await; + assert_eq!( + state + .db + .provenance_for_oid(&fx.public_oid) + .await + .unwrap() + .as_deref(), + Some("repoZ"), + "pin_new_objects records the repo_id it was given as the pin's provenance" + ); + } + + /// U4 (#173, finding 5): a pin whose DB record exhausts its retries must NOT appear + /// in the returned vector. Kubo really is holding the bytes (the `/add` mock is hit), + /// but with no `pinned_cids` row the resolver cannot serve that CID, so reporting it + /// as pinned overclaims. The Kubo return is log-only (`api/repos.rs` counts the pairs + /// and logs each one), which is what makes omitting the row safe here; the pinata + /// twin's return feeds the announcement `cid_map` and keeps its own contract. + /// + /// Two objects, because `with_pin_sources_broken` hides the table process-wide and + /// the harness cannot express per-object DB breakage. Both records fail, and the + /// `/add` mock being hit exactly twice is the batch-survival proof: the first + /// failure warns and continues instead of breaking out of the loop. The healthy + /// direction (a successful record IS returned) is already covered by + /// `pin_new_objects_records_provenance` directly above, so the two together cover + /// both sides without new harness machinery. + #[sqlx::test] + async fn pin_new_objects_omits_objects_whose_db_record_failed(pool: PgPool) { + let state = test_state(pool.clone()).await; + + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyproviderhash"}"#) + .expect(2) + .create_async() + .await; + + let fx = seed_cid_repos("provpin_u4", "ppu4", &["pinsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join("provpin_u4") + .join("pinsrc.git"); + + let pinned = with_pin_sources_broken(&pool, || async { + crate::ipfs_pin::pin_new_objects( + &server.url(), + &bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + vec![fx.public_oid.clone(), fx.secret_oid.clone()], + &state.db, + "repoU4", + // Far above the ~150ms per object the retry ladder spends + // (PIN_RECORD_ATTEMPTS x PIN_RECORD_BACKOFF), so the batch budget gate + // is never what truncates this run. + std::time::Duration::from_secs(60), + ) + .await + }) + .await; + + assert!( + pinned.is_empty(), + "a pin with no durable index row must not be reported as pinned, got {pinned:?}" + ); + // Exactly two adds: the first record failure did not break the batch. + m.assert_async().await; + for oid in [&fx.public_oid, &fx.secret_oid] { + assert_eq!( + state.db.provenance_for_oid(oid).await.unwrap(), + None, + "the record really did fail, so there is no row to report" + ); + } + } + + /// #173 (grok F2): the post-push pin read is BOUNDED, so a wedged/D-state + /// `git cat-file` (stuck NFS/Tigris backend) is reaped at `git_timeout` and + /// `pin_new_objects` RETURNS — reaching `requeue_or_release` in production — + /// instead of hanging forever and pinning the per-repo coalescing key until + /// process death. A fake `git` whose `cat-file` records its pid then sleeps far + /// past a SHORT 1s timeout stands in for the wedged backend; the `run_bounded_git` + /// watchdog (SIGTERM -> grace -> SIGKILL of the process group) must reap it well + /// before its 8s natural exit, and the call must return with nothing pinned. + /// + /// REVERT PROOF (RED): swap `read_object_bounded` back to the bare + /// `store::read_object` at the pin read and the wedged child is STILL RUNNING at + /// the mid-flight liveness poll below (unbounded `Command::output` cannot be + /// reaped at the deadline) — the reap assertion fails. + #[cfg(unix)] + #[sqlx::test] + async fn pin_new_objects_reaps_wedged_read_at_deadline(pool: PgPool) { + use std::time::Duration; + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.unwrap(); + + // Fake `git`: `cat-file` records its own pid then sleeps 8s (>> the 1s + // deadline) so the read is genuinely wedged; the watchdog is what must end it. + let tmp = tempfile::TempDir::new().unwrap(); + let pidfile = tmp.path().join("catfile.pid"); + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + cat-file) echo $$ > \"{}\"; sleep 8 ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + pidfile.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + let repo = tmp.path().to_path_buf(); + let git = git_path.to_str().unwrap().to_string(); + // A never-pinned OID so the call reaches the object-read stage (not the + // already-pinned skip path). + let oid = "f".repeat(64); + // Non-empty ipfs_api so `pin_new_objects` does not early-return; the wedged + // read is reaped and the OID skipped before any `/add`, so this URL is unused. + let ipfs_api = "http://127.0.0.1:1".to_string(); + + // `pin_new_objects` must run on THIS runtime so its `is_pinned` DB call keeps + // the sqlx pool on its home runtime. The bounded read is a synchronous blocking + // call, so the reap poll runs on a separate OS thread (independent of tokio): it + // captures the wedged child's pid, waits past the deadline, records whether it + // was reaped, then SIGKILLs defensively so even a true infinite hang cannot leak + // an orphan or stall the awaited call. + let pidfile_poll = pidfile.clone(); + let poll = std::thread::spawn(move || -> (Option, bool) { + let alive = |pid: i32| unsafe { libc::kill(pid, 0) == 0 }; + let mut pid = None; + for _ in 0..500 { + if let Some(p) = std::fs::read_to_string(&pidfile_poll) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + pid = Some(p); + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + let pid = match pid { + Some(p) => p, + None => return (None, false), + }; + // Past the 1s deadline + SIGTERM grace but well before the 8s natural exit: + // the bounded read must already have reaped the wedged group. The unbounded + // `store::read_object` leaves it running here — the load-bearing RED. + std::thread::sleep(Duration::from_secs(3)); + let reaped = !alive(pid); + unsafe { + libc::kill(pid, libc::SIGKILL); + } + (Some(pid), reaped) + }); + + // The call must RETURN — reaching `requeue_or_release` in production — rather + // than hang on the 8s sleep. The poll thread's defensive SIGKILL guarantees the + // read completes even in the unbounded RED case, so this observes a bounded + // return either way; the reap assertion below is what separates RED from GREEN. + let pinned = tokio::time::timeout( + Duration::from_secs(6), + crate::ipfs_pin::pin_new_objects( + &ipfs_api, + &repo, + &git, + Duration::from_secs(1), + vec![oid], + &db, + "repoWedge", + crate::ipfs_pin::PIN_BATCH_BUDGET, + ), + ) + .await + .expect("pin_new_objects must return within the bound, not hang on the wedged read"); + + let (pid, reaped) = poll.join().expect("poll thread joins"); + pid.expect("the fake cat-file must have spawned and recorded its pid"); + assert!( + reaped, + "the post-push pin read must reap the wedged cat-file child at the deadline, \ + not leave it running (which would pin the coalescing key until process death)" + ); + assert!( + pinned.is_empty(), + "a wedged read pins nothing this pass; a later pass/push retries" + ); + } + + /// #173 (jatmn, F2): a legacy pin with NULL provenance backfills its source + /// via `backfill_pin_provenance`, and the `AND repo_id IS NULL` guard preserves + /// first-pinner-owns (a non-NULL provenance is left untouched). + #[sqlx::test] + async fn backfill_pin_provenance_fills_null_keeps_existing(pool: PgPool) { + let state = test_state(pool).await; + + // A legacy pin: no provenance recorded. + state + .db + .record_pinned_cid("legacy_oid", "legacy_cid", None) + .await + .unwrap(); + assert_eq!( + state.db.provenance_for_oid("legacy_oid").await.unwrap(), + None, + "a legacy pin starts with NULL provenance" + ); + + // Backfill sets the NULL provenance. + state + .db + .backfill_pin_provenance("legacy_oid", "repo-src") + .await + .unwrap(); + assert_eq!( + state + .db + .provenance_for_oid("legacy_oid") + .await + .unwrap() + .as_deref(), + Some("repo-src"), + "backfill fills a NULL provenance from the known source" + ); + + // A pin that already has provenance: backfill must NOT overwrite it. + state + .db + .record_pinned_cid("owned_oid", "owned_cid", Some("repo-first")) + .await + .unwrap(); + state + .db + .backfill_pin_provenance("owned_oid", "repo-second") + .await + .unwrap(); + assert_eq!( + state + .db + .provenance_for_oid("owned_oid") + .await + .unwrap() + .as_deref(), + Some("repo-first"), + "the AND repo_id IS NULL guard keeps the first-pinner's provenance" + ); + } + + /// #173 (jatmn, F2, load-bearing): an object already pinned with NULL provenance + /// (a pre-provenance legacy pin) acquires its source when `pin_new_objects` sees + /// it again. The already-pinned skip path must backfill rather than leave the + /// object stuck on the O(repos) scan fallback — and it must NOT re-pin the bytes + /// (no IPFS `/add` call, the object is already on IPFS). + #[sqlx::test] + async fn pin_new_objects_backfills_legacy_null_provenance(pool: PgPool) { + let state = test_state(pool).await; + + let fx = seed_cid_repos("provpin_backfill", "ppbf", &["pinsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join("provpin_backfill") + .join("pinsrc.git"); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes( + &crate::git::store::read_object(&bare, &fx.public_oid) + .expect("read object bytes") + .expect("object exists") + .1, + ) + .to_string(); + + // Legacy pin: the object is already recorded with NULL provenance. + state + .db + .record_pinned_cid(&fx.public_oid, &cid, None) + .await + .unwrap(); + assert_eq!( + state.db.provenance_for_oid(&fx.public_oid).await.unwrap(), + None, + "the object starts as a legacy pin with NULL provenance" + ); + + // Mock IPFS `/add` and require it is NOT called: the already-pinned object + // must be backfilled, never re-pinned. + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyshouldnothappen"}"#) + .expect(0) + .create_async() + .await; + + let pinned = crate::ipfs_pin::pin_new_objects( + &server.url(), + &bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + vec![fx.public_oid.clone()], + &state.db, + "repoBF", + crate::ipfs_pin::PIN_BATCH_BUDGET, + ) + .await; + + assert!( + pinned.is_empty(), + "an already-pinned object is not re-pinned (no bytes returned)" + ); + m.assert_async().await; // asserts /add was called 0 times + assert_eq!( + state + .db + .provenance_for_oid(&fx.public_oid) + .await + .unwrap() + .as_deref(), + Some("repoBF"), + "pin_new_objects backfills the legacy pin's NULL provenance" + ); + } + + /// Build a legacy provider CID (CIDv1 dag-pb — the Kubo above-block-size root + /// shape, and codec-equivalent to the Pinata CIDv0 legacy key for the cost + /// gate) over the object's own multihash. Non-raw codec, so `is_raw_cidv1` + /// flags it a repair candidate, and a different string from the raw key, so a + /// repair rewrites it. The existing `ipfs_cid_legacy_provider_cid_row_not_served` + /// fixture seeds a raw-codec decoy (an integrity negative the cost gate treats + /// as non-legacy on purpose); this produces the genuine dag-pb legacy shape the + /// repair path targets. Uses only the `cid` crate (already a node dep). + fn legacy_dagpb_cid(raw_cid: &str) -> String { + const DAG_PB: u64 = 0x70; + let parsed = raw_cid + .parse::>() + .expect("the raw CID parses"); + cid::CidGeneric::<64>::new_v1(DAG_PB, *parsed.hash()).to_string() + } + + /// #173 R8 (jatmn round 10, U7 — load-bearing): a legacy row keyed on a PROVIDER + /// CID (Kubo dag-pb / Pinata) is opportunistically rewritten to the raw-content + /// key on a re-push whose pack carries the object, stashing the old value in + /// `legacy_provider_cid`. The advertised key 404s while the row is legacy (the + /// resolver recomputes the raw CID and the stored key does not match) and serves + /// after repair. RED before the skip-branch repair lands (the raw key 404s post + /// pin). Also asserts the repair leaves `pinata_cid` NULL (scenario 3) and that + /// the retired provider CID still refuses to serve (scenario 6, integrity). + #[sqlx::test] + async fn ipfs_cid_legacy_provider_cid_repaired_on_repush(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["provsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("provsrc.git"); + let repo = seed_repo(&owner_did, "provsrc"); // public, no rule + state.db.create_repo(&repo).await.expect("seed repo"); + + // The canonical raw key the resolver accepts once the row is repaired. + let raw_cid = gitlawb_core::cid::Cid::from_git_object_bytes( + &crate::git::store::read_object(&bare, &fx.public_oid) + .unwrap() + .unwrap() + .1, + ) + .to_string(); + // The key stored today: a genuine legacy dag-pb provider CID. + let provider_cid = legacy_dagpb_cid(&raw_cid); + assert_ne!( + provider_cid, raw_cid, + "the provider CID differs from the raw resolver key" + ); + + // Legacy-shape row: cid = the PROVIDER CID (raw SQL — the helpers store the + // raw CID). The object itself is public and servable. + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) VALUES ($1, $2, $3, $4)", + ) + .bind(&fx.public_oid) + .bind(&provider_cid) + .bind("2020-01-01T00:00:00Z") + .bind(&repo.id) + .execute(&pool) + .await + .unwrap(); + + // RED baseline: the raw key a correct client sends 404s while the row is legacy. + let (st_before, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&raw_cid)) + .await + .unwrap(), + ) + .await; + assert_ne!( + st_before, + StatusCode::OK, + "the raw key 404s while the row is keyed on the provider CID" + ); + + // Re-push carries the object again: `pin_new_objects` hits the already-pinned + // skip branch and repairs the row. The `/add` mock must NOT fire — the object + // is already on IPFS, never re-pinned. + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyshouldnothappen"}"#) + .expect(0) + .create_async() + .await; + crate::ipfs_pin::pin_new_objects( + &server.url(), + &bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + vec![fx.public_oid.clone()], + &state.db, + &repo.id, + crate::ipfs_pin::PIN_BATCH_BUDGET, + ) + .await; + m.assert_async().await; + + // GREEN: the key is repaired to the raw CID and the old value is stashed. + let (stored_cid, stashed): (String, Option) = sqlx::query_as( + "SELECT cid, legacy_provider_cid FROM pinned_cids WHERE sha256_hex = $1", + ) + .bind(&fx.public_oid) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!( + stored_cid, raw_cid, + "the key is repaired to the raw-content CID" + ); + assert_eq!( + stashed.as_deref(), + Some(provider_cid.as_str()), + "the old provider CID is stashed in legacy_provider_cid" + ); + + // The advertised (raw) key now serves 200. + let (st_after, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&raw_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st_after, + StatusCode::OK, + "the repaired raw key serves after the re-push" + ); + assert!(body.contains("public bytes"), "the object's bytes serve"); + + // Scenario 3: repair never wrote `pinata_cid`, so the Pinata pin-skip gate + // (`has_pinata_cid`) is untouched and Pinata still pins the object. + assert!( + !state.db.has_pinata_cid(&fx.public_oid).await.unwrap(), + "repair leaves pinata_cid NULL" + ); + + // Scenario 6 (integrity negative): the retired provider CID still 404s — no + // serve-path alias for a CID the bytes do not hash to. + let (st_old, body_old) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&provider_cid)) + .await + .unwrap(), + ) + .await; + assert_ne!( + st_old, + StatusCode::OK, + "the retired provider CID must not serve after repair" + ); + assert!( + !body_old.contains("public bytes"), + "no bytes egress under the retired provider CID" + ); + } + + /// #173 R8 (U7 cost gate): a well-formed CIDv1/raw already-pinned row triggers NO + /// object read on the skip path — the codec check decides candidacy from the + /// stored string alone, so a non-legacy row keeps the DB-only skip cost. Also + /// covers the small-object equivalence: a small legacy object Kubo pins under the + /// raw key (raw-leaves) is already CIDv1/raw and needs no repair. The read counter + /// is the both-ways guard: removing the codec gate reads the raw row and trips it. + #[sqlx::test] + async fn ipfs_cid_repair_codec_gate_skips_raw_row(pool: PgPool) { + let state = test_state(pool).await; + let fx = seed_cid_repos("codecgate", "cg", &["pinsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join("codecgate") + .join("pinsrc.git"); + + // A correct raw-CID row (steady state), recorded via the production helper. + let raw_cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + assert!( + gitlawb_core::cid::is_raw_cidv1(&raw_cid), + "the helper records a CIDv1/raw key" + ); + + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"x"}"#) + .expect(0) + .create_async() + .await; + + crate::ipfs_pin::reset_legacy_repair_reads(); + crate::ipfs_pin::pin_new_objects( + &server.url(), + &bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + vec![fx.public_oid.clone()], + &state.db, + "repoCG", + crate::ipfs_pin::PIN_BATCH_BUDGET, + ) + .await; + m.assert_async().await; + + assert_eq!( + crate::ipfs_pin::legacy_repair_reads(), + 0, + "a CIDv1/raw row triggers no object read on the skip path (cost gate)" + ); + assert_eq!( + state + .db + .cid_for_oid(&fx.public_oid) + .await + .unwrap() + .as_deref(), + Some(raw_cid.as_str()), + "the raw row is left as-is" + ); + } + + /// #173 R8 (U7): a legacy row whose object bytes are gone stays withheld — the + /// repair never destructively rewrites it, so the row is preserved for a future + /// re-push or the deferred one-shot sweep. + #[sqlx::test] + async fn ipfs_cid_repair_unrepairable_row_stays_withheld(pool: PgPool) { + let state = test_state(pool.clone()).await; + let _fx = seed_cid_repos("unrep", "ur", &["pinsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join("unrep") + .join("pinsrc.git"); + + // A legacy dag-pb row for an oid whose bytes are NOT in this bare repo. + let phantom_oid = "b".repeat(64); + let raw_cid = + gitlawb_core::cid::Cid::from_git_object_bytes(b"bytes that live nowhere").to_string(); + let provider_cid = legacy_dagpb_cid(&raw_cid); + sqlx::query("INSERT INTO pinned_cids (sha256_hex, cid, pinned_at) VALUES ($1, $2, $3)") + .bind(&phantom_oid) + .bind(&provider_cid) + .bind("2020-01-01T00:00:00Z") + .execute(&pool) + .await + .unwrap(); + + let mut server = mockito::Server::new_async().await; + server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"x"}"#) + .expect(0) + .create_async() + .await; + + // Skip-branch runs (is_pinned true) but read_object returns None (bytes gone), + // so the repair returns without touching the row. + crate::ipfs_pin::pin_new_objects( + &server.url(), + &bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + vec![phantom_oid.clone()], + &state.db, + "repoUR", + crate::ipfs_pin::PIN_BATCH_BUDGET, + ) + .await; + + let (stored, stashed): (String, Option) = sqlx::query_as( + "SELECT cid, legacy_provider_cid FROM pinned_cids WHERE sha256_hex = $1", + ) + .bind(&phantom_oid) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!( + stored, provider_cid, + "an unrepairable row keeps its provider CID (no destructive rewrite)" + ); + assert_eq!( + stashed, None, + "no legacy_provider_cid is stashed when the bytes are gone" + ); + } + + /// #173 R8 (U7, INV-7 upgrade path): a node already at the prior-max schema (v13) + /// gets `pinned_cids.legacy_provider_cid` from the NEW v21 migration. Simulate the + /// pre-v21 node by dropping the column and un-applying v14, then re-migrate and + /// assert a repair round-trips through the column. RED before the v21 migration + /// exists (the column is never re-added → the repair UPDATE errors). + #[sqlx::test] + async fn pinned_cids_legacy_provider_cid_upgrade_path(pool: PgPool) { + let state = test_state(pool.clone()).await; + + // Pre-v21 shape: drop the column and forget v21 was applied. + sqlx::query("ALTER TABLE pinned_cids DROP COLUMN IF EXISTS legacy_provider_cid") + .execute(&pool) + .await + .unwrap(); + sqlx::query("DELETE FROM schema_migrations WHERE version = 21") + .execute(&pool) + .await + .unwrap(); + + // Upgrade: re-run migrations → v21 re-adds the column. + state.db.run_migrations().await.expect("migrate to v14"); + + // A repair round-trips through the v21 column. + state + .db + .record_pinned_cid("upg_oid", "QmProviderLegacy", None) + .await + .unwrap(); + state + .db + .repair_legacy_provider_cid("upg_oid", "bRawContentKey", "QmProviderLegacy") + .await + .unwrap(); + let (cid, stashed): (String, Option) = sqlx::query_as( + "SELECT cid, legacy_provider_cid FROM pinned_cids WHERE sha256_hex = 'upg_oid'", + ) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(cid, "bRawContentKey", "v21 lets the repair rewrite the key"); + assert_eq!( + stashed.as_deref(), + Some("QmProviderLegacy"), + "the v21 legacy_provider_cid column is present after upgrade" + ); + } + + // ---- #173 U4: legacy provider-CID migration sweep ---- + + /// Seed a legacy PROVIDER-CID `pinned_cids` row for `oid` (the pre-branch shape: + /// `cid` holds the Kubo dag-pb / Pinata key, not the raw-content resolver key). + /// Returns `(raw_cid, provider_cid)`. Raw SQL because every production helper + /// stores the already-correct raw key. + async fn seed_legacy_pin( + pool: &PgPool, + bare: &std::path::Path, + oid: &str, + repo_id: Option<&str>, + ) -> (String, String) { + let (_ty, bytes) = crate::git::store::read_object(bare, oid) + .expect("read object bytes") + .expect("object exists in the bare repo"); + let raw = gitlawb_core::cid::Cid::from_git_object_bytes(&bytes).to_string(); + let provider = legacy_dagpb_cid(&raw); + assert_ne!(provider, raw, "the legacy key differs from the raw key"); + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) VALUES ($1, $2, $3, $4)", + ) + .bind(oid) + .bind(&provider) + .bind("2020-01-01T00:00:00Z") + .bind(repo_id) + .execute(pool) + .await + .unwrap(); + (raw, provider) + } + + /// The `pinned_cids.cid` currently stored for an oid, unfiltered (unlike + /// `list_pinned_cids`, which withholds unrepaired legacy rows). + async fn stored_pin(pool: &PgPool, oid: &str) -> (String, Option) { + sqlx::query_as("SELECT cid, legacy_provider_cid FROM pinned_cids WHERE sha256_hex = $1") + .bind(oid) + .fetch_one(pool) + .await + .unwrap() + } + + /// U4 (#173, INV-7 upgrade path): a node already at the prior-max schema (v15) gets + /// the `pin_repair_sweep` cursor table from the NEW v23 migration. Simulate the + /// pre-v23 node by dropping the table and un-applying v16, then re-migrate and + /// assert the cursor round-trips. RED before the v23 migration exists (the table is + /// never recreated, so the cursor read errors). + #[sqlx::test] + async fn pin_repair_sweep_cursor_upgrade_path(pool: PgPool) { + let state = test_state(pool.clone()).await; + + // Pre-v23 shape: drop the table and forget v23 was applied. + sqlx::query("DROP TABLE IF EXISTS pin_repair_sweep") + .execute(&pool) + .await + .unwrap(); + sqlx::query("DELETE FROM schema_migrations WHERE version = 23") + .execute(&pool) + .await + .unwrap(); + + state.db.run_migrations().await.expect("migrate to v16"); + + // Absent row reads as the "never swept" start, and a write round-trips. + assert_eq!( + state.db.pin_repair_cursor().await.unwrap(), + "", + "a node that has never swept starts at the beginning of the table" + ); + state.db.set_pin_repair_cursor("abc").await.unwrap(); + state.db.set_pin_repair_cursor("def").await.unwrap(); + assert_eq!( + state.db.pin_repair_cursor().await.unwrap(), + "def", + "the v23 cursor table persists the walk position across writes" + ); + let rows: i64 = sqlx::query_scalar("SELECT count(*) FROM pin_repair_sweep") + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(rows, 1, "the cursor is a single row, not an append log"); + } + + /// U4 scenario 1 (#173): a legacy provider-CID row with intact object bytes is + /// repaired to the raw-content resolver key by the SWEEP alone, with the old value + /// stashed in `legacy_provider_cid`. No push, no re-pin: this is the whole point of + /// U4, because normal git negotiation omits objects the node already has, so the + /// skip-branch repair's re-push trigger generally never fires on an upgraded node. + /// RED before the sweep is implemented (the row keeps its provider key). + #[sqlx::test] + async fn sweep_repairs_legacy_row_without_a_push(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["swsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("swsrc.git"); + let repo = seed_repo(&owner_did, "swsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + + let (raw_cid, provider_cid) = + seed_legacy_pin(&pool, &bare, &fx.public_oid, Some(&repo.id)).await; + + let stats = crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ) + .await; + assert_eq!(stats.repaired, 1, "the sweep repairs the one legacy row"); + + let (stored, stashed) = stored_pin(&pool, &fx.public_oid).await; + assert_eq!( + stored, raw_cid, + "the key is rewritten to the raw-content CID" + ); + assert_eq!( + stashed.as_deref(), + Some(provider_cid.as_str()), + "the old provider CID is stashed in legacy_provider_cid" + ); + + // End to end: the repaired key is now advertised AND serves. + assert!( + state + .db + .list_pinned_cids() + .await + .unwrap() + .iter() + .any(|r| r.cid == raw_cid), + "the repaired row is advertised" + ); + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&raw_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "the repaired raw key serves"); + assert!(body.contains("public bytes"), "the object's bytes serve"); + } + + /// One transient database fault must not permanently disable the sweep. + /// + /// The wrapper was made periodic so coverage is wall-clock rather than a reboot + /// count. Returning for good on the first failed pass query undoes exactly that: a + /// single deadlock or connection reset disables legacy-CID repair for the whole + /// process lifetime, `main` never joins the handle, so nothing observes it past one + /// warn, and the node keeps withholding every unrepaired row until someone reboots + /// it. + /// + /// The fixture renames `pinned_cids` out of the way so every pass query fails, waits + /// for the loop to have gone round more than once (which a terminal return cannot + /// do), then renames the table back and asserts the still-running loop picks the + /// repair up. + /// + /// MUTATION (RED): restore the terminal `return` on `PassFailed` and the loop exits + /// on the first failure, so the row is never repaired. + #[sqlx::test] + async fn sweep_rearms_after_a_failed_pass(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let _serialized = crate::ipfs_pin::sweep_run_lock().lock().await; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + + let fx = seed_cid_repos(&slug, &short, &["rearmsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("rearmsrc.git"); + let repo = seed_repo(&owner_did, "rearmsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + let (raw_cid, _provider) = + seed_legacy_pin(&pool, &bare, &fx.public_oid, Some(&repo.id)).await; + + // Every pass query now fails, exactly as a broken database makes them fail. + sqlx::query("ALTER TABLE pinned_cids RENAME TO pinned_cids_hidden") + .execute(&pool) + .await + .unwrap(); + + crate::ipfs_pin::reset_sweep_runs(); + let db = state.db.clone(); + let git_bin = state.git_bin.clone(); + // Short rather than literally zero: the loop is spinning against a real + // Postgres, and the property under test is that it goes round again at all. The + // failure and idle intervals are multiples of this base, so they shrink with it. + let handle = tokio::spawn(async move { + crate::ipfs_pin::run_sweep_rearmed( + std::path::Path::new("/tmp"), + &git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + std::time::Duration::from_millis(10), + &db, + ) + .await + }); + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while crate::ipfs_pin::sweep_runs() < 2 && std::time::Instant::now() < deadline { + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + } + assert!( + crate::ipfs_pin::sweep_runs() >= 2, + "a failed pass must re-arm: the sweep completed {} run(s) and stopped, which \ + is one transient database fault disabling legacy-CID repair for the life of \ + the process", + crate::ipfs_pin::sweep_runs() + ); + assert!( + !handle.is_finished(), + "the re-arm loop must never return; shutdown preempts it from the outside" + ); + + // The database comes back. The loop is still there to notice. + sqlx::query("ALTER TABLE pinned_cids_hidden RENAME TO pinned_cids") + .execute(&pool) + .await + .unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + let mut repaired = false; + while std::time::Instant::now() < deadline { + if stored_pin(&pool, &fx.public_oid).await.0 == raw_cid { + repaired = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + } + handle.abort(); + assert!( + repaired, + "once the database recovers the still-running sweep must repair the row; a \ + wrapper that returned on the first failure never gets here" + ); + } + + /// A run that repairs nothing backs off; a run that repairs keeps the base interval. + /// + /// The base interval is priced against a settled table. It is not priced against the + /// table that never settles: source-less rows whose bytes are permanently gone cost + /// up to `MAX_DEAD_ROW_READS_PER_RUN` object reads per run and repair nothing, every + /// base interval, forever. Backing off on a fruitless run is what stops paying that; + /// resetting on a productive one is what keeps a table that is still yielding + /// repairs being walked often. + /// + /// Both directions, on the wall clock, off the run counter the wrapper exposes: + /// leg 1 is an empty table, where a run repairs nothing and the next run must NOT + /// arrive within a window several base intervals wide; leg 2 seeds a repairable row, + /// so the first run repairs and the second must arrive one BASE interval later. + /// + /// MUTATION (RED): drop the idle branch and leg 1 completes many runs in its window. + #[sqlx::test] + async fn sweep_backs_off_after_a_run_that_repairs_nothing(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let _serialized = crate::ipfs_pin::sweep_run_lock().lock().await; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + // Scaled down from production by a constant factor: the idle interval is a + // multiple of this base, so the ratio under test is the production ratio. + let base = std::time::Duration::from_millis(200); + let window = std::time::Duration::from_millis(800); + + let spawn_loop = |db: std::sync::Arc, git_bin: String| { + tokio::spawn(async move { + crate::ipfs_pin::run_sweep_rearmed( + std::path::Path::new("/tmp"), + &git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + base, + &db, + ) + .await + }) + }; + let await_first_run = || async { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while crate::ipfs_pin::sweep_runs() < 1 && std::time::Instant::now() < deadline { + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + assert!( + crate::ipfs_pin::sweep_runs() >= 1, + "fixture precondition: the sweep completes a first run" + ); + }; + + // Leg 1: nothing to repair. The next run must not arrive inside a window four + // base intervals wide. + crate::ipfs_pin::reset_sweep_runs(); + let idle_loop = spawn_loop(state.db.clone(), state.git_bin.clone()); + await_first_run().await; + tokio::time::sleep(window).await; + let idle_runs = crate::ipfs_pin::sweep_runs(); + idle_loop.abort(); + assert_eq!( + idle_runs, + 1, + "a run that repaired nothing must back off to the longer idle interval; at \ + the base interval this window fits about {} runs, each of which pays up to \ + MAX_DEAD_ROW_READS_PER_RUN fruitless object reads against a table that will \ + never repair", + window.as_millis() / base.as_millis() + ); + + // Leg 2: a repairable row. The run that repairs it must be followed by the BASE + // interval, so a second run lands well inside the same window. + let fx = seed_cid_repos(&slug, &short, &["idlesrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("idlesrc.git"); + let repo = seed_repo(&owner_did, "idlesrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + let (raw_cid, _provider) = + seed_legacy_pin(&pool, &bare, &fx.public_oid, Some(&repo.id)).await; + + crate::ipfs_pin::reset_sweep_runs(); + let busy_loop = spawn_loop(state.db.clone(), state.git_bin.clone()); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while crate::ipfs_pin::sweep_runs() < 2 && std::time::Instant::now() < deadline { + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + let busy_runs = crate::ipfs_pin::sweep_runs(); + busy_loop.abort(); + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + raw_cid, + "fixture precondition: the first run repairs the row" + ); + assert!( + busy_runs >= 2, + "a run that repaired something must keep the BASE interval; backing off \ + after a productive run would stall a table that is still yielding repairs \ + (saw {busy_runs} run(s))" + ); + } + + /// U4 scenario 2 (#173): a legacy row whose object bytes are gone is left exactly + /// as it is by the sweep: never rewritten, never deleted. The row stays withheld + /// until the bytes come back, which is the non-destructive contract the skip-branch + /// repair already holds. + #[sqlx::test] + async fn sweep_leaves_a_bytes_gone_row_untouched(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let _fx = seed_cid_repos(&slug, &short, &["gonesrc"]); + let repo = seed_repo(&owner_did, "gonesrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + + // An oid whose bytes are NOT in the repo, but whose provenance resolves fine. + let phantom_oid = "d".repeat(64); + let raw_cid = + gitlawb_core::cid::Cid::from_git_object_bytes(b"bytes that live nowhere").to_string(); + let provider_cid = legacy_dagpb_cid(&raw_cid); + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) VALUES ($1, $2, $3, $4)", + ) + .bind(&phantom_oid) + .bind(&provider_cid) + .bind("2020-01-01T00:00:00Z") + .bind(&repo.id) + .execute(&pool) + .await + .unwrap(); + + let stats = crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ) + .await; + assert_eq!(stats.repaired, 0, "an unrepairable row is not repaired"); + + let (stored, stashed) = stored_pin(&pool, &phantom_oid).await; + assert_eq!( + stored, provider_cid, + "the bytes-gone row keeps its provider CID (no destructive rewrite)" + ); + assert_eq!(stashed, None, "nothing is stashed when the bytes are gone"); + let count: i64 = sqlx::query_scalar("SELECT count(*) FROM pinned_cids") + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(count, 1, "the row is not deleted"); + } + + /// U4 scenario 3 (#173): the sweep inherits `repair_legacy_provider_cid`'s cost + /// gate, so a row already keyed on a raw CIDv1 is NEVER read for bytes. The + /// test-only `legacy_repair_reads` counter is the both-ways guard: dropping the + /// codec gate reads the raw row and trips it off zero. + #[sqlx::test] + async fn sweep_never_reads_bytes_for_a_raw_cidv1_row(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["rawsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("rawsrc.git"); + let repo = seed_repo(&owner_did, "rawsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + + let raw_cid = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, &repo.id).await; + assert!( + gitlawb_core::cid::is_raw_cidv1(&raw_cid), + "the seeded row is already the canonical resolver key" + ); + + crate::ipfs_pin::reset_legacy_repair_reads(); + let stats = crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ) + .await; + assert_eq!(stats.scanned, 1, "the sweep walked the row"); + assert_eq!(stats.repaired, 0, "a raw row needs no repair"); + assert_eq!( + crate::ipfs_pin::legacy_repair_reads(), + 0, + "a raw-CIDv1 row is never read for bytes (cost gate)" + ); + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + raw_cid, + "the raw row is left as-is" + ); + } + + /// U4 scenario 4 (#173, BOUND): one pass reads at most `batch` rows, so it repairs + /// at most `batch` of them. The exact count is asserted, so raising or removing the + /// bound fails. This is what keeps the sweep from monopolizing the DB on a node + /// with a large `pinned_cids` table. + #[sqlx::test] + async fn sweep_one_pass_is_bounded_by_the_batch_size(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["batchsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("batchsrc.git"); + let repo = seed_repo(&owner_did, "batchsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + + // Five legacy rows, batch of two. + for oid in [ + &fx.public_oid, + &fx.secret_oid, + &fx.public_tree_oid, + &fx.secret_tree_oid, + &fx.commit_oid, + ] { + seed_legacy_pin(&pool, &bare, oid, Some(&repo.id)).await; + } + + let stats = crate::ipfs_pin::sweep_legacy_provider_cids_once( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 2, + &state.db, + &mut Default::default(), + ) + .await + .expect("one pass runs"); + assert_eq!(stats.scanned, 2, "one pass reads exactly the batch size"); + assert_eq!(stats.repaired, 2, "one pass repairs at most the batch size"); + + let repaired: i64 = sqlx::query_scalar( + "SELECT count(*) FROM pinned_cids WHERE legacy_provider_cid IS NOT NULL", + ) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(repaired, 2, "exactly two of the five rows were rewritten"); + } + + /// U4 scenario 5 (#173, RESUMPTION): the walk cursor persists, so a sweep + /// interrupted mid-table continues from where it stopped instead of restarting. + /// Two bounded passes are driven by hand (the restart), and the second pass is + /// asserted to repair the NEXT two rows in cursor order, not the first two again. + /// The read counter proves the already-repaired rows are not re-read. + #[sqlx::test] + async fn sweep_resumes_from_the_persisted_cursor(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["resumesrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("resumesrc.git"); + let repo = seed_repo(&owner_did, "resumesrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + + let mut oids = vec![ + fx.public_oid.clone(), + fx.secret_oid.clone(), + fx.public_tree_oid.clone(), + fx.secret_tree_oid.clone(), + ]; + for oid in &oids { + seed_legacy_pin(&pool, &bare, oid, Some(&repo.id)).await; + } + // The cursor is an ordered walk over the `pinned_cids` primary key. + oids.sort(); + + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let pass1 = crate::ipfs_pin::sweep_legacy_provider_cids_once( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 2, + &state.db, + &mut Default::default(), + ) + .await + .expect("pass 1 runs"); + assert_eq!(pass1.repaired, 2, "pass 1 repairs the first two rows"); + + // The restart: a second pass over the SAME state must continue, not rewind. + crate::ipfs_pin::reset_legacy_repair_reads(); + let pass2 = crate::ipfs_pin::sweep_legacy_provider_cids_once( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 2, + &state.db, + &mut Default::default(), + ) + .await + .expect("pass 2 runs"); + assert_eq!(pass2.repaired, 2, "pass 2 repairs the NEXT two rows"); + assert_eq!( + crate::ipfs_pin::legacy_repair_reads(), + 2, + "pass 2 reads bytes only for the two rows it repaired; the already-repaired \ + rows are not re-read" + ); + for oid in &oids { + let (_cid, stashed) = stored_pin(&pool, oid).await; + assert!( + stashed.is_some(), + "every row is repaired after two resumed passes" + ); + } + } + + /// U4 scenario 7 (#173, cursor liveness): a row that cannot be repaired (NULL + /// provenance, or a provenance whose repo row is gone) is skipped AND the cursor + /// still advances past it. With `batch = 1` the two unrepairable rows sort first, + /// so a cursor that failed to advance would re-read the same row forever and never + /// reach the repairable row behind them. The outer timeout turns that into a + /// FAILURE rather than a hung suite. + #[sqlx::test] + async fn sweep_advances_past_unrepairable_rows(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["skipsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("skipsrc.git"); + let repo = seed_repo(&owner_did, "skipsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + + // Two blockers that sort ahead of any real 64-hex oid: one with NULL + // provenance, one naming a repo row that no longer exists. + let null_prov_oid = "0".repeat(64); + let ghost_repo_oid = format!("{}1", "0".repeat(63)); + seed_legacy_pin(&pool, &bare, &fx.public_oid, Some(&repo.id)).await; + for (oid, prov) in [ + (&null_prov_oid, None), + (&ghost_repo_oid, Some("repo-that-is-gone")), + ] { + let raw = gitlawb_core::cid::Cid::from_git_object_bytes(oid.as_bytes()).to_string(); + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) VALUES ($1, $2, $3, $4)", + ) + .bind(oid) + .bind(legacy_dagpb_cid(&raw)) + .bind("2020-01-01T00:00:00Z") + .bind(prov) + .execute(&pool) + .await + .unwrap(); + } + assert!( + null_prov_oid < fx.public_oid && ghost_repo_oid < fx.public_oid, + "the blockers really do sort ahead of the repairable row" + ); + + let stats = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 1, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the sweep terminates instead of looping on an unrepairable row"); + + assert_eq!( + stats.repaired, 1, + "the sweep advanced past both blockers and repaired the row behind them" + ); + assert!( + stored_pin(&pool, &fx.public_oid).await.1.is_some(), + "the row behind the blockers is the one that got repaired" + ); + for oid in [&null_prov_oid, &ghost_repo_oid] { + assert_eq!( + stored_pin(&pool, oid).await.1, + None, + "an unrepairable row is left untouched" + ); + } + } + + /// U4 scenario 9 (#173, regression): a row skipped for a TRANSIENT reason is + /// retried by a later run. The sweep never pulls a cold repo back from remote + /// storage, so on a Tigris-backed node a repo that is not on local disk at boot + /// contributes nothing to the pass. With the cursor parked at the end of the table + /// that row was skipped FOREVER: every later boot read zero rows and the row stayed + /// unadvertised and unresolvable with nothing left to repair it. Here the repo is + /// off disk for the first run and back for the second, so only a re-walk repairs it. + /// RED before the transient-skip cursor reset (the second run scans nothing). + #[sqlx::test] + async fn sweep_rewalks_after_a_transient_skip(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + + let fx = seed_cid_repos(&slug, &short, &["coldsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("coldsrc.git"); + let repo = seed_repo(&owner_did, "coldsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + let (raw_cid, _provider) = + seed_legacy_pin(&pool, &bare, &fx.public_oid, Some(&repo.id)).await; + + // The repo is COLD: its provenance resolves, but the bytes are not on this + // node's disk right now, exactly the state the sweep refuses to fix by pulling. + let stashed_away = bare.with_extension("git.away"); + let _ = std::fs::remove_dir_all(&stashed_away); + std::fs::rename(&bare, &stashed_away).expect("take the repo off local disk"); + + let first = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the first run terminates"); + assert_eq!( + (first.scanned, first.repaired), + (1, 0), + "the cold repo's row is walked but cannot be repaired yet" + ); + + // The repo is warm again (a later boot, a fetch, an operator restore). + std::fs::rename(&stashed_away, &bare).expect("put the repo back on local disk"); + + let second = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the second run terminates"); + assert_eq!( + second.repaired, 1, + "a later run re-walks the transiently skipped row and repairs it" + ); + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + raw_cid, + "the row now carries the raw-content resolver key" + ); + assert!( + state + .db + .list_pinned_cids() + .await + .unwrap() + .iter() + .any(|r| r.cid == raw_cid), + "the repaired row is advertised again" + ); + } + + /// U4 scenario 10 (#173, the other arm of scenario 9): a PERMANENTLY unrepairable + /// row must not cost anything on a later run. Bytes that are genuinely gone stay + /// gone, so a re-walk must not read object bytes for that row, must not repair it, + /// and must not spin: both runs are timeout-bounded, so a hot loop FAILS here. + /// + /// The assertion is about BOUNDED cost, not about the row going unread (jatmn + /// round 12). It asserted `scanned == 0` while the cursor parked at the table + /// maximum on a clean run; that parking is what let a row written below the cursor + /// by another node go unswept forever, so the run now always rewinds on clean + /// completion. The terminal row is therefore re-walked once per run, and its + /// repair is re-attempted once: the object read is attempted before the bytes are + /// found missing. That cost is real and it is the price of D. What must stay true + /// is that it is exactly ONE attempt per run and never repairs, so a regression + /// that retries the dead row within a run fails here. + #[sqlx::test] + async fn sweep_does_not_rewalk_for_a_terminal_skip(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + + // The repo IS on local disk; the object's bytes are not in it and never will be. + let _fx = seed_cid_repos(&slug, &short, &["termsrc"]); + let repo = seed_repo(&owner_did, "termsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + let phantom_oid = "e".repeat(64); + let raw_cid = + gitlawb_core::cid::Cid::from_git_object_bytes(b"bytes that live nowhere").to_string(); + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) VALUES ($1, $2, $3, $4)", + ) + .bind(&phantom_oid) + .bind(legacy_dagpb_cid(&raw_cid)) + .bind("2020-01-01T00:00:00Z") + .bind(&repo.id) + .execute(&pool) + .await + .unwrap(); + + let first = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the first run terminates"); + assert_eq!( + (first.scanned, first.repaired), + (1, 0), + "the row is walked and cannot be repaired" + ); + + crate::ipfs_pin::reset_legacy_repair_reads(); + let second = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the second run terminates"); + assert_eq!( + (second.repaired, second.passes), + (0, 1), + "the terminal row is still unrepairable and the run does not spin" + ); + assert_eq!( + crate::ipfs_pin::legacy_repair_reads(), + 1, + "the dead row costs exactly one repair attempt per run, never a retry loop" + ); + } + + /// U4 (#173, jatmn round 12): a row inserted BELOW a parked cursor must still be + /// swept. A clean run (no retryable skips) leaves the cursor at the table's maximum + /// `sha256_hex` and every later pass reads only `> cursor`, so a provider-CID row + /// written afterwards by an older node mid-rolling-upgrade whose oid sorts below + /// that maximum is never revisited. The resolver withholds its advertised key, so + /// the object stays unretrievable with nothing left to fix it. The rewind added for + /// the transient-skip case does not cover this: it is gated on `retryable_skips > 0` + /// and a clean pass reports zero. + #[sqlx::test] + async fn sweep_revisits_a_row_written_below_a_parked_cursor(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + + let fx = seed_cid_repos(&slug, &short, &["rollsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("rollsrc.git"); + let repo = seed_repo(&owner_did, "rollsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + + // Two objects from the fixture, ordered by the column the walk is keyed on. + let mut oids = [fx.public_oid.clone(), fx.secret_oid.clone()]; + oids.sort(); + let (low_oid, high_oid) = (oids[0].clone(), oids[1].clone()); + + // First boot: one legacy row, repaired, nothing retryable. Under round 11 this + // is exactly the run that parked the cursor at that row's oid, the table + // maximum, because a clean run reported no retryable skip to rewind for. + seed_legacy_pin(&pool, &bare, &high_oid, Some(&repo.id)).await; + let first = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the first run terminates"); + assert_eq!( + (first.repaired, first.retryable_skips), + (1, 0), + "the first run is a clean completion: nothing retryable to rewind for" + ); + assert_eq!( + state.db.pin_repair_cursor().await.unwrap(), + "", + "a completed run rewinds instead of parking at the table maximum" + ); + + // An older node in the rolling upgrade writes a provider-CID row that sorts + // below where the walk finished, which is where round 11 left the cursor. + let (low_raw, low_provider) = seed_legacy_pin(&pool, &bare, &low_oid, Some(&repo.id)).await; + + // Next boot. + let second = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the second run terminates"); + + let (stored, stashed) = stored_pin(&pool, &low_oid).await; + assert_eq!( + stored, low_raw, + "the row written below the cursor is repaired to the raw-content key \ + (stored {stored}, provider key {low_provider}, second run scanned \ + {} repaired {})", + second.scanned, second.repaired + ); + assert_eq!( + stashed.as_deref(), + Some(low_provider.as_str()), + "its old provider CID is stashed" + ); + assert!( + state + .db + .list_pinned_cids() + .await + .unwrap() + .iter() + .any(|r| r.cid == low_raw), + "the repaired row is advertised again" + ); + } + + /// U4 (#173, round 12, second-model pass): the fruitless reads a run spends on rows + /// whose bytes are permanently gone are bounded per run. The rewind means every + /// later run re-attempts each of them, so without a bound a node that accumulated + /// dead pins (a deleted repo, a force-pushed history) pays `O(dead rows)` git + /// invocations on every boot, forever. The run stops early instead and keeps its + /// cursor, so the next boot resumes past what it already walked. + #[sqlx::test] + async fn sweep_bounds_fruitless_reads_per_run(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let cap = crate::ipfs_pin::MAX_DEAD_ROW_READS_PER_RUN; + let batch: i64 = 16; + + // A real repo on disk, so every row gets as far as spending an object read, and + // objects that were never in it, so every one of those reads is wasted. + let _fx = seed_cid_repos(&slug, &short, &["deadsrc"]); + let repo = seed_repo(&owner_did, "deadsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + let raw_cid = + gitlawb_core::cid::Cid::from_git_object_bytes(b"bytes that live nowhere").to_string(); + let dead_rows = cap + 2 * batch as usize; + for i in 0..dead_rows { + let phantom_oid = format!("{:064x}", i); + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) \ + VALUES ($1, $2, $3, $4)", + ) + .bind(&phantom_oid) + .bind(legacy_dagpb_cid(&raw_cid)) + .bind("2020-01-01T00:00:00Z") + .bind(&repo.id) + .execute(&pool) + .await + .unwrap(); + } + + let stats = tokio::time::timeout( + std::time::Duration::from_secs(120), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + batch, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the run terminates"); + + assert!( + stats.dead_row_reads >= cap, + "the run spends its budget before stopping (spent {})", + stats.dead_row_reads + ); + assert!( + stats.dead_row_reads < cap + batch as usize, + "the run overshoots its budget by at most one batch (spent {}, cap {cap})", + stats.dead_row_reads + ); + assert!( + stats.scanned < dead_rows, + "the run stops short of the table (scanned {} of {dead_rows})", + stats.scanned + ); + + // Not a completed walk, so the cursor is kept and the next run carries on from + // it rather than re-reading the rows this one already paid for. + let cursor = state.db.pin_repair_cursor().await.unwrap(); + assert_ne!(cursor, "", "a run that stops on its budget keeps its place"); + let resumed = state.db.pinned_cids_after(&cursor, batch).await.unwrap(); + assert_eq!( + resumed.first().map(|(sha, _)| sha.as_str()), + Some(format!("{:064x}", stats.scanned).as_str()), + "the next run starts at the row after the last one walked" + ); + } + + /// U4 (#173, round 12, the other side of the unconditional rewind): a run that stops + /// on a pass ERROR keeps its mid-table cursor. The rewind is what a COMPLETED walk + /// does; applying it to a failed one would restart from the beginning of the table + /// on every boot of a node whose DB fails part-way through, and such a node would + /// never reach the rows behind the failure point. The error is induced by renaming + /// `pinned_cids` out from under the walk during the inter-batch sleep. + #[sqlx::test] + async fn sweep_keeps_its_cursor_when_a_pass_fails(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + + let fx = seed_cid_repos(&slug, &short, &["failsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("failsrc.git"); + let repo = seed_repo(&owner_did, "failsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + + let mut oids = [fx.public_oid.clone(), fx.secret_oid.clone()]; + oids.sort(); + for oid in &oids { + seed_legacy_pin(&pool, &bare, oid, Some(&repo.id)).await; + } + + // A batch of one means the first pass is full, so the run sleeps and comes back + // for a second pass. The table is gone by then. + // + // The killer WAITS for the first pass to finish rather than racing a fixed sleep + // against it: the pass writes its cursor as its last act, so a non-empty cursor + // is the signal that the run is now in its inter-batch sleep. A fixed delay here + // fails on a runner slow enough that the rename lands during the first pass's + // own query, which reports `scanned = 0` and asserts something else entirely. + let killer = { + let pool = pool.clone(); + let db = state.db.clone(); + tokio::spawn(async move { + loop { + if !db.pin_repair_cursor().await.unwrap().is_empty() { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + } + sqlx::query("ALTER TABLE pinned_cids RENAME TO pinned_cids_gone") + .execute(&pool) + .await + .expect("rename the table out from under the walk"); + }) + }; + + let stats = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 1, + std::time::Duration::from_millis(300), + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the run terminates on the failed pass"); + killer.await.expect("the killer task completes"); + + assert_eq!( + stats.scanned, 1, + "the first pass read its one row before the table went away" + ); + assert_eq!( + state.db.pin_repair_cursor().await.unwrap(), + oids[0], + "a failed run keeps the position it reached instead of rewinding" + ); + } + + /// U4 scenario 11 (#173, path barrier): the sweep resolves a source repo's disk path + /// through the SAME validated logic the repo store uses, so a repo row whose name + /// carries `..` reads nothing. Names are validated at creation today, so this is a + /// defence-in-depth barrier on a second caller of the raw path helper rather than a + /// live exploit. The escapee repo really does hold the object's bytes, so before the + /// barrier the sweep happily read them from outside `repos_dir` and repaired the row. + /// RED before routing through the validated path (repaired 1). + #[sqlx::test] + async fn sweep_refuses_a_source_path_that_escapes_repos_dir(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + // The bytes live at /tmp/{slug}/escapee.git, OUTSIDE the repos_dir below. + let fx = seed_cid_repos(&slug, &short, &["escapee"]); + let escapee_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("escapee.git"); + let repos_dir = std::path::PathBuf::from("/tmp").join(&slug).join("root"); + std::fs::create_dir_all(repos_dir.join(&slug)).expect("create the repos_dir tree"); + + // A repo row whose name walks back out of repos_dir: repos_dir/{slug}/../../escapee.git + let mut repo = seed_repo(&owner_did, "../../escapee"); + repo.disk_path = escapee_bare.display().to_string(); + state.db.create_repo(&repo).await.expect("seed repo"); + let (_raw_cid, provider_cid) = + seed_legacy_pin(&pool, &escapee_bare, &fx.public_oid, Some(&repo.id)).await; + assert!( + crate::git::store::repo_disk_path(&repos_dir, &owner_did, &repo.name).exists(), + "the unvalidated helper really does resolve to the escapee repo" + ); + + let stats = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + &repos_dir, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the sweep terminates"); + + assert_eq!( + stats.repaired, 0, + "a repo path that escapes repos_dir must never be read" + ); + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + provider_cid, + "the row is untouched because its bytes were never read" + ); + } + + /// U4 scenario 12 (#173, F4): the repair's object read is SYNCHRONOUS `git cat-file`, + /// so running it inline parks the async worker for as long as git takes, up to the + /// whole `git_service_timeout_secs` budget on a wedged read, and the sweep does this + /// per legacy row starting at boot. A slow git stand-in makes that observable: a + /// concurrent 20ms ticker cannot tick at all while the only worker thread is blocked, + /// and ticks freely once the read is on the blocking pool. RED before the + /// `spawn_blocking` (0 ticks). + #[sqlx::test] + async fn repair_object_read_does_not_block_the_async_worker(pool: PgPool) { + use gitlawb_core::identity::Keypair; + use std::sync::atomic::{AtomicUsize, Ordering}; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["slowsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("slowsrc.git"); + let repo = seed_repo(&owner_did, "slowsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + seed_legacy_pin(&pool, &bare, &fx.public_oid, Some(&repo.id)).await; + + // A git that takes 300ms per invocation (the read makes two: type, then content). + let slow_git = std::env::temp_dir().join(format!("gl-slow-git-{short}")); + std::fs::write(&slow_git, "#!/bin/sh\nsleep 0.3\nexec git \"$@\"\n").expect("write shim"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&slow_git, std::fs::Permissions::from_mode(0o755)) + .expect("chmod shim"); + } + + let ticks = std::sync::Arc::new(AtomicUsize::new(0)); + let ticker = { + let ticks = ticks.clone(); + tokio::spawn(async move { + loop { + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + ticks.fetch_add(1, Ordering::Relaxed); + } + }) + }; + + let stats = crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + slow_git.to_str().unwrap(), + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ) + .await; + ticker.abort(); + + assert_eq!(stats.repaired, 1, "the slow git still repairs the row"); + assert!( + ticks.load(Ordering::Relaxed) >= 5, + "the runtime kept running other tasks during the blocking git read (ticks: {})", + ticks.load(Ordering::Relaxed) + ); + } + + /// U4 scenario 8 (#173, degenerate states): an empty `pinned_cids` table and a + /// table with zero legacy rows both complete cleanly, with no repair and no read. + #[sqlx::test] + async fn sweep_completes_on_degenerate_tables(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + + // Empty table. + crate::ipfs_pin::reset_legacy_repair_reads(); + let empty = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 4, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the sweep terminates on an empty table"); + assert_eq!( + (empty.scanned, empty.repaired), + (0, 0), + "an empty table is a clean no-op" + ); + + // Zero legacy rows: every row already carries the canonical raw key. + let fx = seed_cid_repos(&slug, &short, &["degensrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("degensrc.git"); + let repo = seed_repo(&owner_did, "degensrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + for oid in [&fx.public_oid, &fx.secret_oid, &fx.commit_oid] { + pin_cid_for_repo(&bare, oid, &state.db, &repo.id).await; + } + + let clean = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 4, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the sweep terminates on a table with no legacy rows"); + assert_eq!(clean.scanned, 3, "every row is walked"); + assert_eq!(clean.repaired, 0, "nothing needs repair"); + assert_eq!( + crate::ipfs_pin::legacy_repair_reads(), + 0, + "no object bytes are read when no row is legacy" + ); + } + + /// U4 (#173, BOUND): the inter-batch delay is real, observed by wall clock. Five + /// rows at a batch of two means two full batches and a trailing partial one, so the + /// run sleeps twice. Without the sleep the whole run is sub-millisecond DB work and + /// a node's `pinned_cids` table gets walked as fast as Postgres will answer. + #[sqlx::test] + async fn sweep_sleeps_between_batches(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["delaysrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("delaysrc.git"); + let repo = seed_repo(&owner_did, "delaysrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + for oid in [ + &fx.public_oid, + &fx.secret_oid, + &fx.public_tree_oid, + &fx.secret_tree_oid, + &fx.commit_oid, + ] { + pin_cid_for_repo(&bare, oid, &state.db, &repo.id).await; + } + + let delay = std::time::Duration::from_millis(150); + let started = std::time::Instant::now(); + let stats = crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 2, + delay, + &state.db, + &mut Default::default(), + ) + .await; + let elapsed = started.elapsed(); + + assert_eq!( + stats.passes, 3, + "five rows at a batch of two is three passes" + ); + assert!( + elapsed >= delay * 2, + "the run sleeps once between each pair of full batches: {elapsed:?} < {:?}", + delay * 2 + ); + } + + // ---- F1: bounded additive discovery for source-less legacy rows ---- + + /// An empty bare repo at `path`, used as a warm discovery candidate that does not + /// hold the object. sha256 so a 64-hex oid probe is a clean "absent" rather than a + /// format error. + fn init_empty_bare(path: &std::path::Path) { + std::fs::create_dir_all(path.parent().unwrap()).expect("create the owner dir"); + let out = std::process::Command::new("git") + .args([ + "init", + "-q", + "--bare", + "--object-format=sha256", + path.to_str().unwrap(), + ]) + .output() + .expect("git runs"); + assert!( + out.status.success(), + "git init --bare: {}", + String::from_utf8_lossy(&out.stderr) + ); + } + + async fn set_quarantined(pool: &PgPool, repo_id: &str) { + sqlx::query("UPDATE repos SET quarantined = TRUE WHERE id = $1") + .bind(repo_id) + .execute(pool) + .await + .unwrap(); + } + + async fn pinned_repo_id(pool: &PgPool, oid: &str) -> Option { + sqlx::query_scalar("SELECT repo_id FROM pinned_cids WHERE sha256_hex = $1") + .bind(oid) + .fetch_one(pool) + .await + .unwrap() + } + + /// F1 scenario 1 (#173): a pre-provenance row (`repo_id` NULL, no `pin_repo_sources` + /// entry) is repaired by probing warm local repos for the object, and the discovered + /// repo is recorded ADDITIVELY. The must-not half is the last assertion: reading + /// identical bytes proves the repo HOLDS the object, never that it is the FIRST + /// pinner (forks, a shared LICENSE blob and the empty tree all collide), and + /// `backfill_pin_provenance`'s `AND repo_id IS NULL` guard would make a guessed + /// exclusive claim permanent, so `pinned_cids.repo_id` must stay NULL. RED before + /// discovery exists: the source set is empty, the row is skipped, and the cursor + /// advances past it for good. + #[sqlx::test] + async fn sweep_discovery_repairs_sourceless_legacy_row(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["discsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("discsrc.git"); + let repo = seed_repo(&owner_did, "discsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + + // The pre-provenance shape: NULL repo_id and no pin_repo_sources row. + let (raw_cid, provider_cid) = seed_legacy_pin(&pool, &bare, &fx.public_oid, None).await; + assert!( + state + .db + .pin_sources_for_oid(&fx.public_oid) + .await + .unwrap() + .is_empty(), + "the seeded row really has no recorded source" + ); + + let stats = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the sweep terminates"); + assert_eq!(stats.repaired, 1, "discovery repairs the source-less row"); + + let (stored, stashed) = stored_pin(&pool, &fx.public_oid).await; + assert_eq!( + stored, raw_cid, + "the key is rewritten to the raw-content CID from locally verified bytes" + ); + assert_eq!( + stashed.as_deref(), + Some(provider_cid.as_str()), + "the old provider CID is stashed" + ); + assert_eq!( + state.db.pin_sources_for_oid(&fx.public_oid).await.unwrap(), + vec![repo.id.clone()], + "the discovered repo is recorded as an additive source" + ); + assert!( + state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "one discovered holder never proves the set complete, so the marker is set \ + and the resolver's fallback scan stays available" + ); + assert_eq!( + pinned_repo_id(&pool, &fx.public_oid).await, + None, + "discovery makes no exclusive first-pinner claim: repo_id stays NULL" + ); + } + + /// F1 scenario 2 (#173): once discovery has repaired the row it is raw-CIDv1, so a + /// later pass takes the cost gate's cheap path and reads no bytes at all. The cursor + /// is rewound by hand so the second pass really re-walks the row rather than reading + /// nothing because it is behind the cursor. + #[sqlx::test] + async fn sweep_discovery_repaired_row_is_cheap_on_later_passes(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + + let fx = seed_cid_repos(&slug, &short, &["cheapsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("cheapsrc.git"); + let repo = seed_repo(&owner_did, "cheapsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + let (raw_cid, _provider) = seed_legacy_pin(&pool, &bare, &fx.public_oid, None).await; + + let first = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the first run terminates"); + assert_eq!(first.repaired, 1, "the first run repairs by discovery"); + + // Re-walk the same row: the cost gate must spare it every byte read. + state.db.set_pin_repair_cursor("").await.unwrap(); + crate::ipfs_pin::reset_legacy_repair_reads(); + let second = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the second run terminates"); + assert_eq!(second.scanned, 1, "the second run really re-walks the row"); + assert_eq!(second.repaired, 0, "there is nothing left to repair"); + assert_eq!( + crate::ipfs_pin::legacy_repair_reads(), + 0, + "a repaired row is raw-CIDv1, so no later pass reads bytes for it" + ); + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + raw_cid, + "the repaired key survives the second pass" + ); + } + + /// F1 scenario 3 (#173, MUST-NOT): a quarantined repo is hidden from every reader, + /// so it must not become a discovery source either. The only holder here is warm and + /// quarantined, and the filter drops it at candidate-load time, before any probe: the + /// row is left exactly as it is and no bytes are read. + #[sqlx::test] + async fn sweep_discovery_skips_quarantined_holder(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["quarsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("quarsrc.git"); + let repo = seed_repo(&owner_did, "quarsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + set_quarantined(&pool, &repo.id).await; + let (_raw_cid, provider_cid) = seed_legacy_pin(&pool, &bare, &fx.public_oid, None).await; + + crate::ipfs_pin::reset_legacy_repair_reads(); + let stats = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the sweep terminates"); + + assert_eq!( + stats.repaired, 0, + "a quarantined repo never serves as a discovery source" + ); + assert_eq!( + crate::ipfs_pin::legacy_repair_reads(), + 0, + "the quarantine filter drops the candidate before any probe reads bytes" + ); + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + provider_cid, + "the row keeps its provider key" + ); + assert_eq!( + state + .db + .pin_sources_for_oid(&fx.public_oid) + .await + .unwrap() + .len(), + 0, + "no source is recorded from a quarantined repo" + ); + } + + /// F1 scenario 4 (#173, MUST-NOT): a candidate that is not on local disk is COLD. + /// Discovery must not pull it back from remote storage (the sweep is opportunistic + /// background maintenance, not a bulk restore), and it must not mark the row + /// retryable either. + /// + /// The retryable half was originally about the cursor: a cold-candidate retryable + /// would rewind it, and the second run's `scanned` proved it had not. Round 12 made + /// the rewind unconditional on reaching the end of the table, so every completed run + /// rewinds and the second run re-reads the row whatever this one does. What still + /// holds, and what is asserted below, is the COST: a cold candidate is filtered at + /// load, so re-walking the row reads no object bytes and restores nothing. The + /// retryable-skip assertion also still stands on its own terms, since a cold + /// candidate is not evidence about the row. + /// + /// The no-fetch half is asserted here on the EFFECT rather than on the call: the + /// cold candidate's disk path must still not exist after two full runs, which is + /// what any restore (through the repo store, through Tigris, through anything else) + /// would have changed. The control that keeps that assertion from being vacuous is + /// the read from `stashed_away`: the bytes really are still on this node and really + /// would have repaired the row, so declining them is a choice and not an absence. + /// The call-shape half is `sweep_module_never_calls_a_remote_fetch`. + #[sqlx::test] + async fn sweep_discovery_cold_candidates_do_not_rewind(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + + let fx = seed_cid_repos(&slug, &short, &["coldcand"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("coldcand.git"); + let repo = seed_repo(&owner_did, "coldcand"); + state.db.create_repo(&repo).await.expect("seed repo"); + let (raw_cid, provider_cid) = seed_legacy_pin(&pool, &bare, &fx.public_oid, None).await; + + // The only holder goes cold: its row stays in the DB, its bytes leave the disk. + let stashed_away = bare.with_extension("git.away"); + let _ = std::fs::remove_dir_all(&stashed_away); + std::fs::rename(&bare, &stashed_away).expect("take the repo off local disk"); + + crate::ipfs_pin::reset_legacy_repair_reads(); + let first = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the first run terminates"); + assert_eq!( + (first.scanned, first.repaired), + (1, 0), + "the row is walked and no cold candidate can repair it" + ); + // The effect a restore would have left behind. Nothing put the repo back. + assert!( + !bare.exists(), + "the sweep must never materialize a cold candidate on local disk: a repair \ + pass over every pinned row on the node would become a bulk restore" + ); + // Anti-vacuity for the assertion above: the bytes are still reachable on this + // node and still recompute to the raw key, so a fetch would have succeeded and + // repaired the row. The sweep declined an available copy rather than finding + // nothing to take. + let (_ty, stashed_bytes) = crate::git::store::read_object(&stashed_away, &fx.public_oid) + .expect("the stashed copy is readable") + .expect("the stashed copy still holds the object"); + assert_eq!( + gitlawb_core::cid::Cid::from_git_object_bytes(&stashed_bytes).to_string(), + raw_cid, + "the withheld copy is exactly the one that would have repaired the row" + ); + assert_eq!( + first.retryable_skips, 0, + "a cold candidate is not evidence about the row, so it never marks the row \ + retryable and never drives a cursor rewind" + ); + assert_eq!( + crate::ipfs_pin::legacy_repair_reads(), + 0, + "a cold candidate is filtered at load, so nothing is read and nothing is pulled" + ); + + let second = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the second run terminates"); + assert_eq!( + second.retryable_skips, 0, + "the re-walk still finds nothing retryable about the row" + ); + assert_eq!( + crate::ipfs_pin::legacy_repair_reads(), + 0, + "the re-walk costs no object read either: the cold candidate is filtered at \ + load on every run, so an unconditional rewind does not turn into repeated \ + discovery reads for this row" + ); + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + provider_cid, + "the row keeps its provider key" + ); + assert!( + !bare.exists(), + "no later pass materialized the cold candidate either" + ); + } + + /// #173 round 12 (rebase interaction): discovery's probes count against the per-run + /// fruitless-read budget, PER PROBE rather than per row. + /// + /// The two changes met badly. Round 12 made the cursor rewind unconditional on + /// reaching the end of the table, so every completed run re-walks every row; the + /// budget is what stops a node from paying `O(dead rows)` object reads on every boot. + /// But `row_read_attempted` was only ever set in the provenance loop, so a + /// source-less row, the one shape discovery exists for, spent up to + /// `MAX_LEGACY_DISCOVERY_PROBES` reads and contributed nothing to the budget. The + /// per-row cap bounds one row; nothing bounded the run. + /// + /// Counting per row instead of per probe would not do: at 16 probes a row the budget + /// would admit 16 times the reads it names. Several warm candidates here are what + /// distinguishes the two, since the run must stop after far fewer ROWS than the + /// budget's own number. + #[sqlx::test] + async fn sweep_discovery_probes_count_against_the_fruitless_read_budget(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let cap = crate::ipfs_pin::MAX_DEAD_ROW_READS_PER_RUN; + let batch: i64 = 8; + + // Three warm repos, none of which holds the objects below, so every probe reads + // and finds nothing: three fruitless reads per source-less row. + let names = ["u3bdga", "u3bdgb", "u3bdgc"]; + let _fx = seed_cid_repos(&slug, &short, &names); + for n in names { + let repo = seed_repo(&owner_did, n); + state.db.create_repo(&repo).await.expect("seed repo"); + } + let probes_per_row = names.len(); + + // Source-less legacy rows (no repo_id, no pin_repo_sources) for objects that live + // in none of the repos, which is the shape discovery probes and cannot repair. + let raw_cid = + gitlawb_core::cid::Cid::from_git_object_bytes(b"bytes that live nowhere").to_string(); + let rows = cap; // more than the budget allows once each row costs three probes + for i in 0..rows { + sqlx::query("INSERT INTO pinned_cids (sha256_hex, cid, pinned_at) VALUES ($1, $2, $3)") + .bind(format!("{:064x}", i)) + .bind(legacy_dagpb_cid(&raw_cid)) + .bind("2020-01-01T00:00:00Z") + .execute(&pool) + .await + .unwrap(); + } + + let stats = tokio::time::timeout( + std::time::Duration::from_secs(180), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + batch, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the run terminates"); + + assert!( + stats.dead_row_reads >= cap, + "discovery's fruitless probes reach the budget (spent {})", + stats.dead_row_reads + ); + // Per PROBE, not per row: at three probes a row the run must stop after roughly a + // third of the budget's worth of rows, plus at most one batch of overshoot. + // Asserted BEFORE the coarser bound below so that a per-row implementation + // reddens on the property this test is named for. The fixture deliberately holds + // exactly `cap` rows, so per-row counting walks the whole table and would + // otherwise trip the coarse assertion first, reporting the wrong reason. + assert!( + stats.scanned <= cap / probes_per_row + batch as usize, + "the budget counts probes, not rows: scanned {} with a cap of {cap} at \ + {probes_per_row} probes per row", + stats.scanned + ); + assert!( + stats.scanned < rows, + "the run stops short of the table instead of walking all {rows} rows \ + (scanned {})", + stats.scanned + ); + assert_ne!( + state.db.pin_repair_cursor().await.unwrap(), + "", + "a run that stops on its budget keeps its place for the next one" + ); + } + + /// #173 round 12: the RETRYABLE arm of discovery is charged to the budget too, which + /// is the half that differs from the provenance loop and the half an attacker can + /// steer. + /// + /// The cap-reached outcome is retryable BY DESIGN, so that a grindable repo id cannot + /// bury the true holder past the cap permanently. That same design makes it the arm a + /// hostile registrant can hold a row in: register more than the cap's worth of warm + /// repos and every source-less row costs a full cap of reads, on every boot, forever. + /// Charging only the settled arm would leave exactly that uncharged. + #[sqlx::test] + async fn sweep_discovery_charges_a_retryable_cap_reached_row(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let probe_cap = crate::ipfs_pin::MAX_LEGACY_DISCOVERY_PROBES; + + // One more warm repo than the probe cap, so the walk stops with candidates left + // and classifies the row RETRYABLE rather than settled. None holds the object. + let names: Vec = (0..probe_cap + 1).map(|i| format!("u3ret{i}")).collect(); + let name_refs: Vec<&str> = names.iter().map(|s| s.as_str()).collect(); + let _fx = seed_cid_repos(&slug, &short, &name_refs); + for n in &names { + let repo = seed_repo(&owner_did, n); + state.db.create_repo(&repo).await.expect("seed repo"); + } + + let raw_cid = + gitlawb_core::cid::Cid::from_git_object_bytes(b"bytes that live nowhere").to_string(); + sqlx::query("INSERT INTO pinned_cids (sha256_hex, cid, pinned_at) VALUES ($1, $2, $3)") + .bind("a".repeat(64)) + .bind(legacy_dagpb_cid(&raw_cid)) + .bind("2020-01-01T00:00:00Z") + .execute(&pool) + .await + .unwrap(); + + let stats = tokio::time::timeout( + std::time::Duration::from_secs(120), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the run terminates"); + + assert_eq!( + stats.retryable_skips, 1, + "the cap was reached with candidates left, so the row is retryable" + ); + assert_eq!( + stats.repaired, 0, + "no candidate holds the object, so nothing is repaired" + ); + assert_eq!( + stats.dead_row_reads, probe_cap, + "a retryable cap-reached row is charged its full cap of probes, not zero" + ); + } + + /// F1 scenario 5 (#173, BOUND plus anti-burial): the probe cap counts the expensive + /// unit, a bounded object read from a warm repo, so a row costs at most + /// `MAX_LEGACY_DISCOVERY_PROBES` reads however many candidates the node holds. With + /// candidates left over the row is classified RETRYABLE, not terminal: `repo_id` + /// derives from the owner DID, which anyone can grind, so a first-N-wins cap over a + /// sorted set would otherwise let an attacker bury the true holder permanently. + #[sqlx::test] + async fn sweep_discovery_read_probes_are_capped(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + // The bytes live in a bare repo with NO repos row, so it is never a candidate. + let fx = seed_cid_repos(&slug, &short, &["capsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("capsrc.git"); + let (_raw_cid, provider_cid) = seed_legacy_pin(&pool, &bare, &fx.public_oid, None).await; + + // More warm candidates than the cap, none of them holding the object. + let candidates = crate::ipfs_pin::MAX_LEGACY_DISCOVERY_PROBES + 4; + for i in 0..candidates { + let name = format!("capcand{i}"); + init_empty_bare( + &std::path::PathBuf::from("/tmp") + .join(&slug) + .join(format!("{name}.git")), + ); + let repo = seed_repo(&owner_did, &name); + state.db.create_repo(&repo).await.expect("seed candidate"); + } + + crate::ipfs_pin::reset_legacy_repair_reads(); + let stats = tokio::time::timeout( + std::time::Duration::from_secs(60), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the sweep terminates"); + + assert_eq!( + crate::ipfs_pin::legacy_repair_reads(), + crate::ipfs_pin::MAX_LEGACY_DISCOVERY_PROBES, + "one row costs at most the probe cap in object reads, whatever the candidate count" + ); + assert_eq!(stats.repaired, 0, "no candidate holds the object"); + assert_eq!( + stats.retryable_skips, 1, + "cap exhaustion with candidates remaining is RETRYABLE, so a buried holder \ + is re-walked by a later run instead of written off" + ); + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + provider_cid, + "the row is untouched" + ); + } + + /// Write an executable `git` stand-in and return its path. + fn write_git_shim(name: &str, script: &str) -> std::path::PathBuf { + let path = std::env::temp_dir().join(name); + std::fs::write(&path, script).expect("write the git shim"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)) + .expect("chmod the git shim"); + } + path + } + + /// F6 scenario 1 (#173 round 13): one hung candidate must not starve the rows behind + /// it in the same pass. `DiscoveryCtx` is loaded once per pass, so before the per-row + /// slice every source-less row in a pass shared ONE deadline: the first row's wedged + /// `cat-file` spent the whole budget, and every later row reached + /// `repair_legacy_provider_cid` with it already gone, came back retryable without a + /// meaningful probe, and (because `sha256_hex` order is stable) starved on the same + /// row on every boot. + /// + /// Two source-less legacy rows, one warm candidate holding both objects, and a `git` + /// stand-in that wedges on the FIRST row's object and answers the second's for real. + /// With `git_timeout` at 4s the row slice is 1s, so the wedged row costs a quarter of + /// the pass budget and the second row still probes with a live deadline. RED before + /// the slice: the first row burns all 4s and the second is never repaired. + #[sqlx::test] + async fn sweep_discovery_hung_candidate_does_not_starve_later_rows(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["hungsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("hungsrc.git"); + let repo = seed_repo(&owner_did, "hungsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + + // The walk is ordered by `sha256_hex`, so the row that is reached FIRST is the + // lexicographically smaller oid. That is the one the stand-in wedges on. + let mut oids = [fx.public_oid.clone(), fx.secret_oid.clone()]; + oids.sort(); + let (hung_oid, live_oid) = (oids[0].clone(), oids[1].clone()); + let (_hung_raw, hung_provider) = seed_legacy_pin(&pool, &bare, &hung_oid, None).await; + let (live_raw, live_provider) = seed_legacy_pin(&pool, &bare, &live_oid, None).await; + + // The type stage feeds the oid on STDIN (`cat-file --batch-check`) and the + // content stage puts it in argv, so the stand-in has to look in both places. + let git_bin = write_git_shim( + &format!("gl-hung-git-{short}"), + &format!( + "#!/bin/sh\n\ + if [ \"$2\" = \"--batch-check\" ]; then\n\ + \x20 oid=$(cat)\n\ + \x20 case \"$oid\" in\n\ + \x20 {hung_oid}) sleep 30; exit 1 ;;\n\ + \x20 esac\n\ + \x20 printf '%s\\n' \"$oid\" | git \"$@\"\n\ + \x20 exit $?\n\ + fi\n\ + case \"$*\" in\n\ + \x20 *{hung_oid}*) sleep 30; exit 1 ;;\n\ + esac\n\ + exec git \"$@\"\n" + ), + ); + + let stats = tokio::time::timeout( + std::time::Duration::from_secs(60), + crate::ipfs_pin::sweep_legacy_provider_cids_once( + std::path::Path::new("/tmp"), + git_bin.to_str().unwrap(), + std::time::Duration::from_secs(4), + 16, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the pass terminates") + .expect("the pass succeeds"); + + assert_eq!(stats.scanned, 2, "both rows are walked in the one pass"); + assert_eq!( + stored_pin(&pool, &live_oid).await.0, + live_raw, + "the second row still probes with a LIVE deadline and is repaired in the \ + same pass; a hung first row must not spend the whole pass budget" + ); + assert_eq!(stats.repaired, 1, "exactly the second row is repaired"); + assert_eq!( + stored_pin(&pool, &hung_oid).await.0, + hung_provider, + "the wedged row keeps its provider key" + ); + assert_eq!( + stats.retryable_skips, 1, + "the wedged row is retryable, so a later run walks it again" + ); + assert_ne!( + live_raw, live_provider, + "control: the repaired key really differs from the seeded legacy one" + ); + } + + /// F6 scenario 2 (#173 round 13, MUST-NOT): once a pass's whole discovery budget is + /// spent, the rows it has not reached are skipped CHEAPLY and visibly, never folded + /// into ordinary retryable accounting. A row charged for a probe it never meaningfully + /// made burns `MAX_DEAD_ROW_READS_PER_RUN` on nothing, which pauses the run early and + /// (once the discovery continuation lands) would let it advance over windows nobody + /// probed. + /// + /// Seven source-less legacy rows, one warm candidate, and a `git` that wedges on + /// everything. With `git_timeout` at 4s each row slice is 1s, so about four rows spend + /// the pass budget between them and the rest start with it already gone: those charge + /// ZERO reads and the pass reports that it ran out. RED before the skip arm: every row + /// past the first reaches the probe with a dead deadline and is charged a read for it, + /// so `dead_row_reads` equals the row count. + #[sqlx::test] + async fn sweep_discovery_spent_pass_budget_skips_cheaply(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["spentsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("spentsrc.git"); + let repo = seed_repo(&owner_did, "spentsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + + let oids = [ + fx.public_oid.clone(), + fx.secret_oid.clone(), + fx.public_tree_oid.clone(), + fx.secret_tree_oid.clone(), + fx.root_tree_oid.clone(), + fx.commit_oid.clone(), + fx.tag_oid.clone(), + ]; + for oid in &oids { + seed_legacy_pin(&pool, &bare, oid, None).await; + } + + // Wedges on every invocation, so no row can ever be repaired and the only + // question left is what each one COSTS. + let git_bin = write_git_shim( + &format!("gl-spent-git-{short}"), + "#!/bin/sh\nsleep 30\nexit 1\n", + ); + + let stats = tokio::time::timeout( + std::time::Duration::from_secs(60), + crate::ipfs_pin::sweep_legacy_provider_cids_once( + std::path::Path::new("/tmp"), + git_bin.to_str().unwrap(), + std::time::Duration::from_secs(4), + 16, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the pass terminates") + .expect("the pass succeeds"); + + assert_eq!(stats.scanned, oids.len(), "every row is walked"); + assert_eq!(stats.repaired, 0, "a wedged candidate repairs nothing"); + assert!( + stats.dead_row_reads < stats.scanned, + "a row reached after the pass budget is spent must be skipped free, not \ + charged for a probe it cannot make: {} reads charged over {} rows", + stats.dead_row_reads, + stats.scanned + ); + assert!( + stats.dead_row_reads <= 5, + "the pass budget is four row slices wide, so at most the rows that really \ + probed are charged (plus at most one on the boundary); got {}", + stats.dead_row_reads + ); + assert!( + stats.discovery_budget_spent, + "a pass that ran out of discovery budget must SAY so rather than starving \ + its remaining rows silently" + ); + assert_eq!( + stats.retryable_skips, + oids.len(), + "no row is settled: the wedged ones and the unprobed ones are all worth \ + walking again" + ); + } + + // ---- #173 round 13, F5: the per-traversal discovery window continuation ---- + + /// A repo row at a chosen point in the sweep's `(created_at, id)` candidate order, + /// `pos` seconds past a fixed base so the order is the fixture's to set rather than + /// the clock's. Negative positions sort BELOW the base, which is how a fixture models + /// a candidate entering the warm list underneath an already-persisted continuation. + fn seed_repo_at(owner_did: &str, name: &str, pos: i64) -> RepoRecord { + let created_at = chrono::DateTime::parse_from_rfc3339("2020-01-01T12:00:00Z") + .expect("the fixture base parses") + .with_timezone(&Utc) + + chrono::Duration::seconds(pos); + RepoRecord { + created_at, + updated_at: created_at, + ..seed_repo(owner_did, name) + } + } + + /// The keyset key the sweep stores for a candidate: the RAW `created_at` text as + /// `create_repo` wrote it, plus the repo id. + fn candidate_key(repo: &RepoRecord) -> (String, String) { + (repo.created_at.to_rfc3339(), repo.id.clone()) + } + + /// Seed `n` warm candidates in candidate order (position 1 is the oldest). The one at + /// 1-based `holder` is the already-cloned bare named there and really holds the + /// fixture's objects; every other position is an empty bare that holds nothing, so a + /// probe against it costs a real object read and finds nothing. + async fn seed_candidate_ladder( + db: &crate::db::Db, + owner_did: &str, + slug: &str, + prefix: &str, + n: usize, + holder: Option<(usize, &str)>, + ) -> Vec { + let mut rows = Vec::new(); + for pos in 1..=n { + let name = match holder { + Some((hp, hn)) if hp == pos => hn.to_string(), + _ => { + let name = format!("{prefix}{pos}"); + init_empty_bare( + &std::path::PathBuf::from("/tmp") + .join(slug) + .join(format!("{name}.git")), + ); + name + } + }; + let repo = seed_repo_at(owner_did, &name, pos as i64); + db.create_repo(&repo).await.expect("seed candidate"); + rows.push(repo); + } + rows + } + + /// Copy ONE blob between SHA-256 bares, preserving its oid. A bare clone carries + /// every object in the fixture, and the cross-batch scenario needs a candidate that + /// holds exactly one of them. + fn copy_blob_into_bare(src: &std::path::Path, dst: &std::path::Path, oid: &str) { + use std::io::Write; + use std::process::{Command, Stdio}; + let blob = Command::new("git") + .args(["cat-file", "blob", oid]) + .current_dir(src) + .output() + .expect("git runs"); + assert!(blob.status.success(), "cat-file blob {oid}"); + let mut child = Command::new("git") + .args(["hash-object", "-w", "-t", "blob", "--stdin"]) + .current_dir(dst) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .expect("git runs"); + child + .stdin + .take() + .expect("stdin") + .write_all(&blob.stdout) + .expect("feed the blob"); + let out = child.wait_with_output().expect("hash-object finishes"); + assert!(out.status.success(), "hash-object -w"); + assert_eq!( + String::from_utf8_lossy(&out.stdout).trim(), + oid, + "the copied blob keeps its oid, or the fixture is not the object the row names" + ); + } + + /// Poll `f` until it yields a value or `limit` runs out. Several scenarios drive the + /// re-arm wrapper, which on a healthy table never returns, so the assertion has to be + /// on DB state observed while it runs. + async fn poll_until(limit: std::time::Duration, mut f: F) -> Option + where + F: FnMut() -> Fut, + Fut: std::future::Future>, + { + let deadline = std::time::Instant::now() + limit; + loop { + if let Some(v) = f().await { + return Some(v); + } + if std::time::Instant::now() >= deadline { + return None; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + } + + /// The discovery load is bounded to ONE probe window, not to the whole repo table. + /// + /// The exhaustive load was justified as "background maintenance on a timer" whose + /// "paging cost is paid once". That was written when the sweep ran once per boot. + /// The sweep now re-arms on a timer, so a node carrying a single unrepairable + /// source-less row paid a full-table paging pass plus a stat of every warm repo on + /// every re-armed run, forever, to choose sixteen candidates. The idle backoff makes + /// that hourly rather than every five minutes, which is a smaller bill for the same + /// unbounded work. + /// + /// The window itself is unchanged, which is why the assertion is on the PAGING and + /// not on the outcome: an exhaustive load and a bounded one pick the same sixteen + /// candidates and reach the same verdict, so nothing about the result can go red on + /// the difference. The fixture puts more than one window of warm candidates at the + /// front of the `(created_at, id)` order and enough cold rows behind them to push the + /// table past a single page. + /// + /// MUTATION (RED): page to exhaustion and the load buys a second page it has no use + /// for. + #[sqlx::test] + async fn sweep_discovery_load_stops_once_the_window_is_full(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + + // The bytes live in a bare with no `repos` row, so no candidate ever holds them + // and the row stays source-less: the pass runs a full window of probes. + let fx = seed_cid_repos(&slug, &short, &["boundsrc"]); + let src = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("boundsrc.git"); + let _warm = seed_candidate_ladder( + &state.db, + &owner_did, + &slug, + "boundwarm", + crate::ipfs_pin::MAX_LEGACY_DISCOVERY_PROBES + 4, + None, + ) + .await; + // Cold rows: a `repos` row with nothing on disk. They cost a page each but can + // never fill a window slot, so they are what an exhaustive load pages through. + let page_rows = crate::api::ipfs::LEGACY_SCAN_PAGE_ROWS; + for pos in 100..(100 + page_rows) { + let repo = seed_repo_at(&owner_did, &format!("boundcold{pos}"), pos as i64); + state.db.create_repo(&repo).await.expect("seed a cold row"); + } + seed_legacy_pin(&pool, &src, &fx.public_oid, None).await; + + crate::ipfs_pin::reset_discovery_paging(); + let stats = tokio::time::timeout( + std::time::Duration::from_secs(120), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the traversal terminates"); + + let pages = crate::ipfs_pin::discovery_repo_pages(); + let rows = crate::ipfs_pin::discovery_repo_rows(); + assert_eq!( + pages, 1, + "the window fills inside the first page, so the load must stop there; it \ + bought {pages} pages carrying {rows} rows" + ); + assert!( + rows <= page_rows, + "a bounded load reads at most the pages it needs; it read {rows} rows out of \ + a table of {}", + crate::ipfs_pin::MAX_LEGACY_DISCOVERY_PROBES + 4 + page_rows + ); + assert_eq!( + stats.dead_row_reads, + crate::ipfs_pin::MAX_LEGACY_DISCOVERY_PROBES, + "and the window it picked is still a FULL one: bounding the load must not \ + shrink the number of candidates the row actually probes" + ); + } + + /// F5 scenario 1 (#173 round 13): a holder past the probe cap is REACHED. + /// + /// `discover_legacy_row` probes the first `MAX_LEGACY_DISCOVERY_PROBES` of a list + /// ordered `(created_at, id)`. That order is stable and the list was rebuilt from + /// scratch every run, so before the continuation every traversal on every node probed + /// the same oldest sixteen and a holder at position seventeen was unreachable by + /// anything: not a later pass, not a later run, not a reboot. Seventeen warm + /// candidates with only the newest holding the object; the first traversal must + /// repair nothing and persist where it got to, the second must start after that and + /// repair the row, and neither may exceed the probe cap. + /// + /// RED before the rotation: both traversals probe the same first sixteen and the row + /// keeps its provider key forever. + #[sqlx::test] + async fn sweep_discovery_rotation_reaches_later_candidates(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + + // `rotsrc` carries the bytes but has NO repos row, so it is never a candidate; + // `rotheld` is the candidate that really holds them. + let fx = seed_cid_repos(&slug, &short, &["rotsrc", "rotheld"]); + let src = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("rotsrc.git"); + let candidates = seed_candidate_ladder( + &state.db, + &owner_did, + &slug, + "rotcand", + crate::ipfs_pin::MAX_LEGACY_DISCOVERY_PROBES + 1, + Some((crate::ipfs_pin::MAX_LEGACY_DISCOVERY_PROBES + 1, "rotheld")), + ) + .await; + let (raw_cid, provider_cid) = seed_legacy_pin(&pool, &src, &fx.public_oid, None).await; + + let first = tokio::time::timeout( + std::time::Duration::from_secs(120), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the first traversal terminates"); + + assert_eq!( + first.repaired, 0, + "the holder sits past the probe cap, so the first window cannot reach it" + ); + assert_eq!( + first.dead_row_reads, + crate::ipfs_pin::MAX_LEGACY_DISCOVERY_PROBES, + "the first traversal spends exactly one window of probes on the row" + ); + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + provider_cid, + "the row still carries its legacy provider key after the first traversal" + ); + assert_eq!( + state.db.discovery_continuation().await.unwrap(), + candidate_key(&candidates[crate::ipfs_pin::MAX_LEGACY_DISCOVERY_PROBES - 1]), + "a completed traversal that ran out of window persists the last candidate it \ + actually read, so the next one starts after it instead of repeating it" + ); + + let second = tokio::time::timeout( + std::time::Duration::from_secs(120), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the second traversal terminates"); + + assert_eq!( + second.repaired, 1, + "the second traversal's window starts at the seventeenth candidate and \ + repairs the row" + ); + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + raw_cid, + "the key is rewritten to the raw-content CID" + ); + assert!( + second.dead_row_reads <= crate::ipfs_pin::MAX_LEGACY_DISCOVERY_PROBES, + "the rotation moves the window, it does not widen it: {} reads in one \ + traversal", + second.dead_row_reads + ); + assert_ne!(raw_cid, provider_cid, "control: the two keys really differ"); + } + + /// F5 scenario 2 (#173 round 13, MUST-NOT, the steerability negative): candidates + /// appearing between traversals must not move the window off the holder. + /// + /// The continuation is a keyset KEY, not an offset, and this is the difference. + /// Freshly registered repos sort LAST under `created_at` and cannot be backdated, so + /// they can only ever land behind the window. Candidates can also enter BELOW the + /// continuation without any mint at all: a cold repo warms on a Tigris-backed node, + /// an operator restores an archived one. Every one of those silently renumbers an + /// offset, and sixteen of them slide an offset window clean off the candidate it was + /// about to reach, while a key names the boundary itself and does not care what + /// appeared underneath it. + /// + /// RED under an offset continuation: the second traversal's window starts sixteen + /// entries into a list that grew underneath it and never reaches the holder. + #[sqlx::test] + async fn sweep_discovery_rotation_survives_minted_candidates(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let cap = crate::ipfs_pin::MAX_LEGACY_DISCOVERY_PROBES; + + let fx = seed_cid_repos(&slug, &short, &["mintsrc", "mintheld"]); + let src = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("mintsrc.git"); + let candidates = seed_candidate_ladder( + &state.db, + &owner_did, + &slug, + "mintcand", + cap + 1, + Some((cap + 1, "mintheld")), + ) + .await; + let (raw_cid, provider_cid) = seed_legacy_pin(&pool, &src, &fx.public_oid, None).await; + + tokio::time::timeout( + std::time::Duration::from_secs(120), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the first traversal terminates"); + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + provider_cid, + "precondition: the first window does not reach the holder" + ); + let boundary = state.db.discovery_continuation().await.unwrap(); + assert_eq!( + boundary, + candidate_key(&candidates[cap - 1]), + "precondition: the window boundary is the sixteenth candidate" + ); + + // The mint: several brand-new repos. They sort last and are the only thing an + // attacker who can grind owner DIDs actually gets to do. + for i in 0..5 { + let name = format!("minted{i}"); + init_empty_bare( + &std::path::PathBuf::from("/tmp") + .join(&slug) + .join(format!("{name}.git")), + ); + state + .db + .create_repo(&seed_repo_at(&owner_did, &name, 100 + i)) + .await + .expect("seed a minted candidate"); + } + // And a whole window's worth entering BELOW the boundary, which is what an + // offset silently mistakes for a move of the boundary itself. + for i in 0..cap { + let name = format!("warmed{i}"); + init_empty_bare( + &std::path::PathBuf::from("/tmp") + .join(&slug) + .join(format!("{name}.git")), + ); + state + .db + .create_repo(&seed_repo_at(&owner_did, &name, -(i as i64) - 1)) + .await + .expect("seed a candidate below the boundary"); + } + + let second = tokio::time::timeout( + std::time::Duration::from_secs(120), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the second traversal terminates"); + + assert_eq!( + second.repaired, 1, + "the window boundary is a key, so twenty-one candidates arriving between \ + traversals leave it exactly where the first traversal put it and the holder \ + is still the next thing read" + ); + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + raw_cid, + "the holder's row is repaired despite the churn" + ); + } + + /// F5 scenario 3 (#173 round 13): a candidate list that has shrunk below one window + /// RESETS the continuation instead of stranding it past the end of the list. + /// + /// The migration's own success shrinks the list (repos go cold, get deleted), and a + /// continuation left pointing past everything would rotate each later traversal to an + /// empty tail and then wrap to the same prefix forever. Once the whole warm list fits + /// in one window there is no next window to advance to, so the traversal resets. + #[sqlx::test] + async fn sweep_discovery_shrunken_list_resets_the_continuation(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let cap = crate::ipfs_pin::MAX_LEGACY_DISCOVERY_PROBES; + + let fx = seed_cid_repos(&slug, &short, &["shrinksrc"]); + let src = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("shrinksrc.git"); + // Nothing warm holds the object, so the row stays unrepaired and every traversal + // spends a full window on it. + let candidates = + seed_candidate_ladder(&state.db, &owner_did, &slug, "shrinkcand", cap + 1, None).await; + seed_legacy_pin(&pool, &src, &fx.public_oid, None).await; + + tokio::time::timeout( + std::time::Duration::from_secs(120), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the first traversal terminates"); + assert_eq!( + state.db.discovery_continuation().await.unwrap(), + candidate_key(&candidates[cap - 1]), + "precondition: the traversal parked the continuation past the head" + ); + + // The list shrinks to two: everything but the two oldest goes away. + for repo in candidates.iter().skip(2) { + sqlx::query("DELETE FROM repos WHERE id = $1") + .bind(&repo.id) + .execute(&pool) + .await + .unwrap(); + } + + tokio::time::timeout( + std::time::Duration::from_secs(120), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the second traversal terminates"); + + assert_eq!( + state.db.discovery_continuation().await.unwrap(), + (String::new(), String::new()), + "once the whole warm list fits in one window there is no next window, so the \ + continuation resets rather than pointing past the end of a shrunken list" + ); + } + + /// F5 scenario 5 (#173 round 13, MUST-NOT): a traversal may only advance over + /// candidates it really READ, never over the ones it merely walked past with a dead + /// deadline. + /// + /// U3 gives each source-less row a slice of the pass budget and skips a row reached + /// with the pass budget already gone. What it does NOT skip is the candidates behind + /// a wedged one INSIDE a row: those still enter the probe loop, still get charged a + /// read, and still come back retryable, but `db_bounded` returns on the spent + /// deadline without touching the repo. Advancing over them would burn a window nobody + /// looked in, which is the same hole the continuation exists to close. + /// + /// Twenty warm candidates, seven source-less rows, and a `git` that wedges on + /// everything. With `git_timeout` at 4s each row slice is 1s, so each row that probes + /// at all spends its whole slice on candidate ONE and walks the other fifteen with a + /// dead deadline. The traversal may advance to candidate one and no further. + #[sqlx::test] + async fn sweep_discovery_starved_traversal_advances_only_over_live_probes(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let cap = crate::ipfs_pin::MAX_LEGACY_DISCOVERY_PROBES; + + let fx = seed_cid_repos(&slug, &short, &["starvesrc"]); + let src = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("starvesrc.git"); + let candidates = + seed_candidate_ladder(&state.db, &owner_did, &slug, "starvecand", cap + 4, None).await; + for oid in [ + &fx.public_oid, + &fx.secret_oid, + &fx.public_tree_oid, + &fx.secret_tree_oid, + &fx.root_tree_oid, + &fx.commit_oid, + &fx.tag_oid, + ] { + seed_legacy_pin(&pool, &src, oid, None).await; + } + + let git_bin = write_git_shim( + &format!("gl-starve-git-{short}"), + "#!/bin/sh\nsleep 30\nexit 1\n", + ); + + tokio::time::timeout( + std::time::Duration::from_secs(120), + crate::ipfs_pin::sweep_legacy_provider_cids_once( + std::path::Path::new("/tmp"), + git_bin.to_str().unwrap(), + std::time::Duration::from_secs(4), + 16, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the traversal terminates") + .expect("the traversal succeeds"); + + let seen = state.db.discovery_continuation().await.unwrap(); + assert_eq!( + seen, + candidate_key(&candidates[0]), + "the only candidate any row read with a live deadline is the first, so that \ + is exactly how far the traversal may advance" + ); + assert_ne!( + seen, + candidate_key(&candidates[cap - 1]), + "advancing to the end of the window would skip fifteen candidates that were \ + charged a read but never actually looked at" + ); + } + + /// F5 scenario 6 (#173 round 13, MUST-NOT): a candidate that wedges MID-window must + /// not carry the continuation past the candidates behind it. + /// + /// The sharp version of the live-budget rule, and the one a window's-end advance gets + /// wrong while looking correct. Twenty-four warm candidates, the holder at position + /// twelve, and a `git` that wedges only in the repo at position nine. Positions one + /// to eight are read for real, nine eats the row's whole slice, and ten through + /// sixteen are charged a read apiece against a dead deadline without the repo ever + /// being opened. The continuation may advance to nine and no further, or the holder + /// at twelve is skipped by a traversal that never looked at it. + #[sqlx::test] + async fn sweep_discovery_hung_mid_window_does_not_skip_unread_candidates(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let cap = crate::ipfs_pin::MAX_LEGACY_DISCOVERY_PROBES; + + let fx = seed_cid_repos(&slug, &short, &["midsrc", "midheld"]); + let src = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("midsrc.git"); + let candidates = seed_candidate_ladder( + &state.db, + &owner_did, + &slug, + "midcand", + cap + 8, + Some((12, "midheld")), + ) + .await; + let (raw_cid, provider_cid) = seed_legacy_pin(&pool, &src, &fx.public_oid, None).await; + + // Wedges only inside the position-nine repo, which the sweep enters by cwd. + let git_bin = write_git_shim( + &format!("gl-mid-git-{short}"), + "#!/bin/sh\ncase \"$(pwd)\" in\n */midcand9.git) sleep 30; exit 1 ;;\nesac\nexec git \"$@\"\n", + ); + + let first = tokio::time::timeout( + std::time::Duration::from_secs(180), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + git_bin.to_str().unwrap(), + std::time::Duration::from_secs(16), + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the first traversal terminates"); + + assert_eq!( + first.repaired, 0, + "the wedged candidate spends the row's slice, so the holder behind it is \ + charged a read but never actually read" + ); + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + provider_cid, + "precondition: the first traversal leaves the row on its provider key" + ); + assert_eq!( + state.db.discovery_continuation().await.unwrap(), + candidate_key(&candidates[8]), + "the last candidate read with a live deadline is the wedged one at position \ + nine, so that is the boundary; anything further skips unread candidates" + ); + + let second = tokio::time::timeout( + std::time::Duration::from_secs(180), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + git_bin.to_str().unwrap(), + std::time::Duration::from_secs(16), + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the second traversal terminates"); + + assert_eq!( + second.repaired, 1, + "the next traversal starts at position ten and reaches the holder at twelve" + ); + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + raw_cid, + "the row is repaired from the candidate the hang had hidden" + ); + } + + /// F5 scenario 7 (#173 round 13): the continuation survives the future being DROPPED. + /// + /// `spawn_legacy_cid_sweep` runs the sweep inside a `tokio::select!` against the + /// shutdown watcher, so on shutdown the sweep future is dropped wherever it happens + /// to be. The re-arm wrapper never returns on a healthy node, so a persist written on + /// the way out of the wrapper would never be written at all. Persisting inside the + /// traversal-ending pass is what makes a shutdown cost at most a repeated window. + #[sqlx::test] + async fn sweep_continuation_survives_dropped_sweep_future(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let cap = crate::ipfs_pin::MAX_LEGACY_DISCOVERY_PROBES; + + let fx = seed_cid_repos(&slug, &short, &["dropsrc"]); + let src = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("dropsrc.git"); + let candidates = + seed_candidate_ladder(&state.db, &owner_did, &slug, "dropcand", cap + 1, None).await; + seed_legacy_pin(&pool, &src, &fx.public_oid, None).await; + + // The wrapper completes a traversal and then parks on its re-arm sleep, which is + // exactly where a shutdown drops it in production. + let expected = candidate_key(&candidates[cap - 1]); + let observed = tokio::select! { + _ = crate::ipfs_pin::run_sweep_rearmed( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + std::time::Duration::from_secs(3600), + &state.db, + ) => None, + v = poll_until(std::time::Duration::from_secs(120), || async { + let c = state.db.discovery_continuation().await.unwrap(); + (c != (String::new(), String::new())).then_some(c) + }) => v, + }; + + assert_eq!( + observed, + Some(expected), + "the traversal-ending pass persists the continuation, so dropping the sweep \ + future afterwards keeps the advance the traversal earned" + ); + } + + /// F5 scenario 8 (#173 round 13, MUST-NOT, the aliasing negative): two source-less + /// rows in DIFFERENT passes of the same traversal share one window. + /// + /// The continuation advances once per TRAVERSAL, not once per pass. Per pass, a row's + /// window index across traversals is `(t*m + j) mod W` for `m` rows and `W` windows, + /// so with `m` and `W` sharing a factor a given row only ever visits `W / gcd(m, W)` + /// of the windows and orbits a strict subset forever. Two rows, two windows: under + /// per-pass advancement the first row is pinned to window one for the life of the + /// node and its holder in window two is unreachable. + /// + /// Thirty-two warm candidates, `batch = 1` so the two rows land in different passes, + /// the first row's holder at position twenty, the second row's object nowhere at all. + #[sqlx::test] + async fn sweep_discovery_rows_in_different_batches_share_windows(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let cap = crate::ipfs_pin::MAX_LEGACY_DISCOVERY_PROBES; + + let fx = seed_cid_repos(&slug, &short, &["alisrc"]); + let src = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("alisrc.git"); + // The walk is ordered by `sha256_hex`, so the smaller oid is the row read first. + let mut oids = [fx.public_oid.clone(), fx.secret_oid.clone()]; + oids.sort(); + let (first_row, second_row) = (oids[0].clone(), oids[1].clone()); + + let candidates = + seed_candidate_ladder(&state.db, &owner_did, &slug, "alicand", 2 * cap, None).await; + // Position twenty holds ONLY the first row's blob: a bare clone would carry both + // and the second row would stop being the unrepairable control. + copy_blob_into_bare( + &src, + &std::path::PathBuf::from("/tmp") + .join(&slug) + .join("alicand20.git"), + &first_row, + ); + let (first_raw, first_provider) = seed_legacy_pin(&pool, &src, &first_row, None).await; + seed_legacy_pin(&pool, &src, &second_row, None).await; + + let log = std::env::temp_dir().join(format!("gl-ali-log-{short}")); + let _ = std::fs::remove_file(&log); + let git_bin = write_git_shim( + &format!("gl-ali-git-{short}"), + &format!( + "#!/bin/sh\n\ + if [ \"$2\" = \"--batch-check\" ]; then\n\ + \x20 oid=$(cat)\n\ + \x20 printf '%s %s\\n' \"$oid\" \"$(basename $(pwd))\" >> {log}\n\ + \x20 printf '%s\\n' \"$oid\" | git \"$@\"\n\ + \x20 exit $?\n\ + fi\n\ + exec git \"$@\"\n", + log = log.display() + ), + ); + + let mut traversal = crate::ipfs_pin::DiscoveryTraversalState::default(); + for _ in 0..2 { + tokio::time::timeout( + std::time::Duration::from_secs(180), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + git_bin.to_str().unwrap(), + git_timeout, + 1, + std::time::Duration::ZERO, + &state.db, + &mut traversal, + ), + ) + .await + .expect("the traversal terminates"); + } + + assert_eq!( + stored_pin(&pool, &first_row).await.0, + first_raw, + "the first row's holder is in the second window, which it only reaches if \ + both rows moved through the windows together" + ); + assert_ne!( + first_raw, first_provider, + "control: the repaired key really differs from the seeded legacy one" + ); + + // The invocation log proves the shared window rather than inferring it: inside + // one traversal both oids must have been probed against the same repos. + let text = std::fs::read_to_string(&log).expect("the shim logged its probes"); + let _ = std::fs::remove_file(&log); + let repos_for = |oid: &str| -> Vec { + let mut v: Vec = text + .lines() + .filter_map(|l| l.split_once(' ')) + .filter(|(o, _)| *o == oid) + .map(|(_, r)| r.to_string()) + .collect(); + v.sort(); + v.dedup(); + v + }; + let first_probed = repos_for(&first_row); + let second_probed = repos_for(&second_row); + assert!( + first_probed.len() >= cap && second_probed.len() >= cap, + "precondition: both rows really probed a full window ({} and {})", + first_probed.len(), + second_probed.len() + ); + let window_one: Vec = { + let mut v: Vec = candidates[..cap] + .iter() + .map(|r| format!("{}.git", r.name)) + .collect(); + v.sort(); + v + }; + for repo in &window_one { + assert!( + first_probed.contains(repo) && second_probed.contains(repo), + "both rows must have probed {repo} in the first traversal; per-pass \ + advancement would have handed the second row a different window" + ); + } + } + + /// F5 scenario 9 (#173 round 13): the dead-read cap PAUSES a run, and the re-arm is + /// what keeps coverage moving afterwards. + /// + /// Before the wrapper the sweep ran exactly once per boot, so the window advanced + /// once per boot too and a node that never reboots never advanced past its first + /// window. Five unrepairable source-less rows at a full window of probes each blow + /// through `MAX_DEAD_ROW_READS_PER_RUN` inside one run; the holder sits in the second + /// window, so it is reachable only across re-arms. + #[sqlx::test] + async fn sweep_rearm_advances_past_dead_read_cap(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let cap = crate::ipfs_pin::MAX_LEGACY_DISCOVERY_PROBES; + + let fx = seed_cid_repos(&slug, &short, &["rearmsrc", "rearmheld"]); + let src = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("rearmsrc.git"); + seed_candidate_ladder( + &state.db, + &owner_did, + &slug, + "rearmcand", + cap + 4, + Some((cap + 2, "rearmheld")), + ) + .await; + let mut oids = [ + fx.public_oid.clone(), + fx.secret_oid.clone(), + fx.public_tree_oid.clone(), + fx.secret_tree_oid.clone(), + fx.root_tree_oid.clone(), + ]; + oids.sort(); + for oid in &oids { + seed_legacy_pin(&pool, &src, oid, None).await; + } + let target = oids.last().unwrap().clone(); + let target_raw = { + let (_ty, bytes) = crate::git::store::read_object(&src, &target) + .expect("read the object") + .expect("the object exists"); + gitlawb_core::cid::Cid::from_git_object_bytes(&bytes).to_string() + }; + + // One run, driven directly: five rows at a full window each is more fruitless + // reading than one run will do, so it PAUSES rather than completing. + let mut traversal = crate::ipfs_pin::DiscoveryTraversalState::default(); + let run = tokio::time::timeout( + std::time::Duration::from_secs(300), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 1, + std::time::Duration::ZERO, + &state.db, + &mut traversal, + ), + ) + .await + .expect("the run terminates"); + assert_eq!( + run.stop, + crate::ipfs_pin::SweepStop::PausedOnDeadReadCap, + "one run cannot walk this table: it stops on the dead-read cap with the \ + cursor mid-table" + ); + assert_eq!(run.repaired, 0, "the holder is past the first window"); + + let repaired = tokio::select! { + _ = crate::ipfs_pin::run_sweep_rearmed( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 1, + std::time::Duration::ZERO, + std::time::Duration::ZERO, + &state.db, + ) => false, + v = poll_until(std::time::Duration::from_secs(300), || async { + (stored_pin(&pool, &target).await.0 == target_raw).then_some(()) + }) => v.is_some(), + }; + assert!( + repaired, + "a run that pauses on the dead-read cap is re-armed, so traversals keep \ + completing and the window keeps advancing until the holder is reached" + ); + } + + /// F5 scenario 10 (#173 round 13, MUST-NOT): a cap pause AFTER the traversal's last + /// source-less row must not lose the advance that traversal earned. + /// + /// The traversal accumulator is scoped to the TRAVERSAL, not the run, and this is the + /// case that separates the two. The run that probes the window is paused by the + /// dead-read cap before it reaches the end of the table; a LATER run reads the short + /// batch and is the one that applies the advance. Rebuild the accumulator per run and + /// that later run sees nothing recorded, applies the hold arm, and the window never + /// moves however many times the sweep re-arms. + /// + /// One source-less row against twenty warm candidates (a full window of probes), + /// then enough bytes-gone rows behind it to trip the cap before the short batch. + #[sqlx::test] + async fn sweep_cap_pause_after_last_discovery_row_still_advances(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let cap = crate::ipfs_pin::MAX_LEGACY_DISCOVERY_PROBES; + + let fx = seed_cid_repos(&slug, &short, &["pausesrc"]); + let src = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("pausesrc.git"); + let candidates = + seed_candidate_ladder(&state.db, &owner_did, &slug, "pausecand", cap + 4, None).await; + let (_raw, provider) = seed_legacy_pin(&pool, &src, &fx.public_oid, None).await; + assert!( + !fx.public_oid.starts_with("ff"), + "precondition: the discovery row sorts before the synthetic bytes-gone rows" + ); + + // Bytes-gone rows: real provenance pointing at a warm repo that does not hold + // them, so each costs exactly one fruitless read. Enough of them that the run + // trips the cap with the end of the table still ahead of it. + let ghost = &candidates[0]; + let needed = crate::ipfs_pin::MAX_DEAD_ROW_READS_PER_RUN - cap; + for i in 0..needed { + let oid = format!("ff{:062x}", i); + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) + VALUES ($1, $2, $3, NULL)", + ) + .bind(&oid) + .bind(&provider) + .bind("2020-01-01T00:00:00Z") + .execute(&pool) + .await + .unwrap(); + state + .db + .record_pin_source(&oid, &ghost.id) + .await + .expect("record the ghost source"); + } + + let expected = candidate_key(&candidates[cap - 1]); + let observed = tokio::select! { + _ = crate::ipfs_pin::run_sweep_rearmed( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 1, + std::time::Duration::ZERO, + std::time::Duration::ZERO, + &state.db, + ) => None, + v = poll_until(std::time::Duration::from_secs(300), || async { + let c = state.db.discovery_continuation().await.unwrap(); + (c != (String::new(), String::new())).then_some(c) + }) => v, + }; + + assert_eq!( + observed, + Some(expected), + "the run that probed the window was paused by the dead-read cap, so a LATER \ + run ends the traversal; the advance it applies has to come from the \ + traversal's accumulator, not that run's" + ); + } + + /// F1 scenario 6 (#173, the collision case): two warm repos hold identical bytes, + /// which is the shape (forks, a shared LICENSE blob, the empty tree) that makes an + /// exclusive first-pinner claim wrong. Discovery records ONE additive source and + /// sets the incomplete marker, and the marker is what keeps `needs_scan` true so a + /// caller who can only read the OTHER holder is still served. Under an exclusive + /// claim `needs_scan` would be false and that caller would get a 404 for a public + /// object. + #[sqlx::test] + async fn sweep_discovery_multi_holder_serves_both_readers(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["holda", "holdb"]); + let bare_a = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("holda.git"); + + // A is older, so the oldest-first probe order selects it; it is PRIVATE, so it + // denies the anonymous caller. B is public and holds the same bytes. + let mut repo_a = seed_repo(&owner_did, "holda"); + repo_a.is_public = false; + repo_a.created_at = Utc::now() - chrono::Duration::days(2); + let repo_b = seed_repo(&owner_did, "holdb"); + state.db.create_repo(&repo_a).await.expect("seed repo a"); + state.db.create_repo(&repo_b).await.expect("seed repo b"); + + let (raw_cid, _provider) = seed_legacy_pin(&pool, &bare_a, &fx.public_oid, None).await; + + let stats = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the sweep terminates"); + assert_eq!( + stats.repaired, 1, + "the row is repaired from the first holder" + ); + assert_eq!( + state.db.pin_sources_for_oid(&fx.public_oid).await.unwrap(), + vec![repo_a.id.clone()], + "exactly one additive source is recorded, the oldest-first selection" + ); + assert!( + state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "the marker records that discovery does not know the full source set" + ); + assert_eq!( + pinned_repo_id(&pool, &fx.public_oid).await, + None, + "no exclusive claim is written for either holder" + ); + + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&raw_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "a caller who can read only the NON-selected holder is still served" + ); + assert!(body.contains("public bytes"), "the object's bytes serve"); + } + + /// U5 (#173, F7): the discovered-source row and the fallback-arming sentinel commit + /// together or not at all, so discovery can never leave a row in the one state the + /// resolver reads as complete while it is not: a nonempty, below-cap, UNMARKED + /// source set. + /// + /// Same shape as the multi-holder test above (older private holder selected, newer + /// public holder unrecorded), plus a fault that fails ONLY the marker insert: a + /// `BEFORE INSERT` trigger on `pin_source_failures` that raises. Under the pre-fix + /// two-call shape the source insert commits (its own transaction's second statement + /// is a DELETE, which an insert trigger does not fire) and the separate marker insert + /// then fails, leaving the source set holding the private holder alone with no + /// marker; `needs_scan` is `sources.is_empty() || at_cap || incomplete`, so all three + /// signals are off, the resolver drops its fallback scan, and the public duplicate is + /// permanently 404'd for the anonymous caller. One transaction makes that state + /// unreachable: the marker's failure rolls the source row back with it, the set stays + /// EMPTY, and the empty-set signal routes the request to the fallback scan. + #[sqlx::test] + async fn sweep_discovery_failed_marker_does_not_strand_public_copy(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["stranda", "strandb"]); + let bare_a = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("stranda.git"); + + // A is older, so the oldest-first probe order selects it; it is PRIVATE, so it + // denies the anonymous caller. B is public and holds the same bytes. + let mut repo_a = seed_repo(&owner_did, "stranda"); + repo_a.is_public = false; + repo_a.created_at = Utc::now() - chrono::Duration::days(2); + let repo_b = seed_repo(&owner_did, "strandb"); + state.db.create_repo(&repo_a).await.expect("seed repo a"); + state.db.create_repo(&repo_b).await.expect("seed repo b"); + + let (raw_cid, _provider) = seed_legacy_pin(&pool, &bare_a, &fx.public_oid, None).await; + + // The fault, installed AFTER migrations: every insert into `pin_source_failures` + // raises. Postgres triggers cannot raise inline, hence the plpgsql function. + // Deliberately NOT a `DROP TABLE`: the source-record transaction's own DELETE on + // this table would then error too, that transaction would roll back, and the + // pre-fix run would land the same empty set as the post-fix one, so the test + // would pass for the wrong reason. + sqlx::query( + "CREATE FUNCTION fail_pin_source_failure_insert() RETURNS trigger AS $$ + BEGIN RAISE EXCEPTION 'injected pin_source_failures insert failure'; END; + $$ LANGUAGE plpgsql", + ) + .execute(&pool) + .await + .expect("install the fault function"); + sqlx::query( + "CREATE TRIGGER fail_pin_source_failure_insert + BEFORE INSERT ON pin_source_failures + FOR EACH ROW EXECUTE FUNCTION fail_pin_source_failure_insert()", + ) + .execute(&pool) + .await + .expect("install the fault trigger"); + + let stats = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the sweep terminates"); + assert_eq!( + stats.repaired, 1, + "the row is still repaired to its raw key; only the source record is at risk" + ); + + // Gathered before the assertions so the RED output carries the half-state. + let sources = state + .db + .pin_sources_for_oid(&fx.public_oid) + .await + .expect("read the source set"); + let marker_rows: i64 = + sqlx::query_scalar("SELECT count(*) FROM pin_source_failures WHERE sha256_hex = $1") + .bind(&fx.public_oid) + .fetch_one(&pool) + .await + .expect("read the marker table"); + + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&raw_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + (st, body.contains("public bytes")), + (StatusCode::OK, true), + "the anonymous caller must be served the PUBLIC holder's copy through the \ + fallback scan; got {st} with source set {sources:?} and {marker_rows} marker \ + row(s), the nonempty-and-unmarked half-state the resolver reads as complete" + ); + assert!( + sources.is_empty(), + "the failed marker must roll the source row back with it; got {sources:?}" + ); + assert_eq!( + marker_rows, 0, + "the marker insert is what failed, so no marker row can exist" + ); + } + + /// U5 (#173, F7, the healthy direction): one discovery hit writes BOTH rows, asserted + /// against the tables directly rather than through the boolean helper. The sentinel + /// is unconditional because one discovered holder out of a bounded warm-only + /// candidate set never proves the source set complete, and it is written against the + /// empty-string UNKNOWN-repo sentinel so no later real record clears it. + #[sqlx::test] + async fn sweep_discovery_records_source_and_sentinel_in_one_commit(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["bothrows"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("bothrows.git"); + let repo = seed_repo(&owner_did, "bothrows"); + state.db.create_repo(&repo).await.expect("seed repo"); + + seed_legacy_pin(&pool, &bare, &fx.public_oid, None).await; + + let stats = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the sweep terminates"); + assert_eq!(stats.repaired, 1, "the discovered row is repaired"); + + let source_rows: Vec = sqlx::query_scalar( + "SELECT repo_id FROM pin_repo_sources WHERE sha256_hex = $1 ORDER BY repo_id", + ) + .bind(&fx.public_oid) + .fetch_all(&pool) + .await + .expect("read pin_repo_sources"); + assert_eq!( + source_rows, + vec![repo.id.clone()], + "the discovered holder is recorded additively" + ); + + let marker_repos: Vec = sqlx::query_scalar( + "SELECT repo_id FROM pin_source_failures WHERE sha256_hex = $1 ORDER BY repo_id", + ) + .bind(&fx.public_oid) + .fetch_all(&pool) + .await + .expect("read pin_source_failures"); + assert_eq!( + marker_repos, + vec![String::new()], + "the same commit writes the unknown-repo sentinel: one discovered holder never \ + proves the set complete, so the resolver must keep its fallback scan" + ); + } + + /// F1 scenario 7 (#173, degenerate state): the cost gate at the top of the row loop + /// fires before the sources query, so a source-less row that is ALREADY raw-CIDv1 + /// never enters discovery and reads nothing. + #[sqlx::test] + async fn sweep_discovery_never_runs_for_a_raw_cidv1_sourceless_row(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["rawnosrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("rawnosrc.git"); + let repo = seed_repo(&owner_did, "rawnosrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + // No provenance recorded: `pin_cid_for` stores the raw key with a NULL repo_id. + let raw_cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + + crate::ipfs_pin::reset_legacy_repair_reads(); + let stats = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the sweep terminates"); + + assert_eq!(stats.scanned, 1, "the row is walked"); + assert_eq!(stats.repaired, 0, "a raw-CIDv1 row needs no repair"); + assert_eq!( + crate::ipfs_pin::legacy_repair_reads(), + 0, + "the cost gate spares a raw row every byte read, discovery included" + ); + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + raw_cid, + "the raw row is left as-is" + ); + } + + /// F1 scenario 8 (#173, the negative direction of the new branch): a row WITH a + /// recorded source resolves through the existing source loop only. An older warm + /// decoy repo holds identical bytes, so if discovery ran it would record the decoy + /// and set the incomplete marker; neither happens. + #[sqlx::test] + async fn sweep_discovery_is_not_used_for_a_provenanced_row(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["provsrc", "decoysrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("provsrc.git"); + + let mut decoy = seed_repo(&owner_did, "decoysrc"); + decoy.created_at = Utc::now() - chrono::Duration::days(2); + let repo = seed_repo(&owner_did, "provsrc"); + state.db.create_repo(&decoy).await.expect("seed decoy"); + state.db.create_repo(&repo).await.expect("seed repo"); + + let (raw_cid, _provider) = + seed_legacy_pin(&pool, &bare, &fx.public_oid, Some(&repo.id)).await; + + crate::ipfs_pin::reset_legacy_repair_reads(); + let stats = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the sweep terminates"); + + assert_eq!(stats.repaired, 1, "the provenanced row repairs as before"); + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + raw_cid, + "the key is rewritten from the recorded source" + ); + assert_eq!( + crate::ipfs_pin::legacy_repair_reads(), + 1, + "exactly one read, from the recorded source: discovery never probes" + ); + assert_eq!( + state.db.pin_sources_for_oid(&fx.public_oid).await.unwrap(), + vec![repo.id.clone()], + "the decoy is never recorded as a source" + ); + assert!( + !state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "a provenanced row's source set is not marked incomplete" + ); + } + + /// F1 scenario 9 (#173, the degradation posture, by execution): if the additive + /// `record_pin_source` fails after the key rewrite lands, the row is raw-CIDv1 with + /// an empty source set. Nothing in the sweep revisits it (the cost gate skips a raw + /// row free on every later pass), so the source record is best-effort and the + /// resolver's own fallback is the healing path, not a retry. This is that state, + /// driven end to end: an empty source set makes `needs_scan` true and the bounded + /// legacy scan still serves the object. + #[sqlx::test] + async fn sweep_repaired_row_without_source_record_still_served(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["nosrcrec"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("nosrcrec.git"); + let repo = seed_repo(&owner_did, "nosrcrec"); + state.db.create_repo(&repo).await.expect("seed repo"); + + // The post-repair state a failed source record leaves behind: raw key, no + // provenance row, no pin_repo_sources row. + let raw_cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + assert!( + state + .db + .pin_sources_for_oid(&fx.public_oid) + .await + .unwrap() + .is_empty(), + "the row really has no recorded source" + ); + + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&raw_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "an empty source set routes the resolver to its bounded legacy scan, so a \ + repaired row whose source record failed still serves" + ); + assert!(body.contains("public bytes"), "the object's bytes serve"); + } + + /// U2 (#173, F1 proved by execution): a genuinely PRE-PROVENANCE `pinned_cids` row, + /// one written while `pinned_cids.repo_id` and `pin_repo_sources` do not exist, is + /// repaired by the sweep and then served end to end by `GET /ipfs/{cid}`. + /// + /// The fixture un-applies v19 and v20 before the insert on purpose. A source-less + /// row written through the modern schema is a shortcut: it shows the sweep copes + /// with an empty source set, not that it copes with the row shape an upgraded node + /// actually carries. With the column absent, a provenance-carrying insert is + /// impossible, so the row cannot be anything but the real upgrade case. + /// + /// F1 was that the sweep SKIPPED exactly this row. An empty `pin_sources_for_oid` + /// left the `for repo_id in sources` body unentered while the cursor had already + /// advanced past it, so the row kept its provider key and stayed unresolvable with + /// nothing left to fix it. The pre-sweep assertion below brackets the repair, so + /// the serve at the end cannot pass vacuously on a row that was already fine. + #[sqlx::test] + async fn sweep_repairs_pre_provenance_upgrade_row_and_serves(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["preprov"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("preprov.git"); + let repo = seed_repo(&owner_did, "preprov"); // public, no rule + state.db.create_repo(&repo).await.expect("seed repo"); + + // Un-apply the provenance schema: the node is back at the shape it had before + // v19 and v20, where a pin could not carry provenance at all. + sqlx::query("DROP TABLE IF EXISTS pin_repo_sources") + .execute(&pool) + .await + .unwrap(); + sqlx::query("ALTER TABLE pinned_cids DROP COLUMN IF EXISTS repo_id") + .execute(&pool) + .await + .unwrap(); + sqlx::query("DELETE FROM schema_migrations WHERE version IN (19, 20)") + .execute(&pool) + .await + .unwrap(); + + // The legacy row, INSERTed naming only the columns that exist at this schema. + // `seed_legacy_pin` cannot be reused here: it binds `repo_id`, which is the one + // thing this fixture is proving the row never had. + let (_ty, bytes) = crate::git::store::read_object(&bare, &fx.public_oid) + .expect("read object bytes") + .expect("object exists in the bare repo"); + let raw_cid = gitlawb_core::cid::Cid::from_git_object_bytes(&bytes).to_string(); + let provider_cid = legacy_dagpb_cid(&raw_cid); + assert_ne!( + provider_cid, raw_cid, + "the legacy key differs from the raw resolver key" + ); + sqlx::query("INSERT INTO pinned_cids (sha256_hex, cid, pinned_at) VALUES ($1, $2, $3)") + .bind(&fx.public_oid) + .bind(&provider_cid) + .bind("2020-01-01T00:00:00Z") + .execute(&pool) + .await + .unwrap(); + + // Upgrade forward. The row now reads as the exact F1 shape: no first-pinner, no + // source rows, and no incompleteness signal either, so emptiness is the only + // thing the sweep has to go on. + state + .db + .run_migrations() + .await + .expect("re-apply the provenance migrations"); + let first_pinner: Option = + sqlx::query_scalar("SELECT repo_id FROM pinned_cids WHERE sha256_hex = $1") + .bind(&fx.public_oid) + .fetch_one(&pool) + .await + .unwrap(); + assert!( + first_pinner.is_none(), + "the upgraded row carries no first-pinner: the column did not exist when it \ + was written" + ); + assert!( + state + .db + .pin_sources_for_oid(&fx.public_oid) + .await + .unwrap() + .is_empty(), + "the upgraded row has no recorded source of any kind" + ); + assert!( + !state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "and no incompleteness marker either: an upgrade row is indistinguishable \ + from a healthy one except by being empty" + ); + + // The bracket: while the row is unrepaired the resolver withholds it, so the + // raw key a correct client sends does not serve. + let (st_before, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&raw_cid)) + .await + .unwrap(), + ) + .await; + assert_ne!( + st_before, + StatusCode::OK, + "the raw key does not serve while the row is still keyed on the provider CID" + ); + + let stats = crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ) + .await; + assert_eq!( + stats.repaired, 1, + "the sweep repairs the pre-provenance upgrade row" + ); + + let (stored, stashed) = stored_pin(&pool, &fx.public_oid).await; + assert_eq!( + stored, raw_cid, + "the key is rewritten to the raw-content resolver key" + ); + assert_eq!( + stashed.as_deref(), + Some(provider_cid.as_str()), + "the old provider CID is stashed rather than dropped" + ); + assert_eq!( + state.db.pin_sources_for_oid(&fx.public_oid).await.unwrap(), + vec![repo.id.clone()], + "the repo discovery read the bytes from is recorded as a source" + ); + let claimed: Option = + sqlx::query_scalar("SELECT repo_id FROM pinned_cids WHERE sha256_hex = $1") + .bind(&fx.public_oid) + .fetch_one(&pool) + .await + .unwrap(); + assert!( + claimed.is_none(), + "discovery records the holder ADDITIVELY: reading identical bytes proves \ + the repo holds the object, never that it pinned it first, so the exclusive \ + first-pinner column stays NULL" + ); + assert!( + state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "one discovered holder out of a bounded candidate set never proves the set \ + complete, so the resolver keeps its scan fallback for this row" + ); + let marker: Vec = + sqlx::query_scalar("SELECT repo_id FROM pin_source_failures WHERE sha256_hex = $1") + .bind(&fx.public_oid) + .fetch_all(&pool) + .await + .unwrap(); + assert_eq!( + marker, + vec![String::new()], + "the marker is against the UNKNOWN-repo sentinel, not the repo just \ + recorded: no real record can clear it" + ); + + // End to end: the repaired key serves the object's raw bytes to an anonymous + // caller, which is the whole point of repairing it. + let (st, body) = cid_bytes( + cid_router(&state) + .oneshot(cid_anon(&raw_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "the repaired raw key serves"); + assert_eq!( + body, bytes, + "the served body is the object's raw content, byte for byte" + ); + } + + /// F1 scenario 10 (#173, the unsafe-path arm of the candidate load): a `repos` row + /// whose name cannot be turned into a validated disk path is dropped when the + /// candidate list is built, and the drop is both TERMINAL and NON-FATAL. + /// + /// Terminal: nothing a later pass does makes an unsafe name safe, so the rejection + /// must not mark the row retryable and must not consume a probe against + /// `MAX_LEGACY_DISCOVERY_PROBES`. + /// + /// Non-fatal is the half that matters most. `load_discovery_ctx` builds ONE list for + /// the whole pass, so a rejection that propagated instead of warning would fail the + /// load, `discover_legacy_row` would return Retryable for every source-less row in + /// the pass, and a single unsafe `repos` row anywhere on the node would strand every + /// legacy row behind it on every future run. Here the unsafe row sorts first (the + /// candidate order is oldest-first by `(created_at, id)`), so the pass has to survive + /// it before it can reach the warm holder that actually repairs the row. + #[sqlx::test] + async fn sweep_discovery_drops_unsafe_candidate_and_keeps_going(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["safesrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("safesrc.git"); + + // `validate_repo_name` bails on a '..' sequence, so this name can never resolve + // to a disk path. Asserted against the validator itself rather than assumed, so + // the fixture cannot quietly become a safe name and turn the test vacuous. + let bad_name = "../escape"; + assert!( + crate::git::repo_store::validated_repo_disk_path( + std::path::Path::new("/tmp"), + &owner_did, + bad_name, + ) + .is_err(), + "the fixture name is genuinely refused by the validated resolver" + ); + + let mut unsafe_repo = seed_repo(&owner_did, bad_name); + unsafe_repo.created_at = Utc::now() - chrono::Duration::days(2); + let holder = seed_repo(&owner_did, "safesrc"); + state + .db + .create_repo(&unsafe_repo) + .await + .expect("seed the unsafe repos row"); + state + .db + .create_repo(&holder) + .await + .expect("seed the warm holder"); + + // The pre-provenance shape: NULL repo_id and no pin_repo_sources row. + let (raw_cid, _provider) = seed_legacy_pin(&pool, &bare, &fx.public_oid, None).await; + + crate::ipfs_pin::reset_legacy_repair_reads(); + let stats = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("an unsafe candidate never wedges the pass"); + + assert_eq!( + stats.repaired, 1, + "the rejection is non-fatal: a later safe candidate in the same list still \ + repairs the row" + ); + assert_eq!( + crate::ipfs_pin::legacy_repair_reads(), + 1, + "the unsafe candidate is dropped before any probe, so the only object read \ + is the warm holder's" + ); + assert_eq!( + stats.retryable_skips, 0, + "an unsafe name is not a condition a later pass clears, so the drop is \ + terminal and drives no cursor rewind" + ); + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + raw_cid, + "the key is rewritten from the safe candidate's verified bytes" + ); + assert_eq!( + state.db.pin_sources_for_oid(&fx.public_oid).await.unwrap(), + vec![holder.id.clone()], + "only the safe candidate is recorded as a source" + ); + } + + /// Strip Rust line comments, block comments and double-quoted string literals, + /// leaving code. Used by the source scan below, which must not fire on the prose + /// that DESCRIBES the forbidden call (`ipfs_pin.rs` names `repo_store.acquire` in + /// two comments) and must still fire on the call itself. + fn code_only(src: &str) -> String { + let mut out = String::with_capacity(src.len()); + let b: Vec = src.chars().collect(); + let mut i = 0; + while i < b.len() { + if b[i] == '/' && i + 1 < b.len() && b[i + 1] == '/' { + while i < b.len() && b[i] != '\n' { + i += 1; + } + } else if b[i] == '/' && i + 1 < b.len() && b[i + 1] == '*' { + i += 2; + while i + 1 < b.len() && !(b[i] == '*' && b[i + 1] == '/') { + i += 1; + } + i = (i + 2).min(b.len()); + } else if b[i] == '"' { + i += 1; + while i < b.len() && b[i] != '"' { + if b[i] == '\\' { + i += 1; + } + i += 1; + } + i += 1; + } else { + out.push(b[i]); + i += 1; + } + } + out + } + + /// The call-shape half of the sweep's no-remote-fetch guarantee, and a STRUCTURAL + /// guard, not a behavioral one. Say what that means before trusting it. + /// + /// The sweep must never pull a cold repo back from remote storage: it is + /// opportunistic background maintenance over every pinned row on the node, so a + /// fetch would turn a repair pass into a bulk restore. A behavioral test that made + /// the candidate cold, made it fetchable from a live remote, and counted zero fetch + /// attempts cannot be built here, and the reason is worth stating rather than + /// working around: `sweep_pass` takes a `repos_dir`, a `git_bin` and a `&Db`, and + /// holds no `RepoStore` and no `TigrisClient`. A download counter armed on a store + /// the TEST builds could never move no matter what the sweep did, so a zero from it + /// would be vacuous by construction rather than evidence. + /// + /// So the guarantee is asserted from two sides instead. The effect side lives in + /// `sweep_discovery_cold_candidates_do_not_rewind`, which proves the bytes were + /// still available and the cold candidate's path was still absent after two full + /// runs. This is the call side: the module's production code contains none of the + /// fetch-capable call shapes. + /// + /// What it does NOT cover: a fetch reached indirectly through a helper this module + /// calls (`git::store`, `db`) whose own source is not scanned, and git's own lazy + /// fetch if a bare repo on disk were ever configured as a partial clone with a + /// promisor remote. Neither shape exists today; neither is detected here. + #[test] + fn sweep_module_never_calls_a_remote_fetch() { + const SRC: &str = include_str!("ipfs_pin.rs"); + // Scan the PRODUCTION half only. The module's own test module legitimately + // calls `pool.acquire()`, which shares a needle with the store's fetch entry + // points and would otherwise force the needle set to be weakened. + let marker = "\n#[cfg(test)]\nmod tests {"; + let cut = SRC + .find(marker) + .expect("ipfs_pin.rs still opens its test module the usual way"); + let production = &SRC[..cut]; + let code = code_only(production); + + // Anti-vacuity, three ways: a scan that read nothing, a stripper that ate the + // code, or a stripper that left the comments in would each let this pass while + // proving nothing. + assert!( + code.len() > 10_000, + "the scan kept only {} chars of production code, so a clean result proves \ + nothing", + code.len() + ); + assert!( + code.contains("validated_repo_disk_path"), + "the stripper removed real code: the sweep's own path resolver is gone from \ + what was scanned" + ); + assert!( + production.contains("repo_store.acquire"), + "the module no longer names the forbidden call in prose, so this scan is no \ + longer exercising the comment-vs-code distinction it exists to make" + ); + assert!( + !code.contains("bulk restore"), + "the stripper left comments in, so every needle below would fire on the \ + prose that describes it rather than on a call" + ); + + // Every way this crate reaches remote storage. `repo_store::` alone is not a + // needle: the sweep legitimately calls `repo_store::validated_repo_disk_path`, + // the non-fetching path resolver. + for shape in [ + ".acquire(", + "acquire_fresh", + "acquire_write", + "RepoStore", + "TigrisClient", + ".download(", + "tigris", + ] { + assert!( + !code.contains(shape), + "the sweep's production code reaches remote storage through `{shape}`. \ + A repair pass over every pinned row on the node must never pull a cold \ + repo back: that is a bulk restore, not maintenance" + ); + } + } + + /// Poll until some backend in THIS test's database is blocked on a lock, so a test + /// that means to drive an interleaving cannot silently degrade into two calls that + /// simply ran one after the other. Returns false if nothing ever blocked. + async fn wait_for_lock_wait(pool: &PgPool) -> bool { + for _ in 0..600 { + let waiting: i64 = sqlx::query_scalar( + "SELECT count(*) FROM pg_stat_activity + WHERE datname = current_database() AND wait_event_type = 'Lock'", + ) + .fetch_one(pool) + .await + .unwrap_or(0); + if waiting > 0 { + return true; + } + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + } + false + } + + /// GAP 2: discovery's `record_pin_source` then `mark_pin_sources_incomplete` pair + /// under a REAL concurrent writer on the same row, not a reasoned ordering. + /// + /// The order is load-bearing because a `record_pin_source` that actually inserts + /// CLEARS the marker in its own transaction (`rows_affected > 0`), so marking first + /// would have discovery wipe its own marker. The resolver's `needs_scan` is + /// `sources.is_empty() || at_cap || incomplete`, so a non-empty, below-cap, unmarked + /// set is what tells it to stop scanning. Discovery's knowledge is never complete + /// (it stops at the first hit, and its candidate list is capped), so that + /// combination is exactly the state the row must not end in. + /// + /// The interleaving driven here is the one that threatens the pair: a second writer + /// recording a DIFFERENT source for the same oid lands while discovery is mid-row. + /// A row lock parks the sweep inside `repair_legacy_provider_cid`, which is after + /// the source set was read as empty and before either of discovery's own writes, and + /// `wait_for_lock_wait` proves the sweep really is parked rather than already done. + /// The end state must still be a marked row. + #[sqlx::test] + async fn sweep_discovery_marker_survives_a_concurrent_source_record(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + + let fx = seed_cid_repos(&slug, &short, &["concwarm"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("concwarm.git"); + let holder = seed_repo(&owner_did, "concwarm"); + state.db.create_repo(&holder).await.expect("seed holder"); + // The concurrent writer's repo has no directory on disk, so it is filtered out + // of the candidate list and the only thing it contributes to the row is its own + // `record_pin_source`. + let other = seed_repo(&owner_did, "conccold"); + state.db.create_repo(&other).await.expect("seed other"); + + let (raw_cid, _provider) = seed_legacy_pin(&pool, &bare, &fx.public_oid, None).await; + + let mut blocker = pool.begin().await.expect("open the blocking transaction"); + sqlx::query("SELECT sha256_hex FROM pinned_cids WHERE sha256_hex = $1 FOR UPDATE") + .bind(&fx.public_oid) + .execute(&mut *blocker) + .await + .expect("hold the row lock"); + + let driver = async { + let parked = wait_for_lock_wait(&pool).await; + state + .db + .record_pin_source(&fx.public_oid, &other.id) + .await + .expect("the concurrent record lands"); + blocker.commit().await.expect("release the row lock"); + parked + }; + let mut traversal = Default::default(); + let (stats, parked) = tokio::time::timeout(std::time::Duration::from_secs(60), async { + tokio::join!( + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + &mut traversal, + ), + driver + ) + }) + .await + .expect("the interleaved run terminates"); + + assert!( + parked, + "nothing ever blocked, so the two writers did not actually interleave and \ + this test proved nothing about ordering" + ); + assert_eq!(stats.repaired, 1, "discovery still repairs the row"); + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + raw_cid, + "the key is rewritten to the raw-content CID" + ); + let mut sources = state.db.pin_sources_for_oid(&fx.public_oid).await.unwrap(); + sources.sort(); + let mut expected = vec![holder.id.clone(), other.id.clone()]; + expected.sort(); + assert_eq!( + sources, expected, + "both writers' sources are present: the record is additive, so neither \ + writer erases the other" + ); + assert!( + state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "the marker survives a concurrent record landing inside discovery's window: \ + a non-empty, below-cap, unmarked set would tell the resolver to stop \ + scanning while discovery's knowledge of the set is still incomplete" + ); + assert_eq!( + pinned_repo_id(&pool, &fx.public_oid).await, + None, + "no exclusive first-pinner claim is made under concurrency either" + ); + } + + /// GAP 2, the other interleaving: two sweep passes over the same source-less row at + /// the same time. Whichever order the two passes' `repair_legacy_provider_cid`, + /// `record_pin_source` and `mark_pin_sources_incomplete` calls land in, the end + /// state the resolver reads must be the same one a single pass leaves: the raw key, + /// exactly one recorded source, no exclusive claim, and a marked row. + /// + /// The second pass cannot double-record: `record_pin_source` is + /// `ON CONFLICT DO NOTHING` on `(oid, repo)`, so its insert affects no rows, its + /// marker clear is gated on `rows_affected > 0` and does not run, and its own + /// `mark_pin_sources_incomplete` is idempotent. + #[sqlx::test] + async fn sweep_discovery_two_concurrent_passes_leave_one_marked_source(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + + let fx = seed_cid_repos(&slug, &short, &["twopass"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("twopass.git"); + let holder = seed_repo(&owner_did, "twopass"); + state.db.create_repo(&holder).await.expect("seed holder"); + let (raw_cid, _provider) = seed_legacy_pin(&pool, &bare, &fx.public_oid, None).await; + + let (mut ta, mut tb) = (Default::default(), Default::default()); + let (a, b) = tokio::time::timeout(std::time::Duration::from_secs(60), async { + tokio::join!( + crate::ipfs_pin::sweep_legacy_provider_cids_once( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + &state.db, + &mut ta, + ), + crate::ipfs_pin::sweep_legacy_provider_cids_once( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + &state.db, + &mut tb, + ) + ) + }) + .await + .expect("both passes terminate"); + let a = a.expect("the first pass succeeds"); + let b = b.expect("the second pass succeeds"); + assert!( + a.repaired + b.repaired >= 1, + "at least one of the two concurrent passes repairs the row" + ); + + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + raw_cid, + "the key is the raw-content CID whichever pass got there first" + ); + assert_eq!( + state.db.pin_sources_for_oid(&fx.public_oid).await.unwrap(), + vec![holder.id.clone()], + "the holder is recorded exactly once: the second pass's insert conflicts and \ + affects no rows" + ); + assert!( + state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "two concurrent passes still leave the row marked, so the resolver keeps its \ + fallback scan" + ); + assert_eq!( + pinned_repo_id(&pool, &fx.public_oid).await, + None, + "neither pass makes an exclusive first-pinner claim" + ); + } + + /// GAP 2, CLOSED (#173 round 12), kept as the regression that proves it stays closed. + /// + /// This documented a residual: `pin_sources_incomplete` was one boolean per OBJECT, so + /// any later inserting `record_pin_source` cleared it, including one from a repo with + /// nothing to do with discovery, leaving a non-empty source set and no marker, which + /// is the combination that stops the resolver's fallback scan. The marker is now per + /// `(object, repo)` and a record clears only its own pair, so a later writer cannot + /// clear what discovery set. + /// + /// Discovery marks against the unknown-repo sentinel rather than a real repo id, + /// because what it knows is that its bounded warm-only probe may have missed a holder, + /// not that any particular repo failed to record. No real record equals that sentinel, + /// which is what makes the marker survive here. + #[sqlx::test] + async fn sweep_discovery_marker_survives_a_later_record_from_another_repo(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["latewarm"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("latewarm.git"); + let holder = seed_repo(&owner_did, "latewarm"); + state.db.create_repo(&holder).await.expect("seed holder"); + let other = seed_repo(&owner_did, "latecold"); + state.db.create_repo(&other).await.expect("seed other"); + seed_legacy_pin(&pool, &bare, &fx.public_oid, None).await; + + let stats = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the sweep terminates"); + assert_eq!(stats.repaired, 1, "discovery repairs the row"); + assert!( + state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "discovery leaves the row marked" + ); + + // A genuine later pusher of the same object from a different repo. + state + .db + .record_pin_source(&fx.public_oid, &other.id) + .await + .expect("the later record lands"); + + assert!( + state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "a later record from another repo clears only its own pair, so discovery's \ + marker survives and the resolver keeps its fallback" + ); + let mut sources = state.db.pin_sources_for_oid(&fx.public_oid).await.unwrap(); + sources.sort(); + let mut expected = vec![holder.id.clone(), other.id.clone()]; + expected.sort(); + assert_eq!( + sources, expected, + "the bound on that residual: every source left in the set is a repo that \ + really holds the object, so the row stays servable through them" + ); + } + + /// U4 scenario 6 (#173): `list_pinned_cids` never advertises a key the `/ipfs` + /// resolver would withhold. The resolver recomputes the raw CIDv1 from the object + /// bytes and 404s any row keyed on a legacy PROVIDER CID, so advertising that key + /// hands clients a CID this node deliberately refuses. Both states of ONE row are + /// asserted (omitted while legacy, present once repaired) so the test cannot pass + /// by accident. RED before the `is_raw_cidv1` filter lands: the legacy row is + /// advertised. + #[sqlx::test] + async fn list_pinned_cids_omits_unrepaired_legacy_row(pool: PgPool) { + let state = test_state(pool).await; + + let raw_cid = + gitlawb_core::cid::Cid::from_git_object_bytes(b"u4 advertise bytes").to_string(); + let provider_cid = legacy_dagpb_cid(&raw_cid); + let oid = "c".repeat(64); + state + .db + .record_pinned_cid(&oid, &provider_cid, None) + .await + .unwrap(); + + let listed = state.db.list_pinned_cids().await.unwrap(); + assert!( + !listed.iter().any(|r| r.sha256_hex == oid), + "an unrepaired legacy provider-CID row is not advertised" + ); + + // Same row, repaired: it comes back, keyed on the raw CID the resolver serves. + state + .db + .repair_legacy_provider_cid(&oid, &raw_cid, &provider_cid) + .await + .unwrap(); + let listed = state.db.list_pinned_cids().await.unwrap(); + let rec = listed + .iter() + .find(|r| r.sha256_hex == oid) + .expect("the repaired row is advertised again"); + assert_eq!( + rec.cid, raw_cid, + "the advertised key is the raw-content resolver key" + ); + } + + /// #173 (provenance-path throttle): a walk-requiring provenanced candidate whose + /// per-IP walk quota is spent returns 429 (the provenance arm's Throttled outcome, + /// then the fall-through). quota=1, keyed on XFF. The first reader request runs the + /// walk and spends the token; the second from the same IP is throttled → 429. + #[sqlx::test] + async fn ipfs_cid_provenance_walk_throttle_returns_429(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let reader = Keypair::generate(); + let reader_did = reader.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + let fx = seed_cid_repos(&slug, &short, &["provthrottle"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("provthrottle.git"); + let repo = seed_repo(&owner_did, "provthrottle"); + state.db.create_repo(&repo).await.expect("seed repo"); + state + .db + .set_visibility_rule( + &repo.id, + "/secret/**", + VisibilityMode::B, + std::slice::from_ref(&reader_did), + &owner_did, + ) + .await + .expect("path rule"); + let cid = pin_cid_for_repo(&bare, &fx.secret_oid, &state.db, &repo.id).await; + + // 1st reader request runs the walk (reader is allowed) and spends the token. + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_signed_xff(&reader, &cid, "1.2.3.4")) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "1st provenance walk from the IP serves"); + + // 2nd request from the same IP: the walk is throttled → 429 (provenance path). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_signed_xff(&reader, &cid, "1.2.3.4")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::TOO_MANY_REQUESTS, + "a throttled provenance walk returns 429" + ); + } + + /// #173 (multi-oid dispatch, mixed provenance + legacy): one CID mapping to a + /// provenanced-then-denied oid AND a legacy (NULL-provenance) oid must still resolve + /// to the legacy-servable copy — the provenance arm's skip does not abort the loop. + #[sqlx::test] + async fn ipfs_cid_mixed_provenance_and_legacy_serves_legacy(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["mixpriv", "mixpub"]); + + // Private repo holds secret_oid, pinned with provenance = itself (denies anon). + let mut priv_repo = seed_repo(&owner_did, "mixpriv"); + priv_repo.is_public = false; + state + .db + .create_repo(&priv_repo) + .await + .expect("seed private"); + // Public repo holds public_oid, legacy pin (NULL provenance -> scan serves it). + let pub_repo = seed_repo(&owner_did, "mixpub"); + state.db.create_repo(&pub_repo).await.expect("seed public"); + + // One REAL CID (the non-unique cid index) maps to BOTH oids: the public oid as a + // legacy (NULL) pin, and the secret oid provenanced to the private repo. + let pub_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("mixpub.git"); + let shared_cid = pin_cid_for(&pub_bare, &fx.public_oid, &state.db).await; + state + .db + .record_pinned_cid(&fx.secret_oid, &shared_cid, Some(&priv_repo.id)) + .await + .unwrap(); + + // Anon: secret_oid (provenance -> private -> denied), public_oid (legacy -> scan + // -> public -> served). Resolves to the public copy regardless of oid order. + let resp = cid_router(&state) + .oneshot(cid_anon(&shared_cid)) + .await + .unwrap(); + let served = resp + .headers() + .get("x-git-hash") + .and_then(|v| v.to_str().ok()) + .map(str::to_string); + let (st, body) = cid_parts(resp).await; + assert_eq!( + st, + StatusCode::OK, + "a CID mixing a provenanced-denied oid and a legacy-servable oid resolves" + ); + assert_eq!( + served.as_deref(), + Some(fx.public_oid.as_str()), + "the served object is the legacy public oid" + ); + assert!( + body.contains("public bytes"), + "the public content is served" + ); + } + + // ---- #173 round 3: legacy (NULL-provenance) scan bound + 503-on-truncation ---- + // The provenance path targets one repo and is already bounded. These cover the + // legacy scan fallback, where an anonymous request could otherwise fan out to + // O(repos) `acquire` + `cat-file` probes (F1) and a walk-cap truncation could + // false-404 an object that may be readable (F2). The bound is a per-request probe + // BUDGET, not a per-IP brake: a walk-free public fetch stays un-rate-limited + // (ipfs_walk_rate_limited_per_source), while the expensive walk keeps its IP brake. + + /// #173 round 12 (jatmn): a failure of the SIZE stage must reach the client as the + /// retryable 503, never as a definitive 404. `object_size_bounded` mapped every + /// non-timeout failure to `Ok(None)`, which `gate_and_serve` read as a verified + /// absence and did not taint the search for, so a corrupt object or a failed spawn + /// 404'd an authorized caller on an object the type probe had just reported present. + /// + /// The failure is induced between the two stages with a test seam, because the size + /// read uses the real `git` rather than `state.git_bin` and no shim can be injected + /// there. The BEFORE request is what makes the AFTER assertion mean anything: it + /// proves this fixture serves 200 when the size read succeeds, so the 503 is caused + /// by the broken size probe and nothing else. + #[sqlx::test] + async fn ipfs_cid_size_probe_failure_is_retryable_not_a_404(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["sizefault"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("sizefault.git"); + let repo = seed_repo(&owner_did, "sizefault"); // public, no rule + state.db.create_repo(&repo).await.expect("seed repo"); + let cid = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, &repo.id).await; + + let (before, body) = + cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + before, + StatusCode::OK, + "the fixture serves this object when the size read succeeds" + ); + assert!(body.contains("public bytes"), "and serves the real content"); + + // The object vanishes between the type probe and the size probe. Armed on THIS + // repo's path: fixture oids are shared across tests, so an oid-only key would + // reach into another test's repo. + crate::api::ipfs::break_size_probe_for(&bare, &fx.public_oid); + let (after, _) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + after, + StatusCode::SERVICE_UNAVAILABLE, + "a size-stage fault taints the search and tails to a retryable 503; a 404 here \ + would tell an authorized caller the object does not exist" + ); + } + + /// T1 (F1): the probe budget gates BEFORE `acquire`/`cat-file`, so it genuinely + /// bounds the fan-out — a repo past the budget is never probed, even one that + /// WOULD serve. With the budget at 0, a PUBLIC legacy copy that would otherwise + /// serve 200 is not probed at all → 503 truncated (absence unproven). RED before + /// the budget check (the repo is probed and serves 200). + #[sqlx::test] + async fn ipfs_cid_legacy_probe_budget_gates_before_serving(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_max_legacy_probes = 0; // probe nothing → any legacy candidate truncates + + let fx = seed_cid_repos(&slug, &short, &["pubprobe"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("pubprobe.git"); + let repo = seed_repo(&owner_did, "pubprobe"); // public, no path rule → would serve + state.db.create_repo(&repo).await.expect("seed repo"); + // Legacy pin (NULL provenance) → resolver takes the scan fallback. + let cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + + let (st, _) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::SERVICE_UNAVAILABLE, + "the probe budget gates before the probe: a servable copy past the budget is not reached → 503" + ); + } + + /// T7 (F1/F3 pre-limit): EVERY legacy probe is braked on the source IP from the + /// FIRST one, so a hostile caller cannot repeatedly force the whole-node `acquire` + /// fan-out across requests (each cold `acquire` is a Tigris round-trip, INV-10). + /// Since #173-F3 (jatmn) there is no free budget: a single-repo legacy scan is + /// itself charged. quota=1 keyed on XFF, one PUBLIC legacy copy that serves + /// walk-free (never touches the walk brake), so the second same-IP request can only + /// be shed by the probe brake: req1 serves and spends the token, req2 → 429. RED + /// before the probe brake (req2 serves 200). The cross-request bound this proves is + /// exactly the amplification F3 closes. + #[sqlx::test] + async fn ipfs_cid_legacy_fanout_braked_on_ip_past_free_budget(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + // Budget = one full scan of the single seeded repo: 1 page + 1 probe. The page + // is charged because the legacy scan's DB-facing pages draw on this same bucket + // (#173 round 13, F2). Without that charge a denial-only inventory could be + // re-paged for free by re-requesting. Production never sees a bucket this small: + // `AppState::ipfs_work_budget` floors it at probes + pages, so only a fixture + // that sets the limiter by hand has to do the arithmetic itself. + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(2, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + let fx = seed_cid_repos(&slug, &short, &["fanout"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("fanout.git"); + let repo = seed_repo(&owner_did, "fanout"); // public, no path rule → walk-free serve + state.db.create_repo(&repo).await.expect("seed repo"); + let cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; // legacy pin + + let (st1, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon_xff(&cid, "1.2.3.4")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st1, + StatusCode::OK, + "1st legacy fan-out probe from the IP serves" + ); + + let (st2, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon_xff(&cid, "1.2.3.4")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st2, + StatusCode::TOO_MANY_REQUESTS, + "with no free budget, a repeat fan-out from the same IP is braked at the first probe" + ); + } + + /// F3 (jatmn, across-request amplification): the pre-fix free-probe budget was + /// PER REQUEST, so a caller could repeat a known NULL-provenance CID and force a + /// fresh batch of `acquire` + `cat-file` probes every request with zero limiter + /// contact, unbounded anonymous amplification against Tigris. Charging every + /// legacy probe from the first one makes those probes accumulate against the + /// per-IP `ipfs_work_rate_limiter` ACROSS requests. Four repos, none holding the CID, + /// so a full scan probes all four; the per-IP budget is sized to exactly ONE such + /// scan (4 tokens). req1 (a genuine absence) fully scans and 404s, spending the + /// budget; req2 from the SAME IP is shed at the first probe → 429 (it never + /// re-runs the four `acquire` probes). RED with the old free carve-out restored: + /// req2 re-scans un-braked and 404s again (the amplification stays open). This is + /// the load-bearing across-request bound F3 asks for. + #[sqlx::test] + async fn ipfs_cid_legacy_fanout_bounded_across_requests(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + // Budget = one full scan of the four seeded repos: 1 page + 4 probes. A repeat + // scan from the same IP then finds it spent. Keyed on XFF so `oneshot` can + // choose the source IP. The page term is there because the scan's DB-facing + // pages draw on this same bucket (#173 round 13, F2), so re-requesting cannot + // buy the inventory again for free; all four repos fit in one 128-row page, so + // one page covers the whole scan. Production is floored at probes + pages by + // `AppState::ipfs_work_budget`; only a hand-set limiter does this arithmetic. + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(5, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + let names = ["a0", "a1", "a2", "a3"]; + let _fx = seed_cid_repos(&slug, &short, &names); + for n in names { + let repo = seed_repo(&owner_did, n); + state.db.create_repo(&repo).await.expect("seed repo"); + } + // A legacy pin whose oid is absent from every repo → each probed repo misses, + // so req1 scans all four (spending the four-token budget) and 404s cleanly. + let bogus_oid = "0".repeat(64); + let cid = + gitlawb_core::cid::Cid::from_git_object_bytes(b"absent-across-requests").to_string(); + state + .db + .record_pinned_cid(&bogus_oid, &cid, None) + .await + .expect("record legacy pin"); + + let (st1, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon_xff(&cid, "9.9.9.9")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st1, + StatusCode::NOT_FOUND, + "1st scan completes under budget: a genuine absence is a definitive 404" + ); + + let (st2, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon_xff(&cid, "9.9.9.9")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st2, + StatusCode::TOO_MANY_REQUESTS, + "2nd same-IP scan is shed at the first probe (429), not re-run un-braked: the across-request amplification is closed" + ); + } + + /// #173 (jatmn round 8, F3 — INV-10 cost guard): an already-throttled source's + /// legacy NULL-provenance request must be shed by the non-consuming admission peek + /// BEFORE the O(repos) `scan_ctx` preload runs — not after, where the per-probe + /// brake sits. The preload-query counter proves it both ways: 0 for the throttled + /// replay, 1 for an unthrottled source. RED if the peek is removed (the preload runs + /// while throttled → count 1). The two existing `_fanout_` tests confirm the per- + /// probe consuming charge is untouched (no double-charge, no under-charge). + #[sqlx::test] + async fn ipfs_cid_f3_throttled_source_skips_preload(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + // Budget 1, keyed on XFF so `oneshot` can choose the source IP. + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + let _fx = seed_cid_repos(&slug, &short, &["r0"]); + state + .db + .create_repo(&seed_repo(&owner_did, "r0")) + .await + .expect("seed repo"); + // A legacy pin absent from every repo → the scan probes and 404s (spending the + // one token on the first probe). + let bogus_oid = "0".repeat(64); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(b"f3-absent").to_string(); + state + .db + .record_pinned_cid(&bogus_oid, &cid, None) + .await + .expect("legacy pin"); + + // Req1 from 9.9.9.9 spends the one token (and runs the preload once). + let _ = cid_router(&state) + .oneshot(cid_anon_xff(&cid, "9.9.9.9")) + .await + .unwrap(); + + // Measure the throttled replay: the peek must shed it before the preload runs. + crate::api::ipfs::reset_preload_queries(); + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon_xff(&cid, "9.9.9.9")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::TOO_MANY_REQUESTS, + "an already-throttled legacy replay is 429" + ); + assert_eq!( + crate::api::ipfs::preload_queries(), + 0, + "a throttled source must NOT run the O(repos) preload (F3): shed before scan_ctx" + ); + + // Control: an unthrottled source (a different IP) still runs the preload once — + // the peek must not over-block. + crate::api::ipfs::reset_preload_queries(); + let _ = cid_router(&state) + .oneshot(cid_anon_xff(&cid, "8.8.8.8")) + .await + .unwrap(); + assert_eq!( + crate::api::ipfs::preload_queries(), + 1, + "an unthrottled source runs the preload once (the peek must not over-block)" + ); + } + + /// T2 (F1): the legacy scan is bounded per request. With the probe ceiling shrunk + /// to 2 and 3 candidate repos none of which hold the object, the 3rd repo is never + /// probed and the search is reported truncated → 503, not an unbounded fan-out. + /// RED before the probe cap (all 3 probe, none serve, definitive 404). + #[sqlx::test] + async fn ipfs_cid_legacy_scan_probe_cap_truncates_to_503(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_max_legacy_probes = 2; + + let _fx = seed_cid_repos(&slug, &short, &["r0", "r1", "r2"]); + for n in ["r0", "r1", "r2"] { + let repo = seed_repo(&owner_did, n); + state.db.create_repo(&repo).await.expect("seed repo"); + } + // A legacy pin whose oid is absent from every repo: each probed repo misses, + // so the cap (not a hit) decides the outcome. + let bogus_oid = "0".repeat(64); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(b"absent-marker-t2").to_string(); + state + .db + .record_pinned_cid(&bogus_oid, &cid, None) + .await + .expect("record legacy pin"); + + let (st, _) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::SERVICE_UNAVAILABLE, + "a scan truncated by the probe cap is a retryable 503, not a definitive 404" + ); + } + + /// T2b (R5, KTD5): the `GITLAWB_IPFS_MAX_REPOS_WALKED` knob drives the legacy-probe + /// budget end to end. With the knob at 1 (fed through the same production helper the + /// state seeding uses) and two candidate repos that miss, the first repo spends the + /// single probe and the second is skipped at the cap → truncated → 503. If the knob + /// budget were not honoured (unbounded), both would probe, both miss, and the request + /// would be a definitive 404. Proves the wired knob=1 → exactly one probe path. + #[sqlx::test] + async fn ipfs_cid_repos_walked_knob_caps_legacy_probes(pool: PgPool) { + use clap::Parser; + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + // Seed the legacy-probe budget the way production does: from the operator knob. + // The legacy-probe budget knob, renamed in the merge: #174 already owned + // `--ipfs-max-repos-walked` for its expensive-walk cap, so #173's identically + // named knob became `--ipfs-max-legacy-probes`. + let cfg = + crate::config::Config::parse_from(["gitlawb-node", "--ipfs-max-legacy-probes", "1"]); + state.ipfs_max_legacy_probes = AppState::ipfs_legacy_probe_budget(&cfg); + assert_eq!(state.ipfs_max_legacy_probes, 1, "knob=1 → one-probe budget"); + // The knob must not touch the history-walk ceiling (must stay MAX_PIN_SOURCES + 1). + assert_eq!( + state.ipfs_max_history_walks, + crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, + "the repos-walked knob leaves the history-walk ceiling untouched" + ); + + let _fx = seed_cid_repos(&slug, &short, &["k0", "k1"]); + for n in ["k0", "k1"] { + let repo = seed_repo(&owner_did, n); + state.db.create_repo(&repo).await.expect("seed repo"); + } + // A legacy pin whose oid is absent from every repo: the cap, not a hit, decides. + let bogus_oid = "0".repeat(64); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(b"absent-marker-knob").to_string(); + state + .db + .record_pinned_cid(&bogus_oid, &cid, None) + .await + .expect("record legacy pin"); + + let (st, _) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::SERVICE_UNAVAILABLE, + "knob=1 caps the scan at one probe → incomplete search → retryable 503" + ); + } + + /// T3 (F2): a walk-cap truncation must not false-404. Walk ceiling shrunk to 1; + /// two public repos each carry a path-scoped rule over the object and deny anon. + /// The 1st spends the single walk (deny), the 2nd is skipped at the cap — the + /// resolver did NOT prove the object unreadable everywhere, so 503, not 404. + /// RED before the walk-cap `truncated` flag (returns the opaque 404). + #[sqlx::test] + async fn ipfs_cid_legacy_walk_cap_truncates_to_503(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let reader = Keypair::generate(); + let reader_did = reader.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_max_history_walks = 1; + + let fx = seed_cid_repos(&slug, &short, &["wa", "wb"]); + for n in ["wa", "wb"] { + let repo = seed_repo(&owner_did, n); + state.db.create_repo(&repo).await.expect("seed repo"); + state + .db + .set_visibility_rule( + &repo.id, + "/secret/**", + VisibilityMode::B, + std::slice::from_ref(&reader_did), + &owner_did, + ) + .await + .expect("path rule"); + } + // Legacy pin of the path-scoped secret blob (present in both repos, denies anon). + let bare_wa = std::path::PathBuf::from("/tmp").join(&slug).join("wa.git"); + let cid = pin_cid_for(&bare_wa, &fx.secret_oid, &state.db).await; + + let (st, _) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::SERVICE_UNAVAILABLE, + "the walk cap truncated the scan, so absence is unproven → 503, not a false 404" + ); + } + + /// T4 (must-not over-fire): a legacy CID genuinely absent from every repo on a + /// node UNDER the probe cap still returns the definitive 404 — the 503 fires only + /// on real truncation, never as a blanket replacement for not-found. + #[sqlx::test] + async fn ipfs_cid_legacy_true_absence_stays_404(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_max_legacy_probes = 8; // well above the single repo → no truncation + + let _fx = seed_cid_repos(&slug, &short, &["only"]); + let repo = seed_repo(&owner_did, "only"); + state.db.create_repo(&repo).await.expect("seed repo"); + let bogus_oid = "0".repeat(64); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(b"absent-marker-t4").to_string(); + state + .db + .record_pinned_cid(&bogus_oid, &cid, None) + .await + .expect("record legacy pin"); + + let (st, _) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "a fully-scanned genuine absence is a definitive 404, not a 503" + ); + } + + /// T5 (provenance path untouched): the probe cap governs ONLY the legacy scan. + /// With the cap set to 0 (which would truncate any legacy probe immediately) a + /// PROVENANCED pin still resolves to its one repo and serves 200 — proving the + /// `legacy_scan=false` guard exempts the provenance path. RED if the guard were + /// dropped (provenance would truncate to 503). + #[sqlx::test] + async fn ipfs_cid_provenance_serves_despite_zero_probe_cap(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_max_legacy_probes = 0; // would truncate every LEGACY probe + + let fx = seed_cid_repos(&slug, &short, &["provonly"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("provonly.git"); + let repo = seed_repo(&owner_did, "provonly"); // public, no path rule + state.db.create_repo(&repo).await.expect("seed repo"); + let cid = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, &repo.id).await; + + let (st, _) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::OK, + "the provenance path ignores the legacy probe cap and serves" + ); + } + + fn cid_router(state: &AppState) -> Router { + Router::new() + .route( + "/ipfs/{cid}", + axum::routing::get(crate::api::ipfs::get_by_cid), + ) + .layer(axum::middleware::from_fn(crate::auth::optional_signature)) + .with_state(state.clone()) + } + async fn cid_parts(resp: axum::response::Response) -> (StatusCode, String) { + let st = resp.status(); + let b = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .unwrap(); + (st, String::from_utf8_lossy(&b).to_string()) + } + /// Raw body bytes (NOT lossy-decoded). A git tree body stores each child oid + /// as 32 RAW bytes that `from_utf8_lossy` mangles to U+FFFD, so a hex + /// `contains` check on `cid_parts`'s String is vacuous. #135 deny tests must + /// witness the leak on these raw bytes. + async fn cid_bytes(resp: axum::response::Response) -> (StatusCode, Vec) { + let st = resp.status(); + let b = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .unwrap(); + (st, b.to_vec()) + } + /// True if `needle` appears as a contiguous byte subsequence of `haystack`. + fn bytes_contain(haystack: &[u8], needle: &[u8]) -> bool { + !needle.is_empty() && haystack.windows(needle.len()).any(|w| w == needle) + } + fn cid_anon(cid: &str) -> Request { + Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap() + } + /// Anonymous CID request carrying `x-forwarded-for: ` — an anon caller with a + /// resolvable source IP, so the per-IP walk brake keys on it (the walk still + /// denies anon at a path rule). + fn cid_anon_xff(cid: &str, xff_ip: &str) -> Request { + Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .header("x-forwarded-for", xff_ip) + .body(Body::empty()) + .unwrap() + } + fn cid_signed(kp: &gitlawb_core::identity::Keypair, cid: &str) -> Request { + let path = format!("/ipfs/{cid}"); + let s = gitlawb_core::http_sig::sign_request(kp, "GET", &path, b""); + Request::builder() + .method(Method::GET) + .uri(&path) + .header("content-digest", s.content_digest) + .header("signature-input", s.signature_input) + .header("signature", s.signature) + .body(Body::empty()) + .unwrap() + } + /// Signed CID request carrying `x-forwarded-for: `. Used by the walk + /// rate-limit test to key the per-IP limiter off a chosen source under + /// `TrustedProxy::XForwardedFor` (the request goes through `oneshot`, which + /// leaves no socket peer, so the header is the only key source). + fn cid_signed_xff( + kp: &gitlawb_core::identity::Keypair, + cid: &str, + xff_ip: &str, + ) -> Request { + let path = format!("/ipfs/{cid}"); + let s = gitlawb_core::http_sig::sign_request(kp, "GET", &path, b""); + Request::builder() + .method(Method::GET) + .uri(&path) + .header("content-digest", s.content_digest) + .header("signature-input", s.signature_input) + .header("signature", s.signature) + .header("x-forwarded-for", xff_ip) + .body(Body::empty()) + .unwrap() + } + + /// #110: `GET /ipfs/{cid}` must gate a withheld blob by per-caller visibility. + /// RED before U2 (the current handler serves the secret to anon). + #[sqlx::test] + async fn ipfs_cid_gate_withholds_blob_from_unauthorized(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let reader = Keypair::generate(); + let reader_did = reader.did().to_string(); + let stranger = Keypair::generate(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["withhold"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("withhold.git"); + // Request CIDs are the production pin CIDs (content-hash), recorded in + // pinned_cids so get_by_cid resolves each back to its oid (#173). + let secret_cid = pin_cid_for(&bare, &fx.secret_oid, &state.db).await; + let tree_cid = pin_cid_for(&bare, &fx.secret_tree_oid, &state.db).await; + let public_cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + let root_tree_cid = pin_cid_for(&bare, &fx.root_tree_oid, &state.db).await; + let public_tree_cid = pin_cid_for(&bare, &fx.public_tree_oid, &state.db).await; + let commit_cid = pin_cid_for(&bare, &fx.commit_oid, &state.db).await; + let tag_cid = pin_cid_for(&bare, &fx.tag_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "withhold")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "withhold") + .await + .unwrap() + .unwrap(); + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + VisibilityMode::B, + std::slice::from_ref(&reader_did), + &owner_did, + ) + .await + .expect("deny rule"); + + // anon → withheld blob: must 404, must not leak content. (RED on current handler.) + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&secret_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "anon must not read the withheld blob" + ); + assert!( + !body.contains("TOP SECRET"), + "404 body must not leak the secret" + ); + + // signed non-reader → 404. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_signed(&stranger, &secret_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "non-reader must not read the withheld blob" + ); + assert!(!body.contains("TOP SECRET")); + + // owner (signed) → 200 + secret bytes. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_signed(&owner, &secret_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "owner reads the withheld blob"); + assert!(body.contains("TOP SECRET"), "owner gets the content"); + + // listed reader (signed) → 200. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_signed(&reader, &secret_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "listed reader reads the blob"); + assert!(body.contains("TOP SECRET")); + + // #135: anon tree CID under withheld /secret → 404. The 404 body is an opaque + // error string (never the object), so status is the load-bearing deny check; + // the real leak witness is the CONTRAST with the reader below, who DOES get a + // 200 carrying the child structure that anon is denied. + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&tree_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "withheld subtree tree must not be served to anon (#135)" + ); + + // Over-denial guard + positive leak witness: the listed reader (signed) DOES + // read the withheld subtree's tree, and its body carries the exact child + // structure anon was denied — the child filename plus the child oid as the 32 + // RAW bytes a git tree stores (witnessed on raw bytes, since cid_parts's lossy + // decode would mangle them). This proves b.txt / secret_raw are the real leak + // markers and that the anon 404 above actually withheld them. + let secret_raw = hex::decode(&fx.secret_oid).expect("hex oid"); + let (st, body) = cid_bytes( + cid_router(&state) + .oneshot(cid_signed(&reader, &tree_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "listed reader reads the withheld subtree tree" + ); + assert!( + bytes_contain(&body, b"b.txt") && bytes_contain(&body, &secret_raw), + "reader's tree body carries the child filename and raw child oid" + ); + + // Root tree (path "/") stays served to anon who passes the "/" gate. + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&root_tree_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "root tree stays served (must-serve)"); + + // /public subtree tree stays served to anon (allowed path). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&public_tree_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "public subtree tree stays served"); + + // Commit and annotated tag objects stay served (unchanged by #135). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&commit_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "commit object stays served"); + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&tag_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "tag object stays served"); + + // R3: public blob anon → 200 (non-withheld content not affected). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&public_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "public blob stays served"); + + // R5: a genuine unknown CID also 404, uniform with the withheld 404. A + // well-formed pin-style CID that was never recorded in pinned_cids, so the + // oid_for_cid resolve misses (the production not-found path). + let absent_cid = + gitlawb_core::cid::Cid::from_git_object_bytes(b"never pinned to this node").to_string(); + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&absent_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "absent CID 404 (uniform with withheld)" + ); + + // malformed CID → 400 (unchanged). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon("not-a-cid")) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::BAD_REQUEST, "malformed CID still 400"); + } + + /// R4: the same object withheld in one repo but public in another is still + /// served from the public copy; the withholding repo is iterated first. + #[sqlx::test] + async fn ipfs_cid_served_from_public_copy_when_withheld_elsewhere(pool: PgPool) { + use crate::db::VisibilityMode; + use chrono::Utc; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["withhold", "pubcopy"]); + // Same content in both clones -> same oid/CID; read from either. + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("withhold.git"); + let secret_cid = pin_cid_for(&bare, &fx.secret_oid, &state.db).await; + + // Withholding repo, iterated FIRST: the paged scan orders on the immutable + // `(created_at, id)` ASC, so the OLDER created_at leads (#173, jatmn). + let mut withhold = seed_repo(&owner_did, "withhold"); + withhold.created_at = Utc::now() - chrono::Duration::seconds(60); + state + .db + .create_repo(&withhold) + .await + .expect("withhold repo"); + state + .db + .set_visibility_rule( + &withhold.id, + "/secret/**", + VisibilityMode::B, + &[], + &owner_did, + ) + .await + .expect("deny rule"); + + // Public copy, no rules, iterated AFTER (newer created_at). + let mut pubcopy = seed_repo(&owner_did, "pubcopy"); + pubcopy.created_at = Utc::now(); + state.db.create_repo(&pubcopy).await.expect("pubcopy repo"); + + // anon: denied at the withholding repo (continue), served from the public copy. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&secret_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "served from the public copy despite the other deny" + ); + assert!( + body.contains("TOP SECRET"), + "the public copy serves the content" + ); + } + + /// Repo-level "/" gate (KTD2a, first continue branch): a fully private repo + /// (is_public=false, no rules) denies anon before any per-blob check; the + /// owner still reads. The path-scoped tests pass the "/" gate and deny at the + /// per-blob stage, so this exercises the coarser repo-level deny separately. + #[sqlx::test] + async fn ipfs_cid_private_repo_denies_anon_at_repo_gate(pool: PgPool) { + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["priv"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("priv.git"); + let blob_cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + + let mut rec = seed_repo(&owner_did, "priv"); + rec.is_public = false; + state.db.create_repo(&rec).await.expect("private repo"); + + // anon → repo-level deny → 404, no content leaked. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&blob_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "anon denied at a private repo's / gate" + ); + assert!(!body.contains("public bytes"), "404 must not leak content"); + + // owner-signed → 200. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_signed(&owner, &blob_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "owner reads their private repo's object" + ); + assert!(body.contains("public bytes"), "owner gets the content"); + } + + /// Fail-closed walk-error arm: if `withheld_blob_oids` errors (here, a ref + /// pointing at a non-tree-ish blob, which `git ls-tree -r` cannot traverse — + /// the same induction as `visibility_pack::fails_closed_when_a_ref_cannot_be_traversed`), + /// the handler skips the whole repo rather than serving. Asserts no leak of the + /// withheld blob AND that even the *public* blob in that repo is withheld — the + /// latter distinguishes fail-closed-skip from normal per-blob withholding and + /// would serve 200 if the error arm wrongly proceeded. The skip carries no + /// VERDICT (F2), so the response is the retryable truncation 503, not a 404 + /// claiming the object is absent — never-serve-unproven and never-404-unproven + /// hold together. + #[sqlx::test] + async fn ipfs_cid_walk_error_fails_closed(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["withhold"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("withhold.git"); + // Recorded pins so get_by_cid resolves each CID to its oid and reaches the + // walk; the 404s below are then the fail-closed skip, not a table miss. + let secret_cid = pin_cid_for(&bare, &fx.secret_oid, &state.db).await; + let public_cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + + // Force the withheld walk to fail closed: a ref pointing at a blob (not + // tree-ish) makes `git ls-tree -r` error, which `withheld_blob_oids` + // propagates as Err → the handler's `Ok(Err)` arm skips the repo. + std::fs::write( + bare.join("refs/heads/blobref"), + format!("{}\n", fx.secret_oid), + ) + .unwrap(); + + state + .db + .create_repo(&seed_repo(&owner_did, "withhold")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "withhold") + .await + .unwrap() + .unwrap(); + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("deny rule"); + + // Withheld secret CID under a walk error → the repo is skipped without a + // verdict, so the scan is truncated (503), and nothing leaks. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&secret_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::SERVICE_UNAVAILABLE, + "walk error must not serve the withheld blob — the unproven skip sheds 503" + ); + assert!( + !body.contains("TOP SECRET"), + "walk-error 503 must not leak the secret" + ); + + // The PUBLIC blob in the same repo is also not served: the walk error fails + // closed by skipping the whole repo. Without the fail-closed arm this would + // serve 200, so this assertion is the load-bearing discriminator. + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&public_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::SERVICE_UNAVAILABLE, + "walk error fails closed: repo skipped without a verdict, even the public \ + blob is not served and the scan sheds 503" + ); + } + + /// #173 review (F2): the commit/tag reachability walk must FAIL CLOSED on a git + /// error, exactly like the blob/tree walk. A ref pointing at a nonexistent object + /// makes `rev-list --all` fail, so `reachable_commit_tag_oids` returns Err, which + /// the handler's shared `Ok(Err) => continue` arm turns into a repo skip. The + /// load-bearing discriminator is that the PUBLIC commit is ALSO 404: if the arm + /// fail-OPENed (served on error) it would 200. Drives the commit/tag branch of + /// the shared fail-closed arm specifically (the sibling test covers blob/tree). + #[sqlx::test] + async fn ipfs_cid_commit_tag_walk_error_fails_closed(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["cterr"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("cterr.git"); + // A reachable commit CID — would serve 200 if the walk succeeded. + let commit_cid = pin_cid_for(&bare, &fx.commit_oid, &state.db).await; + + // A ref to a NONEXISTENT object: `git rev-list --all` fails ("bad object"), + // so reachable_commit_tag_oids bails → the walk arm skips the repo. + std::fs::write( + bare.join("refs/heads/broken"), + "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef\n", + ) + .unwrap(); + + state + .db + .create_repo(&seed_repo(&owner_did, "cterr")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "cterr") + .await + .unwrap() + .unwrap(); + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("path rule"); + + // Fail-closed: a walk error skips the repo, so even the otherwise-reachable + // public commit is NOT served. A fail-OPEN arm would 200 here. + // + // The skip is a truncation, not an absence verdict (#174 F2): the walk failed, + // so nothing was proven about whether this caller may read the object, and the + // tail sheds a retryable 503 rather than the definitive 404 this asserted + // before the merge. Withholding is the property under test either way; what + // changed is that the response no longer claims the object is absent. + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&commit_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::SERVICE_UNAVAILABLE, + "a commit/tag walk error must fail closed (repo skipped), never serve" + ); + } + + /// #126: a dangling blob (written via `git hash-object -w`, never referenced + /// by any commit/tree) must 404 through `GET /ipfs/{cid}` under path-scoped + /// rules — for anon AND the owner. The pre-#126 deny-set was fail-open by + /// construction: dangling oids were absent from the reachable enumeration + /// and thus absent from the deny-set, so the handler served 200. The + /// allowed-set is fail-closed: dangling oids are absent from the reachable + /// allowed-set, so the handler 404s (per team memory: the owner shift to + /// 404 is the accepted fail-closed default — owners can still + /// `git cat-file` directly). + #[sqlx::test] + async fn ipfs_cid_dangling_blob_fails_closed_under_path_rules(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + // Seed a normal repo with `secret/b.txt` reachable from HEAD, so the + // path-scoped rule has something to match — without this the rule has + // no anchor and we'd be testing nothing. + let _fx = seed_cid_repos(&slug, &short, &["dangling"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("dangling.git"); + + // Write a dangling blob: `git hash-object -w --stdin` adds it to the + // object DB but nothing references it, so the reachable walk never + // enumerates it. + let mut cmd = std::process::Command::new("git"); + cmd.args(["hash-object", "-w", "--stdin"]) + .current_dir(&bare) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()); + let mut child = cmd.spawn().expect("spawn git hash-object"); + { + use std::io::Write; + let stdin = child.stdin.as_mut().expect("stdin"); + stdin.write_all(b"DANGLING SECRET\n").expect("write stdin"); + } + let out = child.wait_with_output().expect("hash-object output"); + assert!( + out.status.success(), + "git hash-object: {}", + String::from_utf8_lossy(&out.stderr) + ); + let dangling_oid = String::from_utf8_lossy(&out.stdout).trim().to_string(); + // Sanity: must be a 64-hex sha256 oid, since the repo is sha256-format. + assert_eq!( + dangling_oid.len(), + 64, + "expected sha256 oid: {dangling_oid}" + ); + // Record the pin so oid_for_cid resolves it — the 404 must then come from + // the allowed-set gate excluding the dangling oid, not from a table miss. + let dangling_cid = pin_cid_for(&bare, &dangling_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "dangling")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "dangling") + .await + .unwrap() + .unwrap(); + // Path-scoped rule triggers the per-blob allowed-set gate (KTD4). + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("deny rule"); + + // anon: the dangling blob is absent from the reachable allowed-set → + // 404, no leak. Pre-#126 (deny-set) would serve 200. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&dangling_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "dangling blob must 404 under path-scoped rules" + ); + assert!( + !body.contains("DANGLING SECRET"), + "404 body must not leak the dangling content" + ); + + // owner (signed): same 404. The dangling blob has no path, so it's + // never visibility-checked → never in the allowed set, even for the + // owner. This is the accepted fail-closed shift documented in the PR. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_signed(&owner, &dangling_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "owner also 404s on dangling blobs under path-scoped rules (fail-closed default)" + ); + assert!(!body.contains("DANGLING SECRET")); + } + + /// #135: a DANGLING tree (in the ODB, referenced by no commit) 404s under + /// path-scoped rules for anon AND owner — the reachable-only allowed-tree-set + /// never enumerates it. Handler-level companion to the helper test + /// `allowed_tree_set_excludes_dangling_tree`, proving the `get_by_cid` tree arm + /// (memo insert + `!in_allowed` continue) fails closed on the dangling case. + #[sqlx::test] + async fn ipfs_cid_dangling_tree_fails_closed_under_path_rules(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["dangtree"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("dangtree.git"); + + // Dangling tree via `git mktree`: a UNIQUE entry name so its oid is + // content-distinct from every reachable tree (a content-identical tree would + // dedup to a reachable oid — that is T2, not danglingness). + let mut child = std::process::Command::new("git") + .args(["mktree"]) + .current_dir(&bare) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .spawn() + .expect("spawn git mktree"); + { + use std::io::Write; + writeln!( + child.stdin.as_mut().unwrap(), + "100644 blob {}\tdangling-only-unreferenced.txt", + fx.secret_oid + ) + .unwrap(); + } + let out = child.wait_with_output().expect("mktree output"); + assert!( + out.status.success(), + "git mktree: {}", + String::from_utf8_lossy(&out.stderr) + ); + let dangling_tree_oid = String::from_utf8_lossy(&out.stdout).trim().to_string(); + assert_eq!(dangling_tree_oid.len(), 64, "expected sha256 oid"); + // Record the pin so the 404 is the allowed-tree-set gate excluding the + // dangling tree, not a table miss. + let dangling_cid = pin_cid_for(&bare, &dangling_tree_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "dangtree")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "dangtree") + .await + .unwrap() + .unwrap(); + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("deny rule"); + + for req in [cid_anon(&dangling_cid), cid_signed(&owner, &dangling_cid)] { + let (st, _) = cid_parts(cid_router(&state).oneshot(req).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "dangling tree must 404 under path-scoped rules (anon + owner)" + ); + } + } + + /// #173 (F1): a QUARANTINED repo must not serve a pinned object by CID, to anon + /// OR to the mirror's own owner — quarantine is "hidden from serve/clone/listings, + /// owner included" (authorize_repo_read / feed_quarantined_mirror_withheld_from_owner). + /// The repo is PUBLIC with no path-scoped rule, so the "/" visibility gate ALLOWS + /// it and quarantine is the sole possible denier: RED before the fix (the loop + /// never checks quarantine → serves 200 + bytes), GREEN after the quarantine skip. + /// The owner-signed 404 is the load-bearing negative — a visibility-only gate + /// would Allow the owner and miss this. + #[sqlx::test] + async fn ipfs_cid_quarantined_repo_withheld_from_anon_and_owner(pool: PgPool) { + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["quar"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("quar.git"); + // Pin a ROOT-readable object (public/a.txt) — no path-scoped rule, so only + // quarantine can deny it. + let public_cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "quar")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "quar") + .await + .unwrap() + .unwrap(); + + // Baseline: before quarantine the object serves 200 (proves the CID resolves + // and the object is otherwise servable, so the 404 below is quarantine's doing). + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&public_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "public root object serves before quarantine" + ); + assert!(body.contains("public bytes"), "baseline serves the content"); + + // Quarantine it. + state + .db + .set_repo_quarantine(&rec.id, true) + .await + .expect("quarantine"); + + // anon AND owner-signed must both 404 with no content leak. + for req in [cid_anon(&public_cid), cid_signed(&owner, &public_cid)] { + let (st, body) = cid_parts(cid_router(&state).oneshot(req).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "quarantined repo must not serve by CID (anon + owner)" + ); + assert!( + !body.contains("public bytes"), + "404 body must not leak quarantined content" + ); + } + } + + /// #173 (F2): a DANGLING commit or annotated tag (in the ODB, referenced by no + /// ref) must 404 under path-scoped rules for anon AND owner. The resolver proved + /// reachability only for blobs/trees, so a dangling commit/tag fell through to + /// serve, leaking its message/metadata. RED before the fix (serves 200 + + /// sentinel), GREEN after (the reachable commit/tag set excludes them). The + /// reachable-commit/tag serve path is covered by + /// ipfs_cid_gate_withholds_blob_from_unauthorized (commit + annotated tag → 200). + #[sqlx::test] + async fn ipfs_cid_dangling_commit_and_tag_fail_closed_under_path_rules(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["dangct"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("dangct.git"); + + // Run a git plumbing command that reads from stdin and prints an oid. + let oid_from_stdin = |args: &[&str], input: &[u8]| -> String { + use std::io::Write; + let mut child = std::process::Command::new("git") + .args(args) + .current_dir(&bare) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .expect("spawn git"); + child.stdin.as_mut().unwrap().write_all(input).unwrap(); + let out = child.wait_with_output().expect("git output"); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + + // Dangling commit: commit-tree with a sentinel message, NO ref update. + let dangling_commit_oid = oid_from_stdin( + &["commit-tree", &fx.root_tree_oid], + b"DANGLING COMMIT SECRET\n", + ); + assert_eq!(dangling_commit_oid.len(), 64, "expected sha256 commit oid"); + // Dangling annotated tag: mktag of the dangling commit, NO ref. + let tag_body = format!( + "object {dangling_commit_oid}\ntype commit\ntag dang\ntagger t 0 +0000\n\nDANGLING TAG SECRET\n" + ); + let dangling_tag_oid = oid_from_stdin(&["mktag"], tag_body.as_bytes()); + assert_eq!(dangling_tag_oid.len(), 64, "expected sha256 tag oid"); + + let commit_cid = pin_cid_for(&bare, &dangling_commit_oid, &state.db).await; + let tag_cid = pin_cid_for(&bare, &dangling_tag_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "dangct")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "dangct") + .await + .unwrap() + .unwrap(); + // Path-scoped rule triggers the per-object gate (KTD4). + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("deny rule"); + + for (cid, sentinel) in [ + (&commit_cid, "DANGLING COMMIT SECRET"), + (&tag_cid, "DANGLING TAG SECRET"), + ] { + for req in [cid_anon(cid), cid_signed(&owner, cid)] { + let (st, body) = cid_parts(cid_router(&state).oneshot(req).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "dangling commit/tag must 404 under path-scoped rules (anon + owner)" + ); + assert!( + !body.contains(sentinel), + "404 body must not leak the dangling message: {sentinel}" + ); + } + } + } + + /// #173 review (F2 hardening): a REACHABLE commit must still serve under a + /// path-scoped rule even when the repo carries a pushable non-commit ref (an + /// annotated tag of a tree, accepted by receive-pack). `reachable_commit_tag_oids` + /// must NOT route through `assert_all_refs_are_commits` (which bails on such a + /// ref and would fail-closed 404 every reachable commit/tag CID in the repo). + /// RED before the decoupling (the guard bails → 404), GREEN after. + #[sqlx::test] + async fn ipfs_cid_reachable_commit_served_despite_non_commit_ref(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["weirdref"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("weirdref.git"); + + // A pushable non-commit ref: an annotated tag pointing at a TREE. `git tag -a` + // in the bare repo creates refs/tags/treetag -> tag object -> tree, which + // peels to a non-commit and makes assert_all_refs_are_commits bail. + let out = std::process::Command::new("git") + .args([ + "tag", + "-a", + "treetag", + &fx.root_tree_oid, + "-m", + "tag of a tree", + ]) + .current_dir(&bare) + .output() + .expect("git tag -a"); + assert!( + out.status.success(), + "git tag -a: {}", + String::from_utf8_lossy(&out.stderr) + ); + + // Pin the REACHABLE root commit. + let commit_cid = pin_cid_for(&bare, &fx.commit_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "weirdref")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "weirdref") + .await + .unwrap() + .unwrap(); + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("path rule"); + + // The reachable commit must still serve — the non-commit ref must not + // fail-closed the whole repo's commit/tag CID retrieval. + let resp = cid_router(&state) + .oneshot(cid_anon(&commit_cid)) + .await + .unwrap(); + let served_hash = resp + .headers() + .get("x-git-hash") + .and_then(|v| v.to_str().ok()) + .map(str::to_string); + let (st, _body) = cid_parts(resp).await; + assert_eq!( + st, + StatusCode::OK, + "a reachable commit must serve despite a pushable non-commit ref in the repo" + ); + assert_eq!( + served_hash.as_deref(), + Some(fx.commit_oid.as_str()), + "the served object is the reachable root commit" + ); + } + + /// #173 review (F-F): an annotated tag pointing at a TREE is pushable through + /// receive-pack, and the TREE allowed-set path + /// (`allowed_tree_set_for_caller` -> `tree_paths` -> `reachable_commits`) runs + /// `assert_all_refs_are_commits`, which bails on that ref and fail-closes the + /// whole repo — 404-ing EVERY tree CID (root + public subtrees) for its owner + /// and readers, not just the offending tag. The tree allowed-set feeds ONLY the + /// CID gate (absence = fail-closed 404), so `tree_paths` uses the lenient + /// reachable-commit enumeration: commit-reachable trees still serve, while a + /// tree reachable only via such a tag stays excluded. `blob_paths` keeps the + /// strict guard (it also feeds serve/replication, where a miss under-withholds). + /// RED before the decoupling (whole-repo bail -> 404 on the root/public tree), + /// GREEN after; the withheld-subtree 404 is the load-bearing must-not. + #[sqlx::test] + async fn ipfs_cid_tree_served_despite_non_commit_ref(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["treeweird"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("treeweird.git"); + + // Pushable non-commit ref: an annotated tag pointing at the ROOT TREE. + let out = std::process::Command::new("git") + .args([ + "tag", + "-a", + "treetag", + &fx.root_tree_oid, + "-m", + "tag of a tree", + ]) + .current_dir(&bare) + .output() + .expect("git tag -a"); + assert!( + out.status.success(), + "git tag -a: {}", + String::from_utf8_lossy(&out.stderr) + ); + + // Pin the reachable root tree and public subtree (both at ALLOWED paths), + // plus the secret subtree (a DENIED path — the fail-closed negative). + let root_tree_cid = pin_cid_for(&bare, &fx.root_tree_oid, &state.db).await; + let public_tree_cid = pin_cid_for(&bare, &fx.public_tree_oid, &state.db).await; + let secret_tree_cid = pin_cid_for(&bare, &fx.secret_tree_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "treeweird")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "treeweird") + .await + .unwrap() + .unwrap(); + // Path-scoped rule triggers the per-object tree gate (KTD4). + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("path rule"); + + // Reachable trees at ALLOWED paths must still serve despite the tag-of-tree. + for (cid, want_oid, label) in [ + (&root_tree_cid, &fx.root_tree_oid, "root tree"), + (&public_tree_cid, &fx.public_tree_oid, "public subtree"), + ] { + let resp = cid_router(&state).oneshot(cid_anon(cid)).await.unwrap(); + let served = resp + .headers() + .get("x-git-hash") + .and_then(|v| v.to_str().ok()) + .map(str::to_string); + let (st, _) = cid_parts(resp).await; + assert_eq!( + st, + StatusCode::OK, + "{label} CID must serve despite a pushable tag-of-tree in the repo" + ); + assert_eq!( + served.as_deref(), + Some(want_oid.as_str()), + "{label}: the served object is the reachable tree" + ); + } + + // Fail-closed preserved: the DENIED subtree's CID is still withheld — the + // lenient walk must not under-withhold a path the caller cannot read. + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&secret_tree_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "a withheld subtree's tree CID stays 404 (lenient walk must not under-withhold)" + ); + } + + /// #173 review (F2 hardening): the INNER tag object of a nested tag-of-a-tag is + /// reachable (via the outer ref tag) and pinnable, so its CID must serve under a + /// path rule. `reachable_commit_tag_oids` peels tag chains to include it. RED + /// before the peel loop (the inner tag is not a ref tip and rev-list dereferences + /// to the commit, so it is absent → 404), GREEN after. + #[sqlx::test] + async fn ipfs_cid_nested_tag_inner_object_served(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["nested"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("nested.git"); + + let git_stdin = |args: &[&str], input: &[u8]| -> String { + use std::io::Write; + let mut child = std::process::Command::new("git") + .args(args) + .current_dir(&bare) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .expect("spawn git"); + child.stdin.as_mut().unwrap().write_all(input).unwrap(); + let out = child.wait_with_output().expect("git output"); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + + // Inner annotated tag of the reachable commit (no ref of its own). + let inner_body = format!( + "object {}\ntype commit\ntag inner\ntagger t 0 +0000\n\ninner\n", + fx.commit_oid + ); + let inner_tag_oid = git_stdin(&["mktag"], inner_body.as_bytes()); + // Outer annotated tag of the inner tag, then a ref to the outer tag. The + // inner tag is reachable only THROUGH the outer, not as a ref tip. + let outer_body = format!( + "object {inner_tag_oid}\ntype tag\ntag outer\ntagger t 0 +0000\n\nouter\n" + ); + let outer_tag_oid = git_stdin(&["mktag"], outer_body.as_bytes()); + let out = std::process::Command::new("git") + .args(["update-ref", "refs/tags/nested", &outer_tag_oid]) + .current_dir(&bare) + .output() + .expect("update-ref"); + assert!( + out.status.success(), + "update-ref: {}", + String::from_utf8_lossy(&out.stderr) + ); + + let inner_cid = pin_cid_for(&bare, &inner_tag_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "nested")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "nested") + .await + .unwrap() + .unwrap(); + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("path rule"); + + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&inner_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "the inner tag of a nested tag-of-a-tag is reachable and must serve" + ); + } + + /// #135: with NO path-scoped rule the per-object gate is skipped, so a tree CID + /// is served (the `"/"` gate is the whole story). Guards against over-gating + /// trees — the tree analog of the blob skip-walk branch. + #[sqlx::test] + async fn ipfs_cid_tree_served_when_no_path_scoped_rule(pool: PgPool) { + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["nopathrule"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("nopathrule.git"); + let tree_cid = pin_cid_for(&bare, &fx.secret_tree_oid, &state.db).await; + + // Public repo, no visibility rules → has_path_scoped_rule is false. + state + .db + .create_repo(&seed_repo(&owner_did, "nopathrule")) + .await + .expect("seed repo"); + + let (st, body) = cid_bytes( + cid_router(&state) + .oneshot(cid_anon(&tree_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "tree served to anon when no path-scoped rule exists" + ); + assert!( + bytes_contain(&body, b"b.txt"), + "served tree carries its child structure" + ); + } + + /// #173 (Fix 1): the pinned_cids lookup must use the canonical base32 CID, not + /// the raw request spelling. A pin is stored under `cid.to_string()` (canonical + /// base32); a request carrying the SAME CID re-encoded to a different multibase + /// (base58btc) parses and passes the sha2-256 check but, on the pre-fix handler, + /// misses the lookup key → false 404. Public repo, no path-scoped rule, so no + /// walk — this isolates the lookup-key canonicalization. + #[sqlx::test] + async fn ipfs_alt_encoding_cid_resolves(pool: PgPool) { + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["altenc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("altenc.git"); + // Canonical base32 CID as stored by the pin path. + let public_cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + + // Public repo, no visibility rules (no path-scoped walk). + state + .db + .create_repo(&seed_repo(&owner_did, "altenc")) + .await + .expect("seed repo"); + + // Re-encode the SAME CID to base58btc — a different, equally-valid spelling + // that is NOT the stored key. The `cid` crate re-exports `multibase`. + let alt = public_cid + .parse::>() + .unwrap() + .to_string_of_base(cid::multibase::Base::Base58Btc) + .unwrap(); + assert_ne!(alt, public_cid, "alt encoding must differ from canonical"); + + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&alt)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::OK, + "alt-multibase spelling of a pinned CID must resolve (canonicalized lookup)" + ); + assert!( + body.contains("public bytes"), + "resolved object serves its content" + ); + } + + /// #173 (Fix 2a, db-level): `oids_for_cid` returns EVERY oid recorded under a + /// CID, not an arbitrary one. `record_pinned_cid` is unique on the git oid and + /// non-unique on cid, so two distinct oids can share one content-CID. Old + /// `oid_for_cid` did `LIMIT 1`; the new plural method must surface both. + #[sqlx::test] + async fn oids_for_cid_returns_all_duplicates(pool: PgPool) { + let state = test_state(pool).await; + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(b"shared content cid").to_string(); + let oid_a = "a".repeat(64); + let oid_b = "b".repeat(64); + state + .db + .record_pinned_cid(&oid_a, &cid, None) + .await + .unwrap(); + state + .db + .record_pinned_cid(&oid_b, &cid, None) + .await + .unwrap(); + + let mut oids = state.db.oids_for_cid(&cid).await.unwrap(); + oids.sort(); + assert_eq!( + oids, + vec![oid_a, oid_b], + "oids_for_cid must return every oid recorded under the shared CID" + ); + } + + /// #173 (Fix 2b, handler-level): when two oids collide on one CID and the + /// first-recorded is absent from every repo while the second is a readable + /// public object, the handler must try both and serve the readable one. The + /// pre-fix handler resolved a single oid (LIMIT 1 → first-inserted for equal + /// keys) and 404'd. Ordering caveat: this relies on `oids_for_cid` returning + /// the absent oid before the readable one (heap/insert order for equal keys); + /// if that ordering ever changes, `oids_for_cid_returns_all_duplicates` remains + /// the load-bearing, deterministic driver for Fix 2. + #[sqlx::test] + async fn ipfs_cid_collision_serves_readable_duplicate(pool: PgPool) { + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["collision"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("collision.git"); + + // A GENUINE content collision: the shared CID is the readable object's REAL + // content CID, and a second (absent) oid is recorded under the SAME cid. The + // handler must try every oid and serve the one whose bytes hash to the CID. + // (F2, #173: the served bytes must match the requested content address, so the + // shared cid has to be the object's real cid — an arbitrary seed would now be + // withheld by the integrity check as an unverifiable provider-CID-style row.) + let (_ty, raw) = crate::git::store::read_object(&bare, &fx.public_oid) + .unwrap() + .unwrap(); + let shared_cid = gitlawb_core::cid::Cid::from_git_object_bytes(&raw).to_string(); + let absent_oid = "c".repeat(64); + state + .db + .record_pinned_cid(&absent_oid, &shared_cid, None) .await - .expect("withhold repo"); + .expect("record absent oid first"); state .db - .set_visibility_rule( - &withhold.id, - "/secret/**", - VisibilityMode::B, - &[], - &owner_did, - ) + .record_pinned_cid(&fx.public_oid, &shared_cid, None) .await - .expect("deny rule"); + .expect("record readable oid second"); - // Public copy, no rules, iterated AFTER. - let mut pubcopy = seed_repo(&owner_did, "pubcopy"); - pubcopy.updated_at = Utc::now() - chrono::Duration::seconds(60); - state.db.create_repo(&pubcopy).await.expect("pubcopy repo"); + // Public repo, no rules → the readable public object is served if reached. + state + .db + .create_repo(&seed_repo(&owner_did, "collision")) + .await + .expect("seed repo"); - // anon: denied at the withholding repo (continue), served from the public copy. let (st, body) = cid_parts( cid_router(&state) - .oneshot(cid_anon(&secret_cid)) + .oneshot(cid_anon(&shared_cid)) .await .unwrap(), ) @@ -3459,54 +13026,191 @@ mod tests { assert_eq!( st, StatusCode::OK, - "served from the public copy despite the other deny" + "handler must try every oid under the CID and serve the readable duplicate" ); assert!( - body.contains("TOP SECRET"), - "the public copy serves the content" + body.contains("public bytes"), + "the readable duplicate's content is served" ); } - /// Repo-level "/" gate (KTD2a, first continue branch): a fully private repo - /// (is_public=false, no rules) denies anon before any per-blob check; the - /// owner still reads. The path-scoped tests pass the "/" gate and deny at the - /// per-blob stage, so this exercises the coarser repo-level deny separately. + /// #173 (Fix 3/F3, INV-10): the expensive legacy fan-out is rate-limited per + /// source IP. A valid tree CID makes the object-type pre-check pass, so each + /// repeat request pays a fresh walk (request-scoped memo only) — unbounded + /// amplification. Since #173-F3 (jatmn) the source charge sits on the LEGACY + /// PROBE (`acquire` + `cat-file`), which precedes the walk, so every legacy + /// candidate is charged to the non-farmable source IP from the first probe; a + /// second identical request from the same IP is shed with 429, but a targeted + /// PROVENANCE fetch (no scan) and a request from a different IP are unaffected. + /// The limiter is sized to admit one full scan of the two seeded repos (2 probes) + /// so the first request serves; the repeat then finds the bucket spent. #[sqlx::test] - async fn ipfs_cid_private_repo_denies_anon_at_repo_gate(pool: PgPool) { + async fn ipfs_walk_rate_limited_per_source(pool: PgPool) { + use crate::db::VisibilityMode; use gitlawb_core::identity::Keypair; let owner = Keypair::generate(); let owner_did = owner.did().to_string(); + let reader = Keypair::generate(); + let reader_did = reader.did().to_string(); let slug = owner_did.replace([':', '/'], "_"); let short = owner_did.split(':').next_back().unwrap().to_string(); - let state = test_state(pool).await; - let fx = seed_cid_repos(&slug, &short, &["priv"]); - let blob_cid = cid_for_oid(&fx.public_oid); + let mut state = test_state(pool).await; + // The scan reads both seeded repos (walklimit + walkpublic) in one page and + // probes each, so size the per-IP budget to admit exactly one full scan: + // 1 page + 2 probes. A repeat scan from the same IP then finds the bucket spent. + // Keyed on the rightmost X-Forwarded-For hop so the test can choose a source IP + // under `oneshot`. The page is charged because the scan's DB-facing pages draw + // on this same bucket (#173 round 13, F2). Production is floored at + // probes + pages by `AppState::ipfs_work_budget`; a hand-set limiter is not. + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(3, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + let fx = seed_cid_repos(&slug, &short, &["walklimit"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("walklimit.git"); + // The tree CID drives a path-scoped walk (the load-bearing amplification + // surface). The reader is allowed under /secret so the walk returns 200. + let secret_tree_cid = pin_cid_for(&bare, &fx.secret_tree_oid, &state.db).await; + + // NEWEST `created_at` → the paged scan (ORDER BY created_at, id ASC) probes + // this serving repo LAST, so a scan deterministically charges the walk-free + // `walkpublic` miss first then this serve: exactly 2 probes per scan. + let mut walklimit = seed_repo(&owner_did, "walklimit"); + walklimit.created_at = chrono::Utc::now() + chrono::Duration::seconds(60); + state.db.create_repo(&walklimit).await.expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "walklimit") + .await + .unwrap() + .unwrap(); + // Mode B path rule over /secret with the reader allowed → the reader's + // secret-tree fetch runs the allowed-tree walk and returns 200. + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + VisibilityMode::B, + std::slice::from_ref(&reader_did), + &owner_did, + ) + .await + .expect("path rule"); + + // The MUST-NOT object must be a genuinely CHEAP fetch: an object served + // from a repo with NO path-scoped rule takes the no-walk path, so the WALK + // brake never rate-limits it. It has to live in a repo that carries no path + // rule AND whose object graph does not overlap `walklimit` (a blob shared + // with the path-scoped repo would still walk there), so we seed a second bare + // repo with UNIQUE content. `acquire(owner, "walkpublic")` resolves to + // `/tmp//walkpublic.git`. This copy is PROVENANCED (`pin_cid_for_repo`) + // so it resolves straight to its repo and skips the legacy probe brake: the + // point here is the WALK brake, and post-#173-F3 a walk-free LEGACY fetch is + // itself source-charged at the probe, so a legacy pin would (correctly) be + // shed from the exhausted IP and no longer isolate the walk brake. + let pub_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("walkpublic.git"); + { + use std::process::Command; + let run = |args: &[&str], cwd: &std::path::Path| { + let out = Command::new("git") + .args(args) + .current_dir(cwd) + .output() + .expect("git runs"); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); + }; + let src = std::env::temp_dir().join(format!("gl-cid-pub-{short}")); + let _ = std::fs::remove_dir_all(&src); + std::fs::create_dir_all(&src).unwrap(); + std::fs::write(src.join("cheap.txt"), b"cheap public bytes\n").unwrap(); + run(&["init", "-q", "--object-format=sha256"], &src); + run(&["config", "user.email", "t@t"], &src); + run(&["config", "user.name", "t"], &src); + run(&["add", "."], &src); + run(&["commit", "-qm", "cheap"], &src); + let _ = std::fs::remove_dir_all(&pub_bare); + run( + &[ + "clone", + "--bare", + "-q", + src.to_str().unwrap(), + pub_bare.to_str().unwrap(), + ], + &src, + ); + let _ = std::fs::remove_dir_all(&src); + } + let cheap_oid = { + use std::process::Command; + let out = Command::new("git") + .args(["rev-parse", "HEAD:cheap.txt"]) + .current_dir(&pub_bare) + .output() + .unwrap(); + assert!(out.status.success(), "rev-parse cheap.txt"); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + // Public repo, NO visibility rules → the cheap object takes the no-walk path. + state + .db + .create_repo(&seed_repo(&owner_did, "walkpublic")) + .await + .expect("seed public repo"); + let pub_rec = state + .db + .get_repo(&owner_did, "walkpublic") + .await + .unwrap() + .unwrap(); + let public_cid = pin_cid_for_repo(&pub_bare, &cheap_oid, &state.db, &pub_rec.id).await; - let mut rec = seed_repo(&owner_did, "priv"); - rec.is_public = false; - state.db.create_repo(&rec).await.expect("private repo"); + // 1st legacy scan from 1.2.3.4 → 200 (its two probes fit the budget; the + // walk ran, reader allowed). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_signed_xff(&reader, &secret_tree_cid, "1.2.3.4")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "1st legacy scan from a source IP is served" + ); - // anon → repo-level deny → 404, no content leaked. - let (st, body) = cid_parts( + // 2nd identical scan from the SAME IP → 429 (per-IP probe budget spent). + let (st, _) = cid_parts( cid_router(&state) - .oneshot(cid_anon(&blob_cid)) + .oneshot(cid_signed_xff(&reader, &secret_tree_cid, "1.2.3.4")) .await .unwrap(), ) .await; assert_eq!( st, - StatusCode::NOT_FOUND, - "anon denied at a private repo's / gate" + StatusCode::TOO_MANY_REQUESTS, + "2nd legacy scan from the same source IP is shed with 429" ); - assert!(!body.contains("public bytes"), "404 must not leak content"); - // owner-signed → 200. + // MUST-NOT: a targeted PROVENANCE fetch (no scan, no probe brake) from the + // SAME limited IP, even after the 429, is served: the brake is on the legacy + // scan, not the route. let (st, body) = cid_parts( cid_router(&state) - .oneshot(cid_signed(&owner, &blob_cid)) + .oneshot(cid_signed_xff(&reader, &public_cid, "1.2.3.4")) .await .unwrap(), ) @@ -3514,70 +13218,309 @@ mod tests { assert_eq!( st, StatusCode::OK, - "owner reads their private repo's object" + "a provenance (non-scan) fetch is never rate-limited, even from the exhausted IP" + ); + assert!( + body.contains("cheap public bytes"), + "the cheap fetch serves content" + ); + + // PER-SOURCE isolation: the same tree-CID scan from a DIFFERENT IP → 200. + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_signed_xff(&reader, &secret_tree_cid, "5.6.7.8")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "one source's exhaustion must not shed another source's walk" ); - assert!(body.contains("public bytes"), "owner gets the content"); } - /// Fail-closed walk-error arm: if `withheld_blob_oids` errors (here, a ref - /// pointing at a non-tree-ish blob, which `git ls-tree -r` cannot traverse — - /// the same induction as `visibility_pack::fails_closed_when_a_ref_cannot_be_traversed`), - /// the handler skips the whole repo rather than serving. Asserts no leak of the - /// withheld blob AND that even the *public* blob in that repo is withheld — the - /// latter distinguishes fail-closed-skip from normal per-blob withholding and - /// would serve 200 if the error arm wrongly proceeded. The skip carries no - /// VERDICT (F2), so the response is the retryable truncation 503, not a 404 - /// claiming the object is absent — never-serve-unproven and never-404-unproven - /// hold together. + /// #173 review (F-C): a SKIPPED legacy candidate (a walk-and-deny denier, OR a + /// probe-throttled repo since #173-F3) must not end the whole request: the scan + /// keeps going so a later walk-free copy still serves, and a spent probe budget is + /// a clean 429, never a false 404/503. Otherwise a public CID would 404/429 solely + /// because a path-scoped duplicate sorts ahead of a no-rule copy under the scan's + /// `(created_at, id)` ASC order. Two same-oid legacy copies: a `/secret`-scoped + /// denier iterated first and a no-rule public copy behind it. + /// + /// Two requests from the SAME IP, budget = 2 (one full scan of both copies): + /// req1 probes the denier (charged), its allowed-blob walk denies anon → skip and + /// keep scanning, then probes+serves the walk-free public copy → 200. That proves + /// the denier skip is non-fatal (`continue`, not `break`). req2 from the same IP + /// finds the probe budget spent, so the denier's probe throttles → skip-continue, + /// the public copy's probe throttles too → nothing servable → a clean 429 (not a + /// truncation 503 nor a false 404), proving the throttle is likewise non-fatal but + /// correctly shed. RED before `continue` (a `break` on the skipped denier 404s + /// req1 outright). #[sqlx::test] - async fn ipfs_cid_walk_error_fails_closed(pool: PgPool) { + async fn ipfs_walk_quota_skips_denier_and_serves_public_copy(pool: PgPool) { use crate::db::VisibilityMode; + use chrono::Utc; use gitlawb_core::identity::Keypair; let owner = Keypair::generate(); let owner_did = owner.did().to_string(); let slug = owner_did.replace([':', '/'], "_"); let short = owner_did.split(':').next_back().unwrap().to_string(); - let state = test_state(pool).await; + let mut state = test_state(pool).await; + // Budget = one full two-repo scan: 1 page + 2 probes. Keyed on the rightmost XFF + // hop so `oneshot` can choose a source IP (no socket peer). A repeat scan from + // the same IP then finds the budget spent. The page is charged because the + // scan's DB-facing pages draw on this same bucket (#173 round 13, F2); both + // repos fit in one 128-row page. Production is floored at probes + pages by + // `AppState::ipfs_work_budget`, so only a hand-set limiter counts this out. + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(3, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + // Identical secret-blob content in both bare clones → one CID resolves to + // `secret_oid` in each. A NEWER path-scoped denier (walk-and-deny anon) and an + // OLDER no-rule public copy (walk-free serve). + let fx = seed_cid_repos(&slug, &short, &["scopeddenier", "publiccopy"]); + let denier_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("scopeddenier.git"); + let secret_cid = pin_cid_for(&denier_bare, &fx.secret_oid, &state.db).await; + + // First-iterated denier: public at "/", `/secret/**` Mode B empty readers → an + // anon blob fetch clears "/", runs the allowed-blob walk, is denied → continue. + // The paged scan orders on the immutable `(created_at, id)` ASC (#173, jatmn). + let mut denier = seed_repo(&owner_did, "scopeddenier"); + denier.created_at = Utc::now() - chrono::Duration::seconds(60); + state.db.create_repo(&denier).await.expect("seed denier"); + state + .db + .set_visibility_rule(&denier.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("path rule"); - let fx = seed_cid_repos(&slug, &short, &["withhold"]); - let secret_cid = cid_for_oid(&fx.secret_oid); - let public_cid = cid_for_oid(&fx.public_oid); + // Public copy behind it — NO rule → the secret blob serves via the no-walk path. + let mut public = seed_repo(&owner_did, "publiccopy"); + public.created_at = Utc::now(); + state + .db + .create_repo(&public) + .await + .expect("seed public copy"); - // Force the withheld walk to fail closed: a ref pointing at a blob (not - // tree-ish) makes `git ls-tree -r` error, which `withheld_blob_oids` - // propagates as Err → the handler's `Ok(Err)` arm skips the repo. - let bare = std::path::PathBuf::from("/tmp") - .join(&slug) - .join("withhold.git"); - std::fs::write( - bare.join("refs/heads/blobref"), - format!("{}\n", fx.secret_oid), + // req1 from 1.2.3.4: the denier is skipped (walk denies anon) and the scan + // keeps going to serve the older walk-free public copy. Both probes fit the + // budget, so this leaves the IP bucket spent. + let resp = cid_router(&state) + .oneshot(cid_anon_xff(&secret_cid, "1.2.3.4")) + .await + .unwrap(); + let served_hash = resp + .headers() + .get("x-git-hash") + .and_then(|v| v.to_str().ok()) + .map(str::to_string); + let (st, _body) = cid_parts(resp).await; + assert_eq!( + st, + StatusCode::OK, + "a skipped walk-requiring denier must not end the scan: the later walk-free public copy still serves" + ); + assert_eq!( + served_hash.as_deref(), + Some(fx.secret_oid.as_str()), + "the served object is the secret blob from the no-rule public copy" + ); + + // req2 from the SAME exhausted IP: every legacy probe is now throttled. The + // throttle is non-fatal (skip and keep scanning), but nothing is servable, so + // it resolves to a clean 429, not a truncation 503, not a false 404. + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon_xff(&secret_cid, "1.2.3.4")) + .await + .unwrap(), ) - .unwrap(); + .await; + assert_eq!( + st, + StatusCode::TOO_MANY_REQUESTS, + "with the probe budget spent, the repeat legacy scan is shed with a clean 429" + ); + } + + /// INV-10 amplification bound: a single `GET /ipfs/{cid}` must not fan out an + /// unbounded number of full-history walks. The route brake (`ipfs_rate_limiter`) + /// fires once per request and the per-walk `ipfs_work_rate_limiter` charge bounds + /// walk work across requests, but within ONE request the same object can exist under + /// path-scoped rules in many repos, each paying its own walk. + /// `MAX_HISTORY_WALKS_PER_REQUEST` caps that fan-out. + /// + /// Load-bearing witness (#173, F4): a readable public copy (no path rule → + /// served via the no-walk path, exactly like + /// `ipfs_cid_served_from_public_copy_when_withheld_elsewhere`) is given the + /// NEWEST `created_at` so the paged scan (ORDER BY created_at, id ASC) iterates it + /// LAST. Ahead of it sit `cap + 1` path-scoped deniers, each forcing an + /// allowed-blob walk that denies anon. The cap bounds SPAWNED walks to `cap`, but + /// hitting it must `continue` (skip only the walk-requiring denier), NOT `break` + /// the whole repo loop: the walk-free public copy needs no walk, so it is still + /// reached and served (200, `x-git-hash` = the blob oid). The old `break` + /// wrongly 404'd this publicly-readable content. Reverting `continue`→`break` + /// turns this 200 back into a 404: the RED proof that the loop keeps scanning for + /// a cheap readable copy after the cap. The `cap` walk ceiling still holds — only + /// `cap` walks are spawned across the deniers regardless (the amplification bound + /// is proven separately by `ipfs_walk_cap_still_serves_walk_free_candidate`). + #[sqlx::test] + async fn ipfs_walk_fanout_capped_per_request(pool: PgPool) { + use crate::db::VisibilityMode; + use chrono::Utc; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let cap = crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST as usize; + // `cap + 1` deniers guarantee the fan-out crosses the ceiling before the + // readable copy (iterated last) is reached. All bare clones share identical + // content, so the one secret-BLOB CID resolves to `secret_oid` in every repo. + let denier_names: Vec = (0..=cap).map(|i| format!("denier{i}")).collect(); + let mut names: Vec<&str> = vec!["readable"]; + names.extend(denier_names.iter().map(|s| s.as_str())); + let fx = seed_cid_repos(&slug, &short, &names); + + let readable_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("readable.git"); + // The secret BLOB CID drives the path-scoped allowed-blob walk in every + // denier (the amplification surface) and is served cheaply from the + // no-rule public copy — the proven serve path. + let secret_cid = pin_cid_for(&readable_bare, &fx.secret_oid, &state.db).await; + + // 1) Readable public copy — NEWEST created_at → iterated LAST under the paged + // `(created_at, id)` ASC order (#173, jatmn). Public with NO visibility + // rule, so the blob serves via the no-walk path. This is the copy an + // uncapped fan-out would eventually reach and serve. + let mut readable = seed_repo(&owner_did, "readable"); + readable.created_at = Utc::now() + chrono::Duration::seconds(60); state .db - .create_repo(&seed_repo(&owner_did, "withhold")) + .create_repo(&readable) .await - .expect("seed repo"); - let rec = state - .db - .get_repo(&owner_did, "withhold") + .expect("seed readable copy"); + + // 2) cap+1 deniers with OLDER created_at → iterated before the copy. Public + // at "/", but a `/secret/**` Mode B rule with an EMPTY reader list, so an + // anon blob fetch clears the "/" gate, runs the allowed-blob walk, and is + // denied (the secret blob is in no one's set) → continue. Each distinct + // repo.id is its own walk (the memo only dedups the same repo). + for name in &denier_names { + let mut denier = seed_repo(&owner_did, name); + denier.created_at = Utc::now(); + state.db.create_repo(&denier).await.expect("seed denier"); + state + .db + .set_visibility_rule(&denier.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("path rule"); + } + + // Anon (no peer, no XFF → the IP brake is skipped, so the walk cap is the + // only thing in play). After the cap, `continue` skips only the + // walk-requiring deniers and keeps scanning, reaching the walk-free public + // copy (iterated last) → served 200. The served object is the secret blob + // from the no-rule public copy, which is legitimately public THERE. + let resp = cid_router(&state) + .oneshot(cid_anon(&secret_cid)) .await - .unwrap() .unwrap(); + let served_hash = resp + .headers() + .get("x-git-hash") + .and_then(|v| v.to_str().ok()) + .map(str::to_string); + let (st, _body) = cid_parts(resp).await; + assert_eq!( + st, + StatusCode::OK, + "hitting the walk cap must skip only the walk-requiring candidate, not abandon the walk-free readable copy" + ); + assert_eq!( + served_hash.as_deref(), + Some(fx.secret_oid.as_str()), + "the served object is the blob from the no-rule public copy reached after the cap" + ); + } + + /// Multi-oid companion to `ipfs_walk_fanout_capped_per_request`: exercises the + /// outer oid loop and proves the per-request walk budget PERSISTS across oid + /// candidates, so a commit/tag candidate cannot re-open the fan-out. Since #173 + /// (F2) a `commit`/`tag` under a path-scoped rule is itself walk-gated (its + /// reachability is proven by a `rev-list` walk via `reachable_commit_tag_oids`), + /// so it is NOT walk-free — it draws from the same budget as the blob/tree walks. + /// + /// One CID → TWO oids (the non-unique cid index, #173): a withheld `/secret` + /// blob (walk-triggering, denied to anon in every denier) recorded FIRST so a + /// seq scan tries it first and burns the whole walk budget across the deniers; + /// the reachable root commit is second. Because the budget is already spent, the + /// commit candidate's reachability walk is also capped in every denier, so the + /// request 404s — proving commit/tag walks (F2) respect the fan-out ceiling and + /// cannot be used to bypass it (R6/F3). A reachable commit served with budget to + /// spare is covered by `ipfs_cid_gate_withholds_blob_from_unauthorized`. The + /// withheld blob must not leak. Since #173 F2 a scan the walk cap truncated + /// returns 503 (absence unproven), not the old opaque 404. + #[sqlx::test] + async fn ipfs_walk_commit_tag_candidate_respects_the_walk_cap(pool: PgPool) { + use crate::db::VisibilityMode; + use chrono::Utc; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let cap = crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST as usize; + + // cap+1 path-scoped deniers, all carrying identical content (same oids). + let denier_names: Vec = (0..=cap).map(|i| format!("m{i}")).collect(); + let names: Vec<&str> = denier_names.iter().map(|s| s.as_str()).collect(); + let fx = seed_cid_repos(&slug, &short, &names); + let bare = std::path::PathBuf::from("/tmp").join(&slug).join("m0.git"); + + // ONE cid → TWO oids. The withheld blob is recorded first (seq scan lists it + // first → tried first → burns the budget); the reachable commit is second. + let multi_cid = pin_cid_for(&bare, &fx.secret_oid, &state.db).await; state .db - .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .record_pinned_cid(&fx.commit_oid, &multi_cid, None) .await - .expect("deny rule"); + .expect("co-locate the commit oid under the same cid"); - // Withheld secret CID under a walk error → the repo is skipped without a - // verdict, so the scan is truncated (503), and nothing leaks. + for name in &denier_names { + let mut d = seed_repo(&owner_did, name); + d.updated_at = Utc::now(); + state.db.create_repo(&d).await.expect("seed denier"); + state + .db + .set_visibility_rule(&d.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("path rule"); + } + + // Anon: the blob candidate is denied in every denier (a walk each, spending + // the budget); the commit candidate's reachability walk is then also capped + // in every denier — so no candidate is served AND the walk cap truncated the + // scan, leaving absence unproven → 503 (not the old false 404, #173 F2). + // Either way commit/tag walks respect the ceiling and cannot re-open the + // fan-out (R6/F3). The withheld blob must not leak in the body. let (st, body) = cid_parts( cid_router(&state) - .oneshot(cid_anon(&secret_cid)) + .oneshot(cid_anon(&multi_cid)) .await .unwrap(), ) @@ -3585,141 +13528,251 @@ mod tests { assert_eq!( st, StatusCode::SERVICE_UNAVAILABLE, - "walk error must not serve the withheld blob — the unproven skip sheds 503" + "a commit/tag reachability walk respects the per-request cap; a truncated scan is 503, not a false 404" ); assert!( !body.contains("TOP SECRET"), - "walk-error 503 must not leak the secret" + "the withheld blob must not leak in the truncation response" ); + } - // The PUBLIC blob in the same repo is also not served: the walk error fails - // closed by skipping the whole repo. Without the fail-closed arm this would - // serve 200, so this assertion is the load-bearing discriminator. + /// #173 (F3, INV-15): the per-IP quota debits ONE token per expensive legacy + /// candidate, not once per request, so one IP cannot drive an unbounded fan-out. + /// With quota=1 and two path-scoped deniers holding one CID, a SINGLE request is + /// shed at 429: since #173-F3 (jatmn) each legacy PROBE (`acquire` + `cat-file`, + /// which precedes the walk) debits, so the first denier probes+walks+denies on + /// token 1 and the second denier's probe finds no token → 429. (Before F3 the + /// debit sat on the walk; the outcome is unchanged, the charge point moved earlier + /// to also bound walk-free probes.) Defeating the per-candidate debit let one IP + /// drive up to MAX_HISTORY_WALKS_PER_REQUEST × quota expensive ops/hour. + #[sqlx::test] + async fn ipfs_walk_quota_debited_per_walk(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + // Signed but NOT a reader → cleared at "/", denied at /secret → forces a walk. + let stranger = Keypair::generate(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + + let mut state = test_state(pool).await; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + let fx = seed_cid_repos(&slug, &short, &["w0", "w1"]); + let bare = std::path::PathBuf::from("/tmp").join(&slug).join("w0.git"); + // The secret BLOB CID forces a path-scoped allowed-blob walk in each denier. + let secret_cid = pin_cid_for(&bare, &fx.secret_oid, &state.db).await; + + // Two path-scoped deniers (Mode B /secret, empty readers): each forces a + // walk that denies the signed stranger, so ONE request spawns two walks. + for name in ["w0", "w1"] { + let d = seed_repo(&owner_did, name); + state.db.create_repo(&d).await.expect("seed denier"); + state + .db + .set_visibility_rule(&d.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("path rule"); + } + + // ONE request, quota 1: walk 1 debits the token, walk 2 has none → 429. let (st, _) = cid_parts( cid_router(&state) - .oneshot(cid_anon(&public_cid)) + .oneshot(cid_signed_xff(&stranger, &secret_cid, "1.2.3.4")) .await .unwrap(), ) .await; assert_eq!( st, - StatusCode::SERVICE_UNAVAILABLE, - "walk error fails closed: repo skipped without a verdict, even the public \ - blob is not served and the scan sheds 503" + StatusCode::TOO_MANY_REQUESTS, + "the second full-history walk in one request must be shed with 429 (per-walk debit)" ); } - /// #126: a dangling blob (written via `git hash-object -w`, never referenced - /// by any commit/tree) must 404 through `GET /ipfs/{cid}` under path-scoped - /// rules — for anon AND the owner. The pre-#126 deny-set was fail-open by - /// construction: dangling oids were absent from the reachable enumeration - /// and thus absent from the deny-set, so the handler served 200. The - /// allowed-set is fail-closed: dangling oids are absent from the reachable - /// allowed-set, so the handler 404s (per team memory: the owner shift to - /// 404 is the accepted fail-closed default — owners can still - /// `git cat-file` directly). + /// The periodic cleanup task must sweep the ipfs walk limiter, not only its + /// five siblings. Drives `AppState::sweep_rate_limiters` — the exact method the + /// 300s loop calls — and asserts the ipfs limiter's expired entry is evicted. + /// Dropping `ipfs_rate_limiter.cleanup()` from that method leaves the entry in + /// place (`tracked_keys` stays 1): the RED proof that the sweep covers it. #[sqlx::test] - async fn ipfs_cid_dangling_blob_fails_closed_under_path_rules(pool: PgPool) { - use crate::db::VisibilityMode; - use gitlawb_core::identity::Keypair; + async fn sweep_rate_limiters_includes_ipfs_limiter(pool: PgPool) { + let mut state = test_state(pool).await; + // Short window so a single recorded hit is already expired at sweep time. + state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(5, Duration::from_millis(50)); + + assert!( + state.ipfs_rate_limiter.check("1.2.3.4").await, + "record a hit on the ipfs limiter" + ); + assert_eq!( + state.ipfs_rate_limiter.tracked_keys().await, + 1, + "the source-IP key is tracked before the sweep" + ); + // Expire the entry (still mapped — cleanup hasn't run), then sweep. + tokio::time::sleep(Duration::from_millis(60)).await; + state.sweep_rate_limiters().await; + + assert_eq!( + state.ipfs_rate_limiter.tracked_keys().await, + 0, + "the periodic sweep must evict the ipfs limiter's expired entries" + ); + } + + /// U5 (R6, KTD6), the observed defect: the `/ipfs` route rate limit and the + /// resolver's per-probe WORK budget are SEPARATE buckets, so a single request with + /// one probe COMPLETES even at route limit = 1. Through the production router the + /// `rate_limit_by_ip` middleware charges `ipfs_rate_limiter` once (its 1-slot bucket + /// is now full); the handler's legacy pre-scan peek and per-probe charge then draw + /// from `ipfs_work_rate_limiter`, a different bucket, so the walk-free public copy + /// still serves 200. RED before the split (both charges on `ipfs_rate_limiter`): the + /// middleware fills the one slot, the pre-scan peek reads it throttled, nothing is + /// servable → 429 on the FIRST request. Trust None so the middleware and the handler + /// resolve the same `ConnectInfo` peer IP. + #[sqlx::test] + async fn ipfs_route_limit_1_still_serves_one_probe(pool: PgPool) { + use gitlawb_core::identity::Keypair; let owner = Keypair::generate(); let owner_did = owner.did().to_string(); let slug = owner_did.replace([':', '/'], "_"); let short = owner_did.split(':').next_back().unwrap().to_string(); - let state = test_state(pool).await; - - // Seed a normal repo with `secret/b.txt` reachable from HEAD, so the - // path-scoped rule has something to match — without this the rule has - // no anchor and we'd be testing nothing. - let _fx = seed_cid_repos(&slug, &short, &["dangling"]); + let mut state = test_state(pool).await; + state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(600, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + // Public, no-rule legacy pin (NULL provenance) → the resolver takes the scan + // fallback and serves walk-free (exactly one probe). + let fx = seed_cid_repos(&slug, &short, &["routeone"]); let bare = std::path::PathBuf::from("/tmp") .join(&slug) - .join("dangling.git"); + .join("routeone.git"); + let repo = seed_repo(&owner_did, "routeone"); + state.db.create_repo(&repo).await.expect("seed repo"); + let cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; - // Write a dangling blob: `git hash-object -w --stdin` adds it to the - // object DB but nothing references it, so the reachable walk never - // enumerates it. - let mut cmd = std::process::Command::new("git"); - cmd.args(["hash-object", "-w", "--stdin"]) - .current_dir(&bare) - .stdin(std::process::Stdio::piped()) - .stdout(std::process::Stdio::piped()); - let mut child = cmd.spawn().expect("spawn git hash-object"); - { - use std::io::Write; - let stdin = child.stdin.as_mut().expect("stdin"); - stdin.write_all(b"DANGLING SECRET\n").expect("write stdin"); - } - let out = child.wait_with_output().expect("hash-object output"); - assert!( - out.status.success(), - "git hash-object: {}", - String::from_utf8_lossy(&out.stderr) - ); - let dangling_oid = String::from_utf8_lossy(&out.stdout).trim().to_string(); - // Sanity: must be a 64-hex sha256 oid, since the repo is sha256-format. + let router = crate::server::build_router(state); + let peer: std::net::SocketAddr = "203.0.113.7:5000".parse().unwrap(); + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut() + .insert(axum::extract::ConnectInfo(peer)); + let resp = router.oneshot(req).await.unwrap(); assert_eq!( - dangling_oid.len(), - 64, - "expected sha256 oid: {dangling_oid}" + resp.status(), + StatusCode::OK, + "a single /ipfs request with one probe must serve even at route limit = 1 \ + (the route brake and the resolver's work budget are separate buckets)" ); - let dangling_cid = cid_for_oid(&dangling_oid); + } + /// U5 (R6): the two buckets are independent — the WORK budget can be exhausted + /// (429) WITHOUT draining the ROUTE bucket. Through the production router, route + /// generous (5) but work tight (1): one request drives two legacy probes, so the + /// second probe finds the work bucket spent → 429 (the route middleware admitted it). + /// The route bucket, charged once by the middleware, still has room afterward — the + /// work charges never touched it, so it admits four more direct checks. + #[sqlx::test] + async fn ipfs_work_exhaustion_leaves_route_bucket_intact(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(5, Duration::from_secs(3600)); + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + // A legacy pin absent from every repo so the scan probes both seeded repos: two + // probes, work budget 1 → the second probe is shed → 429. + let names = ["we0", "we1"]; + let _fx = seed_cid_repos(&slug, &short, &names); + for n in names { + state + .db + .create_repo(&seed_repo(&owner_did, n)) + .await + .expect("seed repo"); + } + let bogus_oid = "0".repeat(64); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(b"work-exhaustion").to_string(); state .db - .create_repo(&seed_repo(&owner_did, "dangling")) - .await - .expect("seed repo"); - let rec = state - .db - .get_repo(&owner_did, "dangling") - .await - .unwrap() - .unwrap(); - // Path-scoped rule triggers the per-blob allowed-set gate (KTD4). - state - .db - .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .record_pinned_cid(&bogus_oid, &cid, None) .await - .expect("deny rule"); + .expect("legacy pin"); - // anon: the dangling blob is absent from the reachable allowed-set → - // 404, no leak. Pre-#126 (deny-set) would serve 200. - let (st, body) = cid_parts( - cid_router(&state) - .oneshot(cid_anon(&dangling_cid)) - .await - .unwrap(), - ) - .await; + let route_bucket = state.ipfs_rate_limiter.clone(); + let peer_ip = "203.0.113.8"; + let peer: std::net::SocketAddr = format!("{peer_ip}:5000").parse().unwrap(); + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut() + .insert(axum::extract::ConnectInfo(peer)); + let resp = router.oneshot(req).await.unwrap(); assert_eq!( - st, - StatusCode::NOT_FOUND, - "dangling blob must 404 under path-scoped rules" + resp.status(), + StatusCode::TOO_MANY_REQUESTS, + "a request whose probes exceed the work budget is shed 429 (work bucket), \ + not blocked at the route (route bucket generous)" ); + // The route bucket recorded only the single request the middleware charged; the + // work charges did not drain it. Sized 5, one used by the request → four left. + for i in 0..4 { + assert!( + route_bucket.check(peer_ip).await, + "route check {i} must still admit — work charges never drained the route bucket" + ); + } + } + + /// U5 (R6): the periodic cleanup task sweeps the NEW work-budget limiter too, not + /// only the route limiter and its siblings. Mirrors + /// `sweep_rate_limiters_includes_ipfs_limiter`: drive `sweep_rate_limiters` and + /// assert the work limiter's expired entry is evicted. Dropping the + /// `ipfs_work_rate_limiter.cleanup()` call from that method leaves the entry in place + /// (`tracked_keys` stays 1): the RED proof the sweep covers it. + #[sqlx::test] + async fn sweep_rate_limiters_includes_ipfs_work_limiter(pool: PgPool) { + let mut state = test_state(pool).await; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(5, Duration::from_millis(50)); + assert!( - !body.contains("DANGLING SECRET"), - "404 body must not leak the dangling content" + state.ipfs_work_rate_limiter.check("1.2.3.4").await, + "record a hit on the work limiter" + ); + assert_eq!( + state.ipfs_work_rate_limiter.tracked_keys().await, + 1, + "the source-IP key is tracked before the sweep" ); - // owner (signed): same 404. The dangling blob has no path, so it's - // never visibility-checked → never in the allowed set, even for the - // owner. This is the accepted fail-closed shift documented in the PR. - let (st, body) = cid_parts( - cid_router(&state) - .oneshot(cid_signed(&owner, &dangling_cid)) - .await - .unwrap(), - ) - .await; + tokio::time::sleep(Duration::from_millis(60)).await; + state.sweep_rate_limiters().await; + assert_eq!( - st, - StatusCode::NOT_FOUND, - "owner also 404s on dangling blobs under path-scoped rules (fail-closed default)" + state.ipfs_work_rate_limiter.tracked_keys().await, + 0, + "the periodic sweep must evict the work limiter's expired entries" ); - assert!(!body.contains("DANGLING SECRET")); } // --------------------------------------------------------------------------- @@ -5428,4 +15481,1039 @@ mod tests { "result includes the deep cert matching the prefix" ); } + + /// Coalesced-drain behavior of the detached post-push encrypt/pin task. + /// + /// A push arriving while a task is in flight does not spawn a second task; its + /// (old_sha, new_sha) tip pairs are merged into the in-flight key's pending slot + /// and the task loop-drains them before releasing the key. These tests drive the + /// real task through `run_encrypt_pin_task_for_test` and assert on the WORK + /// PERFORMED (what is pinned, what is sealed, whether the key is released), not + /// on control flow. The drain re-reads repo state FRESH, so a rule tightened + /// between the coalesced push and its drain must be honored, fail closed. + mod u3_requeue { + use super::*; + use crate::db::VisibilityMode; + use crate::state::{BeginOutcome, PendingWork}; + use std::path::{Path, PathBuf}; + use std::process::Command; + + fn git(args: &[&str], dir: &Path) { + let ok = Command::new("git") + .args(args) + .current_dir(dir) + .status() + .unwrap() + .success(); + assert!(ok, "git {args:?} failed"); + } + fn oid(rev: &str, dir: &Path) -> String { + let out = Command::new("git") + .args(["rev-parse", rev]) + .current_dir(dir) + .output() + .unwrap(); + assert!(out.status.success(), "rev-parse {rev}: {out:?}"); + String::from_utf8_lossy(&out.stdout).trim().to_string() + } + struct Repo { + _td: tempfile::TempDir, + path: PathBuf, + } + fn init_repo() -> Repo { + let td = tempfile::TempDir::new().unwrap(); + let path = td.path().to_path_buf(); + git(&["init", "-q"], &path); + git(&["config", "user.email", "t@t"], &path); + git(&["config", "user.name", "t"], &path); + Repo { _td: td, path } + } + /// Commit `content` at `rel`, return the blob oid. + fn commit(repo: &Path, rel: &str, content: &str) -> String { + let full = repo.join(rel); + std::fs::create_dir_all(full.parent().unwrap()).unwrap(); + std::fs::write(&full, content).unwrap(); + git(&["add", "."], repo); + git(&["commit", "-qm", rel], repo); + oid(&format!("HEAD:{rel}"), repo) + } + /// Write a loose, UNREACHABLE blob (dangling object). + fn write_dangling_blob(repo: &Path, content: &str) -> String { + let out = Command::new("git") + .args(["hash-object", "-w", "--stdin"]) + .current_dir(repo) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .spawn() + .unwrap(); + use std::io::Write; + out.stdin + .as_ref() + .unwrap() + .write_all(content.as_bytes()) + .unwrap(); + let o = out.wait_with_output().unwrap(); + assert!(o.status.success()); + String::from_utf8_lossy(&o.stdout).trim().to_string() + } + fn new_did() -> String { + Keypair::generate().did().to_string() + } + /// Admit push A on the in-flight key, or fail the test. + fn admit(state: &AppState, key: &str) -> crate::state::EncryptInflightGuard { + match state.encrypt_inflight.try_begin(key, Vec::new()) { + BeginOutcome::Admitted(g) => g, + BeginOutcome::Coalesced => panic!("push A must be admitted, nothing is in flight"), + } + } + /// Coalesce push B's tip pairs into the in-flight key, or fail the test. + fn coalesce(state: &AppState, key: &str, pairs: Vec<(String, String)>) { + match state.encrypt_inflight.try_begin(key, pairs) { + BeginOutcome::Coalesced => {} + BeginOutcome::Admitted(_) => { + panic!("push B must coalesce, a task is already in flight") + } + } + } + + /// SCENARIO 2 + 5 (pin half, TAIL-PLACEMENT guard). A coalesced push on a PUBLIC + /// repo with NO path-scoped rule must still drain its pin half: the second + /// push's new object is pinned after the task. RED without the drain (the stale + /// spawn object_list never lists obj2), and RED if the drain sits inside the + /// `has_path_scoped_rule` block (a rules-free repo would never reach it). + #[sqlx::test] + async fn u3_rules_free_public_repo_requeues_pin_half(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u3-pin"); + state.db.create_repo(&repo).await.expect("seed repo"); + let key = crate::state::repo_identity_key(&owner, &repo.name); + let git_repo = init_repo(); + let obj1 = commit(&git_repo.path, "a.txt", "one\n"); + let tip_a = oid("HEAD", &git_repo.path); + // The coalesced push B adds obj2 (present at drain time, NOT in the stale + // push-A spawn object_list). + let obj2 = commit(&git_repo.path, "b.txt", "two\n"); + let tip_b = oid("HEAD", &git_repo.path); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // Push A admits (guard); push B coalesces its tip pair into the slot. + let guard = admit(&state, &key); + coalesce(&state, &key, vec![(tip_a, tip_b)]); + + // Spawn-time (push A) captures are STALE: object_list lists only obj1, no rule. + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + owner.clone(), + repo.name.clone(), + server.url(), + vec![obj1.clone()], + Some(vec![]), + true, + ) + .await; + + assert!( + state.db.is_pinned(&obj1).await.unwrap(), + "push A's object is pinned on the first pass" + ); + assert!( + state.db.is_pinned(&obj2).await.unwrap(), + "the coalesced push's new object is pinned by the DRAIN lap (RED without \ + the drain, or if the drain sits inside the encrypt gate)" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the guard key is released once the task exits clean" + ); + } + + /// SCENARIO 1 + 3 (encrypt half, FRESH re-read). A coalesced push adds a + /// path-scoped rule withholding a blob. The task must re-read rules FRESH on + /// the drain lap and seal the newly-withheld blob's recovery copy. RED without + /// the fresh read (pass one's stale empty rule set seals nothing). + #[sqlx::test] + async fn u3_requeue_seals_blob_withheld_by_coalesced_rule_change(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let reader = new_did(); + let repo = seed_repo(&owner, "u3-enc"); + state.db.create_repo(&repo).await.expect("seed repo"); + let key = crate::state::repo_identity_key(&owner, &repo.name); + let git_repo = init_repo(); + let _pub_oid = commit(&git_repo.path, "public/a.txt", "public\n"); + let tip_a = oid("HEAD", &git_repo.path); + let secret_oid = commit(&git_repo.path, "secret/b.txt", "TOP SECRET\n"); + let tip_b = oid("HEAD", &git_repo.path); + + // Coalesced push B changes .gitlawb: withhold /secret/** from anon, grant reader. + state + .db + .set_visibility_rule(&repo.id, "/secret/**", VisibilityMode::B, &[reader], &owner) + .await + .expect("set rule"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + let guard = admit(&state, &key); + coalesce(&state, &key, vec![(tip_a, tip_b)]); + + // Push A captures are STALE: no rule, public repo. + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + owner.clone(), + repo.name.clone(), + server.url(), + vec![], + Some(vec![]), + true, + ) + .await; + + assert!( + state + .db + .encrypted_blob_recipients_tag(&repo.id, &secret_oid) + .await + .unwrap() + .is_some(), + "the coalesced push's newly-withheld blob is sealed after the DRAIN re-read \ + (RED without the fresh read: pass one's stale empty rules seal nothing)" + ); + assert!(state.encrypt_inflight.is_empty(), "guard key released"); + } + + /// SCENARIO 4 (visibility-leak negative). The drain's full scan must feed + /// `list_all_objects` through the fail-closed filter, never pin it bare: a + /// withheld secret blob and a dangling blob must NOT land in the public pin set. + /// + /// The full scan is forced through the public API: one coalescing push carrying + /// more than the pending tip-pair cap degrades the slot to `PendingWork::FullScan`, + /// which is also the overflow path itself. + #[sqlx::test] + async fn u3_requeue_full_scan_does_not_publicly_pin_withheld_or_dangling(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let reader = new_did(); + let repo = seed_repo(&owner, "u3-leak"); + state.db.create_repo(&repo).await.expect("seed repo"); + let key = crate::state::repo_identity_key(&owner, &repo.name); + let git_repo = init_repo(); + let pub_oid = commit(&git_repo.path, "public/a.txt", "public\n"); + let secret_oid = commit(&git_repo.path, "secret/b.txt", "TOP SECRET\n"); + state + .db + .set_visibility_rule(&repo.id, "/secret/**", VisibilityMode::B, &[reader], &owner) + .await + .expect("set rule"); + // Coalesced push adds a new public object and a dangling blob. + let new_pub_oid = commit(&git_repo.path, "public/c.txt", "more public\n"); + let tip = oid("HEAD", &git_repo.path); + let dangling_oid = write_dangling_blob(&git_repo.path, "orphan bytes\n"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + let rules = state.db.list_visibility_rules(&repo.id).await.unwrap(); + + let guard = admit(&state, &key); + // 1025 pairs is one past the pending cap, so the slot degrades to FullScan. + coalesce(&state, &key, vec![(tip.clone(), tip.clone()); 1025]); + assert_eq!( + state.encrypt_inflight.pending_for(&key), + Some(PendingWork::FullScan), + "an overflowing coalesce degrades the pending slot to a forced full scan" + ); + + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + owner.clone(), + repo.name.clone(), + server.url(), + vec![pub_oid.clone()], + Some(rules), + true, + ) + .await; + + assert!( + state.db.is_pinned(&new_pub_oid).await.unwrap(), + "the coalesced push's new PUBLIC object is pinned by the drain full scan" + ); + assert!( + !state.db.is_pinned(&secret_oid).await.unwrap(), + "a WITHHELD blob is never publicly pinned by the drain enumeration (leak guard)" + ); + assert!( + !state.db.is_pinned(&dangling_oid).await.unwrap(), + "a DANGLING blob is never publicly pinned by the drain enumeration (leak guard)" + ); + // The withheld blob still gets its ENCRYPTED recovery copy (not a public pin). + assert!( + state + .db + .encrypted_blob_recipients_tag(&repo.id, &secret_oid) + .await + .unwrap() + .is_some(), + "withheld blob is sealed as an encrypted recovery copy, not pinned in the clear" + ); + } + + /// SCENARIO 8 (no-coalesce happy path). A single push with no coalesced follower + /// runs exactly one pass, pins its object, and releases the key. No drain lap. + #[sqlx::test] + async fn u3_no_coalesce_single_pass_pins_and_releases(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u3-happy"); + state.db.create_repo(&repo).await.expect("seed repo"); + let key = crate::state::repo_identity_key(&owner, &repo.name); + let git_repo = init_repo(); + let obj1 = commit(&git_repo.path, "a.txt", "one\n"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // No second try_begin: nothing is ever merged into the pending slot. + let guard = admit(&state, &key); + assert_eq!( + state.encrypt_inflight.pending_for(&key), + Some(PendingWork::Tips(vec![])), + "clean, no coalesce" + ); + + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + owner.clone(), + repo.name.clone(), + server.url(), + vec![obj1.clone()], + Some(vec![]), + true, + ) + .await; + + assert!( + state.db.is_pinned(&obj1).await.unwrap(), + "the single push's object is pinned" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the key is released after one pass" + ); + } + + mod u2_reread_retry { + use super::*; + use crate::api::repos::drain_faults; + + /// Process-wide tracing capture so a test can assert the give-up is logged at + /// ERROR. A global default subscriber can only be installed once per process, + /// so it is shared by every test here and assertions filter on the repo id, + /// which is a fresh uuid per test. + mod logcap { + use std::sync::{Arc, Mutex, OnceLock}; + use tracing::{Event, Level, Subscriber}; + use tracing_subscriber::layer::{Context, Layer}; + use tracing_subscriber::prelude::*; + + type Lines = Arc>>; + + fn lines() -> &'static Lines { + static LINES: OnceLock = OnceLock::new(); + LINES.get_or_init(|| Arc::new(Mutex::new(Vec::new()))) + } + + struct Capture; + impl Layer for Capture { + fn on_event(&self, event: &Event<'_>, _ctx: Context<'_, S>) { + struct V(String); + impl tracing::field::Visit for V { + fn record_debug( + &mut self, + field: &tracing::field::Field, + value: &dyn std::fmt::Debug, + ) { + self.0.push_str(&format!(" {}={:?}", field.name(), value)); + } + } + let mut v = V(String::new()); + event.record(&mut v); + lines() + .lock() + .unwrap() + .push((*event.metadata().level(), v.0)); + } + } + + pub(super) fn install() { + static ONCE: OnceLock<()> = OnceLock::new(); + ONCE.get_or_init(|| { + let _ = tracing::subscriber::set_global_default( + tracing_subscriber::registry().with(Capture), + ); + }); + } + + pub(super) fn errors_containing(needle: &str) -> Vec { + lines() + .lock() + .unwrap() + .iter() + .filter(|(lvl, msg)| *lvl == Level::ERROR && msg.contains(needle)) + .map(|(_, msg)| msg.clone()) + .collect() + } + } + + /// SCENARIO 1. The repo re-read fails once, then succeeds: the drain lap + /// must still RUN, under the refreshed state, and pin the coalesced push's + /// object. RED before the fix (the single `Err` returned `None`, the lap + /// pinned nothing, and `finish_or_take_pending` had already taken the + /// pending work out of the slot, so it was gone for good). + #[sqlx::test] + async fn u2_transient_repo_reread_failure_is_retried_and_work_lands(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u2-retry"); + state.db.create_repo(&repo).await.expect("seed repo"); + let key = crate::state::repo_identity_key(&owner, &repo.name); + let git_repo = init_repo(); + let obj1 = commit(&git_repo.path, "a.txt", "one\n"); + let tip_a = oid("HEAD", &git_repo.path); + // The coalesced push B adds obj2, absent from push A's spawn captures. + let obj2 = commit(&git_repo.path, "b.txt", "two\n"); + let tip_b = oid("HEAD", &git_repo.path); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // One transient repo re-read failure, then the real DB answers. + drain_faults::inject(&repo.id, 1, 0); + + let guard = admit(&state, &key); + coalesce(&state, &key, vec![(tip_a, tip_b)]); + + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + owner.clone(), + repo.name.clone(), + server.url(), + vec![obj1.clone()], + Some(vec![]), + true, + ) + .await; + + assert!( + state.db.is_pinned(&obj2).await.unwrap(), + "the coalesced push's object is pinned after the retried re-read (RED \ + before this unit: the Err arm dropped the lap and the work with it)" + ); + let c = drain_faults::counters(&repo.id); + assert_eq!( + c.repo_read_attempts, 2, + "the failed re-read is retried exactly once before it succeeds" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the guard key is released once the task exits" + ); + } + + /// SCENARIO 2. Every re-read attempt fails: the loop must give up on a BOUND + /// (asserted as a literal, so raising or removing the bound goes RED) and log + /// the give-up at ERROR so the residual loss is observable rather than silent. + #[sqlx::test] + async fn u2_sustained_repo_reread_failure_is_bounded_and_logged(pool: PgPool) { + logcap::install(); + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u2-bounded"); + state.db.create_repo(&repo).await.expect("seed repo"); + let key = crate::state::repo_identity_key(&owner, &repo.name); + let git_repo = init_repo(); + let obj1 = commit(&git_repo.path, "a.txt", "one\n"); + let tip_a = oid("HEAD", &git_repo.path); + let obj2 = commit(&git_repo.path, "b.txt", "two\n"); + let tip_b = oid("HEAD", &git_repo.path); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // Far more failures than the bound allows: the outage never clears. + drain_faults::inject(&repo.id, 10_000, 0); + + let guard = admit(&state, &key); + coalesce(&state, &key, vec![(tip_a, tip_b)]); + + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + owner.clone(), + repo.name.clone(), + server.url(), + vec![obj1.clone()], + Some(vec![]), + true, + ) + .await; + + let c = drain_faults::counters(&repo.id); + assert_eq!( + c.repo_read_attempts, 3, + "the re-read is bounded at 3 attempts; unbounded retry or a raised \ + bound must fail here" + ); + assert!( + !state.db.is_pinned(&obj2).await.unwrap(), + "with the read never succeeding there is nothing fresh to act on" + ); + let errs = logcap::errors_containing(&repo.id); + assert!( + !errs.is_empty(), + "the exhausted drain re-read is logged at ERROR with the repo id, so \ + the residual work loss is observable; captured: {errs:?}" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the guard key is still released on the give-up path" + ); + } + + /// SCENARIO 3. `Ok(None)` (the repo was deleted during the in-flight window) + /// is NOT a transient failure: it must release immediately without burning the + /// retry budget. The repo row is never created, so the re-read legitimately + /// returns `Ok(None)`. + #[sqlx::test] + async fn u2_repo_gone_releases_without_consuming_retries(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let missing_id = uuid::Uuid::new_v4().to_string(); + let missing_name = "u2-gone".to_string(); + let key = crate::state::repo_identity_key(&owner, &missing_name); + let git_repo = init_repo(); + let _obj1 = commit(&git_repo.path, "a.txt", "one\n"); + let tip_a = oid("HEAD", &git_repo.path); + let _obj2 = commit(&git_repo.path, "b.txt", "two\n"); + let tip_b = oid("HEAD", &git_repo.path); + + let server = mockito::Server::new_async().await; + + drain_faults::inject(&missing_id, 0, 0); + + let guard = admit(&state, &key); + // A real pair, so a drain lap actually runs and reaches the re-read. + coalesce(&state, &key, vec![(tip_a, tip_b)]); + + // Empty object list: pass one touches no pin rows for a repo that is gone. + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + missing_id.clone(), + owner.clone(), + missing_name.clone(), + server.url(), + vec![], + Some(vec![]), + true, + ) + .await; + + let c = drain_faults::counters(&missing_id); + assert_eq!( + c.repo_read_attempts, 1, + "a deleted repo is a terminal answer, never retried" + ); + assert_eq!( + c.rules_read_attempts, 0, + "no rules read is attempted once the repo row is gone" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the guard key is released cleanly" + ); + } + + /// SCENARIO 4. A failed visibility-rule read is transient, never an empty + /// policy. RED before the fix, where `.ok()` made "the rules read failed" and + /// "this repo has no rules" the same value: the withheld blob was then neither + /// sealed nor covered, because a `None` rule set skips the entire lap. + #[sqlx::test] + async fn u2_transient_rules_read_failure_is_retried_not_read_as_empty(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let reader = new_did(); + let repo = seed_repo(&owner, "u2-rules"); + state.db.create_repo(&repo).await.expect("seed repo"); + let key = crate::state::repo_identity_key(&owner, &repo.name); + let git_repo = init_repo(); + let pub_oid = commit(&git_repo.path, "public/a.txt", "public\n"); + let tip_a = oid("HEAD", &git_repo.path); + let secret_oid = commit(&git_repo.path, "secret/b.txt", "TOP SECRET\n"); + let tip_b = oid("HEAD", &git_repo.path); + + // The coalesced push B is what added the path-scoped rule. + state + .db + .set_visibility_rule( + &repo.id, + "/secret/**", + VisibilityMode::B, + &[reader], + &owner, + ) + .await + .expect("set rule"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // The repo row reads fine; the RULES read is the one that blips. + drain_faults::inject(&repo.id, 0, 1); + + let guard = admit(&state, &key); + coalesce(&state, &key, vec![(tip_a, tip_b)]); + + // Push A's captures are stale: no rule, nothing withheld. + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + owner.clone(), + repo.name.clone(), + server.url(), + vec![pub_oid.clone()], + Some(vec![]), + true, + ) + .await; + + assert!( + state + .db + .encrypted_blob_recipients_tag(&repo.id, &secret_oid) + .await + .unwrap() + .is_some(), + "the withheld blob is sealed under the RETRIED rule set (RED with \ + list_visibility_rules(..).ok(): an empty policy seals nothing)" + ); + let c = drain_faults::counters(&repo.id); + assert_eq!( + c.rules_read_attempts, 2, + "the failed rules read is retried, not collapsed into an empty rule set" + ); + assert!( + !state.db.is_pinned(&secret_oid).await.unwrap(), + "the withheld blob is never pinned in the clear by the drain" + ); + } + + /// SCENARIO 5. The fault-free control for scenario 4: the rules applied by the + /// drain are the COALESCED push's fresh ones, never the spawn-time capture, + /// and the retry path does not perturb that (exactly one read of each). + #[sqlx::test] + async fn u2_requeue_applies_fresh_rules_not_spawn_captures(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let reader = new_did(); + let repo = seed_repo(&owner, "u2-fresh"); + state.db.create_repo(&repo).await.expect("seed repo"); + let key = crate::state::repo_identity_key(&owner, &repo.name); + let git_repo = init_repo(); + let pub_oid = commit(&git_repo.path, "public/a.txt", "public\n"); + let tip_a = oid("HEAD", &git_repo.path); + let secret_oid = commit(&git_repo.path, "secret/b.txt", "TOP SECRET\n"); + let tip_b = oid("HEAD", &git_repo.path); + state + .db + .set_visibility_rule( + &repo.id, + "/secret/**", + VisibilityMode::B, + &[reader], + &owner, + ) + .await + .expect("set rule"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + drain_faults::inject(&repo.id, 0, 0); + + let guard = admit(&state, &key); + coalesce(&state, &key, vec![(tip_a, tip_b)]); + + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + owner.clone(), + repo.name.clone(), + server.url(), + vec![pub_oid.clone()], + Some(vec![]), + true, + ) + .await; + + let c = drain_faults::counters(&repo.id); + assert_eq!( + (c.repo_read_attempts, c.rules_read_attempts), + (1, 1), + "a healthy DB is read exactly once per drain lap" + ); + assert!( + state.db.is_pinned(&pub_oid).await.unwrap(), + "the visible object is pinned under the fresh rules" + ); + assert!( + !state.db.is_pinned(&secret_oid).await.unwrap(), + "the freshly-read rule withholds the secret blob (the spawn-time \ + capture had no rules at all)" + ); + } + + /// SCENARIO 6. Regression guard on the property the fix must not disturb: the + /// finish-or-take critical section is atomic, so a push coalescing during it is + /// still covered by exactly one more lap, and the key is released after. + #[sqlx::test] + async fn u2_coalesced_push_still_covered_by_exactly_one_requeue_pass(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u2-coalesce"); + state.db.create_repo(&repo).await.expect("seed repo"); + let key = crate::state::repo_identity_key(&owner, &repo.name); + let git_repo = init_repo(); + let obj1 = commit(&git_repo.path, "a.txt", "one\n"); + let tip_a = oid("HEAD", &git_repo.path); + let obj2 = commit(&git_repo.path, "b.txt", "two\n"); + let tip_b = oid("HEAD", &git_repo.path); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + drain_faults::inject(&repo.id, 0, 0); + + let guard = admit(&state, &key); + // Push B lands during the in-flight window: its tip pair is merged. + coalesce(&state, &key, vec![(tip_a.clone(), tip_b.clone())]); + assert_eq!( + state.encrypt_inflight.pending_for(&key), + Some(PendingWork::Tips(vec![(tip_a, tip_b)])), + "the coalesced push recorded its work in the pending slot" + ); + + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + owner.clone(), + repo.name.clone(), + server.url(), + vec![obj1.clone()], + Some(vec![]), + true, + ) + .await; + + assert_eq!( + drain_faults::counters(&repo.id).repo_read_attempts, + 1, + "one coalesced push means exactly one drain lap, no re-spin" + ); + assert!( + state.db.is_pinned(&obj1).await.unwrap(), + "push A's object is pinned" + ); + assert!( + state.db.is_pinned(&obj2).await.unwrap(), + "the coalesced push's object is covered by the drain lap" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the key is released once the task is clean" + ); + } + + /// Wait for `finish_or_take_pending` to take the pending work out of the slot + /// (`Tips(nonempty)` -> `Tips(empty)`), which is the exact instant the task + /// enters `drain_refresh_state`'s retry window. Deterministic, so the + /// coalescing push below lands INSIDE that window rather than on a sleep + /// guess. `None` means the key is already gone (the task exited), which the + /// caller reports as its own failure. + async fn wait_for_drain_window( + inflight: &crate::state::EncryptInflight, + key: &str, + ) -> bool { + for _ in 0..5_000 { + match inflight.pending_for(key) { + Some(PendingWork::Tips(acc)) if acc.is_empty() => return true, + None => return false, + Some(_) => {} + } + tokio::time::sleep(std::time::Duration::from_millis(1)).await; + } + false + } + + /// SCENARIO 7 (RED-before/GREEN-after). A push that coalesces WHILE the + /// re-read is retrying must not be thrown away when that re-read finally + /// gives up. Breaking the drain loop on the give-up would let + /// `EncryptInflightGuard::drop` remove the key with push C's work still + /// recorded, and push C's lap would never run: a silent drop with no + /// reconciliation sweep behind it. + /// + /// Exactly `DRAIN_REREAD_MAX_ATTEMPTS` injected repo-read faults, so the + /// first refresh exhausts its budget and the DB is healthy for the next one. + /// Push C coalesces inside that window. + #[sqlx::test] + async fn u2_failed_reread_keeps_a_push_that_coalesced_during_the_window(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u2-window"); + state.db.create_repo(&repo).await.expect("seed repo"); + let key = crate::state::repo_identity_key(&owner, &repo.name); + let git_repo = init_repo(); + let obj_a = commit(&git_repo.path, "a.txt", "one\n"); + let tip_a = oid("HEAD", &git_repo.path); + let _obj_b = commit(&git_repo.path, "b.txt", "two\n"); + let tip_b = oid("HEAD", &git_repo.path); + let obj_c = commit(&git_repo.path, "c.txt", "three\n"); + let tip_c = oid("HEAD", &git_repo.path); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // Exactly the bound: the FIRST refresh burns all three attempts and gives + // up; every later refresh sees a healthy DB. + drain_faults::inject(&repo.id, 3, 0); + + let guard = admit(&state, &key); + coalesce(&state, &key, vec![(tip_a.clone(), tip_b.clone())]); + + // Push C lands during the retry window, after the loop already took push + // B's pending work out of the slot. It MUST carry a real tip pair: an + // empty merge leaves the slot empty and no extra lap runs at all. + let inflight = state.encrypt_inflight.clone(); + let watch_key = key.clone(); + let coalesced = tokio::spawn(async move { + if !wait_for_drain_window(&inflight, &watch_key).await { + return false; + } + matches!( + inflight.try_begin(&watch_key, vec![(tip_b, tip_c)]), + BeginOutcome::Coalesced + ) + }); + + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + owner.clone(), + repo.name.clone(), + server.url(), + vec![obj_a.clone()], + Some(vec![]), + true, + ) + .await; + + assert!( + coalesced.await.expect("coalescing task"), + "push C must have coalesced inside the retry window for this test to \ + mean anything" + ); + assert!( + state.db.is_pinned(&obj_c).await.unwrap(), + "the push that coalesced during the retry window must still get a lap \ + once the DB recovers (RED if the give-up breaks the loop: the pending \ + work was already taken, so the lap was dropped with nothing to \ + re-derive it)" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the guard key is released once the task exits" + ); + } + + /// SCENARIO 8 (the sustained-outage guard on the fall-through). Continuing the + /// loop on a give-up means `finish_or_take_pending` runs again, so a DB that + /// never recovers must still TERMINATE rather than spin. It does: an extra lap + /// only happens when a push actually coalesced, and each lap pays a full + /// bounded re-read (3 attempts with backoff). One coalescing push during the + /// window buys exactly one extra lap: 6 repo-read attempts, then exit. + #[sqlx::test] + async fn u2_sustained_failure_with_a_coalesce_terminates_after_one_more_lap( + pool: PgPool, + ) { + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u2-sustained-window"); + state.db.create_repo(&repo).await.expect("seed repo"); + let key = crate::state::repo_identity_key(&owner, &repo.name); + let git_repo = init_repo(); + let obj_a = commit(&git_repo.path, "a.txt", "one\n"); + let tip_a = oid("HEAD", &git_repo.path); + let _obj_b = commit(&git_repo.path, "b.txt", "two\n"); + let tip_b = oid("HEAD", &git_repo.path); + let _obj_c = commit(&git_repo.path, "c.txt", "three\n"); + let tip_c = oid("HEAD", &git_repo.path); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // The outage never clears. + drain_faults::inject(&repo.id, 10_000, 0); + + let guard = admit(&state, &key); + coalesce(&state, &key, vec![(tip_a.clone(), tip_b.clone())]); + + let inflight = state.encrypt_inflight.clone(); + let watch_key = key.clone(); + let coalesced = tokio::spawn(async move { + if !wait_for_drain_window(&inflight, &watch_key).await { + return false; + } + matches!( + inflight.try_begin(&watch_key, vec![(tip_b, tip_c)]), + BeginOutcome::Coalesced + ) + }); + + // The watchdog is the real assertion: a loop that re-spins without the + // pending gate would never return here. + tokio::time::timeout( + std::time::Duration::from_secs(60), + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + owner.clone(), + repo.name.clone(), + server.url(), + vec![obj_a.clone()], + Some(vec![]), + true, + ), + ) + .await + .expect( + "the task must terminate under a sustained outage; a fall-through that \ + does not gate on the pending slot spins forever", + ); + + assert!( + coalesced.await.expect("coalescing task"), + "push C must have coalesced inside the retry window" + ); + assert_eq!( + drain_faults::counters(&repo.id).repo_read_attempts, + 6, + "one coalescing push buys exactly one more bounded re-read lap \ + (3 + 3 attempts), never an unbounded retry" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the guard key is released on the give-up path" + ); + } + } + } } diff --git a/crates/gitlawb-node/src/visibility.rs b/crates/gitlawb-node/src/visibility.rs index 56616872..a8d7ddba 100644 --- a/crates/gitlawb-node/src/visibility.rs +++ b/crates/gitlawb-node/src/visibility.rs @@ -437,6 +437,31 @@ mod tests { ); } + // #135 T1: a Mode-B rule on `/secret/**` must DENY the withheld directory's + // OWN path `/secret` (the `path == prefix` arm), not just strict descendants — + // otherwise get_by_cid's tree gate would serve the /secret tree object and leak + // its children. Pins parity with get_tree, which denies the /secret path. + #[test] + fn subtree_rule_denies_the_withheld_directory_itself() { + let reader = "did:key:z6MkReader"; + let rules = [rule("/secret/**", VisibilityMode::B, &[reader])]; + assert_eq!( + visibility_check(&rules, true, OWNER, None, "/secret"), + Decision::Deny, + "anon denied at the withheld directory's OWN path /secret" + ); + assert_eq!( + visibility_check(&rules, true, OWNER, None, "/public"), + Decision::Allow, + "anon allowed at a sibling path outside the withheld subtree" + ); + assert_eq!( + visibility_check(&rules, true, OWNER, Some(reader), "/secret"), + Decision::Allow, + "listed reader allowed at the withheld directory (caller-aware)" + ); + } + // #153 regression: cross-method DID must still be denied even when the // trailing segment collides with a bare owner key. #[test] diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs index 32e23dd0..48381e3d 100644 --- a/crates/gitlawb-node/tests/inv22_gates.rs +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -29,6 +29,7 @@ fn inv22_concurrency_gates_present_and_not_bypassed() { let repos = src("api/repos.rs"); let smart_http = src("git/smart_http.rs"); let vis = src("git/visibility_pack.rs"); + let ipfs = src("api/ipfs.rs"); // U1 / P1-a: run_bounded_git stands the watchdog down only after confirming the // child actually terminated (WNOWAIT), not on the raw stdout-drain EOF — otherwise @@ -68,6 +69,57 @@ fn inv22_concurrency_gates_present_and_not_bypassed() { withheld_recipients_gated, which acquires git_encrypt_semaphore" ); + // U1 / R2 (#173 round-10): the path-scoped filtered-pack serve must thread the + // caller's AdmissionGuard through BOTH git stages so read + per-caller admission is + // held until the pack-objects group is reaped on disconnect, closing the cap bypass + // the plain upload_pack path already fixed. Two load-bearing markers: rev-list hands + // the disarmed guard back (its tuple return type), and build_filtered_pack forwards + // that guard into the pack-objects stage (the `admission` arg after the + // "pack-objects" label). Reverting either — dropping the guard between stages, or + // passing `None` to pack-objects — trips this. + assert!( + smart_http.contains("Result<(Vec, Option)>") + && smart_http.contains("\"pack-objects\",\n admission,"), + "U1/R2 gate missing: build_filtered_pack must thread the AdmissionGuard through \ + rev-list -> pack-objects so the permits are held until the pack-objects group \ + is reaped on disconnect (the path-scoped half of #174 P1-a)" + ); + + // U2 / R1: a cancelled or timed-out `GET /ipfs/{cid}` must release admission only + // after the blocking work it admitted has finished, not the instant the handler + // future drops (the /ipfs half of #174 P1-a). The mechanism is the shared + // `Arc`: both walk permits are moved into it once per request and a + // clone rides every `spawn_blocking`, so the permits release when the LAST holder + // drops. Reverting to handler-local permits trips this; the per-site clones are + // bound separately by `inv22_ipfs_walk_admission_reaches_every_blocking_site`. + // + // This previously required the permits to be owned by a detached `tokio::spawn` + // running the whole pipeline. That closed the same bypass but kept an abandoned + // request's full legacy scan running against a held slot, so the merge of #173 and + // #174 kept the Arc and dropped the detached task. + assert!( + ipfs.contains("struct WalkAdmission") + && ipfs.contains("let admission = std::sync::Arc::new(WalkAdmission {"), + "U2/R1 gate missing: get_by_cid must move both /ipfs admission permits into a \ + shared Arc whose clones ride the blocking work, so admission is \ + released only once that work completes (the /ipfs half of #174 P1-a)" + ); + + // U2 / KTD2 (#173 round-10): the probe/read children on the /ipfs path must be the + // duration-bounded twins (process-group teardown via run_bounded_git), not the bare + // `store::object_type` / `read_object_content` (or an unbounded `cat-file -s`) a tokio + // timeout cannot cancel — otherwise a wedged cat-file lingers and pins the held + // admission past the deadline. Reverting any twin call site back to a bare read trips + // this. + assert!( + ipfs.contains("object_type_bounded(") + && ipfs.contains("object_size_bounded(") + && ipfs.contains("read_object_content_bounded("), + "U2/KTD2 gate missing: the /ipfs probe+read must call the run_bounded_git-backed \ + *_bounded twins so a wedged cat-file is reaped at the deadline, not left to pin \ + the held /ipfs walk admission" + ); + // P1-e non-bypass tripwire: the bounded recipients walk is spawn_blocking'd nowhere // but inside withheld_recipients_gated. A second call site (count > 1) is a new // detached git walk that skips the admission gate — exactly the class U5 closed. @@ -95,8 +147,12 @@ fn inv22_concurrency_gates_present_and_not_bypassed() { // recovery copies are absent until an unrelated later push. Scan only the // production half of the file — the u5 tests in its `mod tests` also name the // drain call, and matching them would make this check vacuous. + // Split at the TEST MODULE, not at the first `#[cfg(test)]`. api/repos.rs carries + // test-only items (the drain fault seam, the task entry point) ABOVE the production + // code these gates scan for, so splitting on the attribute truncated the production + // half above every line being checked and the gate went vacuously blind. let repos_production = repos - .split("#[cfg(test)]") + .split("\nmod tests {") .next() .expect("split always yields a first chunk"); assert!( @@ -201,10 +257,10 @@ fn f4_release_keeps_conn_owned_until_unlock_resolves() { ); } -/// F6/KTD-5 (initial IPFS metadata queries deadline-wrapped): `get_by_cid` acquires -/// the scarce walk permits (RAII, held for the whole request) BEFORE its two initial -/// metadata queries, and the per-repo loop's first budget gate runs only later. So -/// both `list_all_repos` and `list_visibility_rules_for_repos` must be clamped to the +/// F6/KTD-5 (IPFS metadata queries deadline-wrapped): `get_by_cid` acquires +/// the scarce walk permits (RAII, held for the whole request) BEFORE its metadata +/// queries, and the per-repo loop's first budget gate runs only later. So +/// both `list_repos_page_for_scan` and `list_visibility_rules_for_repos` must be clamped to the /// remaining request budget — otherwise a query blocked in Postgres pins the walk slot /// for the whole stall, past the budget. This scans the PRODUCTION half of `api/ipfs.rs` /// (the `mod tests` half names the same calls in its own harness and would make the @@ -214,31 +270,44 @@ fn f4_release_keeps_conn_owned_until_unlock_resolves() { #[test] fn f6_ipfs_metadata_queries_are_deadline_wrapped() { let ipfs = src("api/ipfs.rs"); + // Split at the TEST MODULE, not at the first `#[cfg(test)]`: the handler carries + // in-line `#[cfg(test)]` query counters, so splitting on the attribute cut the + // production half off above the calls this guard exists to check and the scan went + // vacuously quiet (found 0 of each rather than failing on a missing wrapper). let production = ipfs - .split("#[cfg(test)]") + .split("\nmod tests {") .next() .expect("split always yields a first chunk"); - for call in [".list_all_repos()", ".list_visibility_rules_for_repos("] { - assert_eq!( - production.matches(call).count(), - 1, - "F6 guard stale: `{call}` must appear exactly once in the production half \ - of api/ipfs.rs (the deadline-wrapped handler call) — update this guard" - ); - let idx = production - .find(call) - .unwrap_or_else(|| panic!("F6 gate: api/ipfs.rs no longer calls `{call}`")); - // The wrapper opens a few lines above the call (match tokio::time::timeout( ... - // remaining budget ..., )); a 240-char lookback covers it without - // reaching the previous statement. - let window = &production[idx.saturating_sub(240)..idx]; + // EVERY occurrence must be wrapped, not just the first. The handler now reaches + // these queries from two places (the provenance fast path, one repo at a time, and + // the legacy scan's preload), and an exact-count check would have to be relaxed + // every time a call site is added, which is how a guard quietly stops covering the + // site that matters. Requiring all of them scales with the code instead. + for call in [ + ".list_repos_page_for_scan(", + ".list_visibility_rules_for_repos(", + ".get_repo_by_id(", + ".is_repo_quarantined(", + ] { + let occurrences = production.matches(call).count(); assert!( - window.contains("tokio::time::timeout("), - "F6 gate missing: `{call}` must be wrapped in tokio::time::timeout(...) \ - clamped to the remaining request budget. An unwrapped await pins the held \ - walk permit for the whole DB stall, past GITLAWB_IPFS_REQUEST_BUDGET_SECS." + occurrences >= 1, + "F6 guard stale: api/ipfs.rs no longer calls `{call}` — update this guard" ); + for (n, (idx, _)) in production.match_indices(call).enumerate() { + // The wrapper opens a few lines above the call (match tokio::time::timeout( + // ... remaining budget ..., )); a 240-char lookback covers it + // without reaching the previous statement. + let window = &production[idx.saturating_sub(240)..idx]; + assert!( + window.contains("tokio::time::timeout("), + "F6 gate missing: occurrence {n} of `{call}` (of {occurrences}) is not \ + wrapped in tokio::time::timeout(...) clamped to the remaining request \ + budget. An unwrapped await pins the held walk permit for the whole DB \ + stall, past GITLAWB_IPFS_REQUEST_BUDGET_SECS." + ); + } } } @@ -261,8 +330,12 @@ fn f6_ipfs_metadata_queries_are_deadline_wrapped() { #[test] fn f2_pinata_enqueues_refs_not_retained_object_lists() { let repos = src("api/repos.rs"); + // Split at the TEST MODULE, not at the first `#[cfg(test)]`. api/repos.rs carries + // test-only items (the drain fault seam, the task entry point) ABOVE the production + // code these gates scan for, so splitting on the attribute truncated the production + // half above every line being checked and the gate went vacuously blind. let production = repos - .split("#[cfg(test)]") + .split("\nmod tests {") .next() .expect("split always yields a first chunk"); @@ -321,8 +394,12 @@ fn f2_pinata_enqueues_refs_not_retained_object_lists() { fn f3_second_writer_leased_until_reap() { let repos = src("api/repos.rs"); let smart_http = src("git/smart_http.rs"); + // Split at the TEST MODULE, not at the first `#[cfg(test)]`. api/repos.rs carries + // test-only items (the drain fault seam, the task entry point) ABOVE the production + // code these gates scan for, so splitting on the attribute truncated the production + // half above every line being checked and the gate went vacuously blind. let repos_production = repos - .split("#[cfg(test)]") + .split("\nmod tests {") .next() .expect("split always yields a first chunk"); @@ -372,7 +449,7 @@ fn f3_second_writer_leased_until_reap() { /// exists: it binds all three sites, and any blocking site added to this loop /// later, without reversing that deliberate independence. /// -/// MUTATION (RED): delete any one `Arc::clone(&admission)` binding, or drop the +/// MUTATION (RED): delete any one `Arc::clone(ctx.admission)` binding, or drop the /// clone from inside its closure, and the count falls below three. #[test] fn inv22_ipfs_walk_admission_reaches_every_blocking_site() { @@ -386,7 +463,10 @@ fn inv22_ipfs_walk_admission_reaches_every_blocking_site() { ); // Every blocking site in the scan takes its own clone... - let clones = ipfs.matches("Arc::clone(&admission)").count(); + // `Arc::clone(ctx.admission)` in the gate, `Arc::clone(&admission)` if a site is + // ever added back in the handler body itself; count both spellings. + let clones = ipfs.matches("Arc::clone(ctx.admission)").count() + + ipfs.matches("Arc::clone(&admission)").count(); assert!( clones >= 3, "U1 gate bypassed: expected an admission clone for each of the three \ @@ -443,8 +523,12 @@ fn inv22_ipfs_walk_admission_reaches_every_blocking_site() { fn inv22_replication_tail_spawns_at_the_durability_boundary() { let repos = src("api/repos.rs"); // Production half only — the tests below name these identifiers too. + // Split at the TEST MODULE, not at the first `#[cfg(test)]`. api/repos.rs carries + // test-only items (the drain fault seam, the task entry point) ABOVE the production + // code these gates scan for, so splitting on the attribute truncated the production + // half above every line being checked and the gate went vacuously blind. let production = repos - .split("#[cfg(test)]") + .split("\nmod tests {") .next() .expect("split always yields a first chunk"); @@ -460,7 +544,7 @@ fn inv22_replication_tail_spawns_at_the_durability_boundary() { .find("tokio::spawn(post_receive_replication_tail(") .expect("U5 gate missing: the replication tail must be spawned by git_receive_pack"); let release = production - .find("guard.release(push_succeeded)") + .find(".release(push_succeeded)") .expect("U5 gate stale: release must consume the same success flag as the tail gate"); let touch = production .find("state.db.touch_repo(") diff --git a/crates/gl/Cargo.toml b/crates/gl/Cargo.toml index 3d7ddb9f..2b973a4c 100644 --- a/crates/gl/Cargo.toml +++ b/crates/gl/Cargo.toml @@ -11,7 +11,7 @@ name = "gl" path = "src/main.rs" [dependencies] -gitlawb-core = { path = "../gitlawb-core" } +gitlawb-core = { path = "../gitlawb-core", features = ["redirect"] } icaptcha-client = { path = "../icaptcha-client" } base64 = { workspace = true } tokio = { workspace = true } diff --git a/crates/gl/src/clone.rs b/crates/gl/src/clone.rs index 6754c5e0..926a2d57 100644 --- a/crates/gl/src/clone.rs +++ b/crates/gl/src/clone.rs @@ -12,9 +12,58 @@ use serde::Deserialize; use std::path::Path; use std::process::Command; -use crate::http::NodeClient; +use crate::http::{sanitize_node_msg, NodeClient}; use crate::identity::load_keypair_from_dir; +/// Report why one candidate blob could not be recovered, in the register of the +/// recovery loop's other warnings. +/// +/// Both halves are gateway-supplied text on their way to a terminal: the oid comes +/// out of a manifest fetched from a caller-chosen Arweave gateway, and a transport +/// error carries the URL the same manifest chose. So the whole line is defanged and +/// length-capped, like every other node/gateway string this crate prints. +/// +/// Under `cfg(test)` the line is also mirrored into a per-thread buffer, so a test +/// can assert the loop actually says what it skipped rather than only that it +/// returned nothing. +fn warn_skip(oid: &str, why: &str) { + emit_warning(&format!( + "warning: could not fetch encrypted blob {oid}: {why}; skipping" + )); +} + +/// The same report for a manifest rather than a blob. The manifest is one step +/// earlier in the same recovery: losing it silently means every blob it would have +/// named is missing with no reason given anywhere. +fn warn_skip_manifest(id: &str, why: &str) { + emit_warning(&format!( + "warning: could not fetch blob manifest {id}: {why}; skipping" + )); +} + +/// Sanitize a warning and write it, once. +/// +/// The write goes to [`warn_sink`], which is stderr in a normal build and the tests' +/// per-thread mirror under `cfg(test)`. That indirection is the point: the mirror +/// used to sit BESIDE an `eprintln!`, so deleting the user-visible write left every +/// assertion on the mirror green and the shipped behaviour could be removed with the +/// tests unchanged. There is one write now, and the tests observe it. +fn emit_warning(line: &str) { + use std::io::Write; + let line = sanitize_node_msg(line); + let _ = writeln!(warn_sink(), "{line}"); +} + +#[cfg(not(test))] +fn warn_sink() -> impl std::io::Write { + std::io::stderr() +} + +#[cfg(test)] +fn warn_sink() -> impl std::io::Write { + tests::WarnMirror +} + #[derive(Args)] pub struct CloneArgs { /// Repo to clone: gitlawb:/// or /. @@ -312,10 +361,25 @@ async fn recover_encrypted_blobs( .await { Ok(r) if r.status().is_success() => r, - _ => continue, + // The node path has the same silent exit the gateway path had: a 403, a + // 404, or a dead connection all reached the caller as "blob not + // recoverable" with no reason attached. One unreachable blob still must + // not end the recovery of the rest, so it warns and moves on. + Ok(r) => { + warn_skip(oid, &format!("node returned {}", r.status())); + continue; + } + Err(e) => { + warn_skip(oid, &format!("node request failed: {e}")); + continue; + } }; - let Ok(envelope) = env_resp.bytes().await else { - continue; + let envelope = match env_resp.bytes().await { + Ok(b) => b, + Err(e) => { + warn_skip(oid, &format!("reading the envelope failed: {e}")); + continue; + } }; let plaintext = match open_blob(&envelope, keypair) { Ok(p) => p, @@ -642,7 +706,14 @@ async fn recover_from_arweave( for r in refs { let m = match client.get(format!("{ag}/{}", r.id)).send().await { Ok(resp) if resp.status().is_success() => resp, - _ => continue, + Ok(resp) => { + warn_skip_manifest(&r.id, &format!("gateway returned {}", resp.status())); + continue; + } + Err(e) => { + warn_skip_manifest(&r.id, &format!("gateway request failed: {e}")); + continue; + } }; if let Ok(parsed) = m.json::().await { manifests.push((parsed, r.height)); @@ -684,10 +755,31 @@ async fn recover_from_arweave( } let env_resp = match client.get(format!("{ig}/ipfs/{cid}")).send().await { Ok(r) if r.status().is_success() => r, - _ => continue, + // Every other outcome used to leave through a bare `continue`, so a + // gateway that answered 503, 404, or nothing at all was reported to the + // caller as "blob not recoverable" with no reason attached. This is the + // second in-repo client of GET /ipfs/{cid} and it has no resume ladder, + // so saying what happened is all the recourse there is. It still skips to + // the next candidate either way: one unreachable blob must not end the + // recovery of the rest. + Ok(r) => { + warn_skip(&oid, &format!("gateway returned {}", r.status())); + continue; + } + Err(e) => { + warn_skip(&oid, &format!("gateway request failed: {e}")); + continue; + } }; - let Ok(envelope) = env_resp.bytes().await else { - continue; + // A body that dies part-way through is the same mid-read failure the capped + // read exists to stop rendering as silence, and it sits INSIDE the loop whose + // status arms were already fixed. + let envelope = match env_resp.bytes().await { + Ok(b) => b, + Err(e) => { + warn_skip(&oid, &format!("reading the envelope failed: {e}")); + continue; + } }; // open_blob succeeds only if this caller is a recipient: this is the // authorization gate (no node, no DID check needed). @@ -801,9 +893,42 @@ pub async fn run(args: CloneArgs) -> Result<()> { #[cfg(test)] mod tests { use super::*; + use std::cell::RefCell; use std::process::Command; use tempfile::TempDir; + thread_local! { + /// Test-visible copy of the stderr warnings `warn_skip` emits. + /// `#[tokio::test]` runs the future on the test's own thread, so a + /// thread-local is enough. + static WARNINGS: RefCell = const { RefCell::new(String::new()) }; + } + + /// The `cfg(test)` warning sink. `emit_warning` writes here instead of to + /// stderr, so the assertions below observe the shipped write rather than a copy + /// made beside it: delete that write and they go red. + pub(super) struct WarnMirror; + + impl std::io::Write for WarnMirror { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + let text = String::from_utf8_lossy(buf).into_owned(); + WARNINGS.with(|w| w.borrow_mut().push_str(&text)); + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + fn reset_warnings() { + WARNINGS.with(|w| w.borrow_mut().clear()); + } + + fn warnings() -> String { + WARNINGS.with(|w| w.borrow().clone()) + } + fn g(args: &[&str], dir: &Path) { assert!(Command::new("git") .args(args) @@ -1255,6 +1380,132 @@ mod tests { assert_eq!(a.get("o1").map(String::as_str), Some("cidTS")); } + /// A gateway that refuses one blob must SAY so, and must not take the rest of the + /// recovery down with it. + /// + /// The IPFS fetch used to leave through a bare `_ => continue` on every + /// non-success, so a 503 (the truncation status a tuned node makes more likely), + /// a 404, and a dead connection all reached the caller as the same silent "blob + /// not recoverable". Two withheld blobs here: the gateway answers 503 for the + /// first and serves the second. The recovered path proves the loop continued; the + /// warning proves the skip was reported and names both the object and the status. + #[tokio::test] + async fn recover_from_arweave_reports_a_refusing_gateway_and_continues() { + use gitlawb_core::encrypt::seal_blob; + use gitlawb_core::identity::Keypair; + + reset_warnings(); + let (td, url) = bare_remote(&[ + ("public/a.txt", b"pub\n"), + ("secret/b.txt", b"SECRET B\n"), + ("secret/c.txt", b"SECRET C\n"), + ]); + let dest = td.path().join("dest"); + let bare = url.strip_prefix("file://").unwrap(); + assert!(Command::new("git") + .args(["-C", bare, "config", "uploadpack.allowFilter", "true"]) + .status() + .unwrap() + .success()); + setup_partial_clone(&dest, &url, &["/secret/**".to_string()], &[], None).unwrap(); + + let oid_of = |path: &str| { + let out = Command::new("git") + .args([ + "-C", + dest.to_str().unwrap(), + "rev-parse", + &format!("HEAD:{path}"), + ]) + .output() + .unwrap(); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + let refused_oid = oid_of("secret/b.txt"); + let served_oid = oid_of("secret/c.txt"); + + // Origin death, so recovery has to go through the gateways. + std::fs::remove_dir_all(bare).unwrap(); + + let reader = Keypair::generate(); + let envelope = seal_blob(b"SECRET C\n", &[reader.verifying_key()]).unwrap(); + + let refused_cid = "cidrefused"; + let served_cid = "cidserved"; + let mut server = mockito::Server::new_async().await; + let _gql = server + .mock("POST", "/graphql") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"data":{"transactions":{"edges":[{"node":{"id":"TX1"}}]}}}"#) + .create_async() + .await; + let manifest_body = serde_json::json!({ + "timestamp": "2026-06-11T00:00:00Z", + "blobs": [ + { "oid": refused_oid, "cid": refused_cid }, + { "oid": served_oid, "cid": served_cid }, + ], + }) + .to_string(); + let _tx = server + .mock("GET", "/TX1") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(manifest_body) + .create_async() + .await; + let refusal = server + .mock("GET", format!("/ipfs/{refused_cid}").as_str()) + .with_status(503) + .with_header("content-type", "application/json") + .with_body(r#"{"error":"search_incomplete","message":"scan truncated"}"#) + .expect(1) + .create_async() + .await; + let ok = server + .mock("GET", format!("/ipfs/{served_cid}").as_str()) + .with_status(200) + .with_body(envelope) + .expect(1) + .create_async() + .await; + + let paths = recover_from_arweave( + &server.url(), + &server.url(), + "alice", + "myrepo", + &dest, + &reader, + ) + .await + .unwrap(); + + refusal.assert_async().await; + ok.assert_async().await; + assert_eq!( + paths, + vec!["secret/c.txt".to_string()], + "a refused candidate must not stop the loop reaching the next one" + ); + + let warnings = warnings(); + assert!( + warnings.contains(&refused_oid), + "the skip must name the object it could not fetch, got: {warnings}" + ); + assert!( + warnings.contains("503"), + "the skip must name the gateway's status rather than failing silently, \ + got: {warnings}" + ); + assert!( + !warnings.contains(&served_oid), + "the blob that was served must not be reported as skipped, got: {warnings}" + ); + } + /// Read-path end-to-end over a mocked Arweave + IPFS gateway: discover the /// manifest via GraphQL, fetch it, fetch the envelope, decrypt with the /// caller's key, and install the previously-withheld blob. diff --git a/crates/gl/src/http.rs b/crates/gl/src/http.rs index 4a51dc45..7a28c6f7 100644 --- a/crates/gl/src/http.rs +++ b/crates/gl/src/http.rs @@ -15,6 +15,47 @@ use icaptcha_client::IcaptchaCfg; /// (absorbs proof expiry / first-seen replay). const MAX_ICAPTCHA_RETRIES: usize = 2; +/// Total request timeout: from the start of connecting through the end of the +/// response body, so it bounds a slow download and not just a slow handshake. +const TOTAL_REQUEST_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); + +/// Follow a redirect only when it stays on the origin that issued it AND re-issues the +/// identical request-target, and only for as long as the chain bound allows. +/// +/// The decision itself is [`gitlawb_core::redirect::may_follow`], shared with +/// `git-remote-gitlawb` so the two signing clients cannot drift apart on it. Refusal +/// is `stop`, not `error`: the 3xx comes back as an ordinary response and each caller +/// reports it through the status path it already has. +/// +/// `Policy::custom` replaces reqwest's built-in limit, so the chain bound is restated +/// here. Same-origin redirects can cycle, and this is what stops a node answering 302 +/// to itself from being followed indefinitely. It is not what makes the request +/// finite: `.timeout(...)` on the same builder is a TOTAL request timeout covering the +/// whole chain, so without this bound the worst case is a 30 second spin, not an +/// endless one. The bound is what keeps that spin from costing the node a request per +/// round trip for the full 30 seconds. +/// +/// `>` and not `>=`: reqwest pushes the redirecting URL onto `previous` before +/// consulting the policy, so on the first redirect `previous.len()` is already 1, and +/// `>=` would permit `MAX_REDIRECTS - 1` follows. `Policy::limited(max)` refuses at +/// `previous.len() > max`, and matching it is the point of reusing its value. +fn same_origin_redirect(attempt: reqwest::redirect::Attempt<'_>) -> reqwest::redirect::Action { + let Some(previous) = attempt.previous().last() else { + // No previous URL to compare against. Unreachable through reqwest, which + // pushes the redirecting URL before consulting the policy, but the safe + // reading of "cannot prove same-origin" is to refuse. + return attempt.stop(); + }; + if attempt.previous().len() > gitlawb_core::redirect::MAX_REDIRECTS { + return attempt.stop(); + } + if gitlawb_core::redirect::may_follow(previous, attempt.url()) { + attempt.follow() + } else { + attempt.stop() + } +} + pub struct NodeClient { inner: reqwest::Client, pub node_url: String, @@ -23,8 +64,20 @@ pub struct NodeClient { impl NodeClient { pub fn new(node_url: impl Into, keypair: Option) -> Self { + Self::with_timeout(node_url, keypair, TOTAL_REQUEST_TIMEOUT) + } + + /// `new` with the total request timeout as a parameter, so a test can drive the + /// timeout's behaviour without waiting out the shipped value. `new` supplies the + /// shipped one, which is what makes the scaled-down test cover the real client. + fn with_timeout( + node_url: impl Into, + keypair: Option, + timeout: std::time::Duration, + ) -> Self { let inner = reqwest::Client::builder() - .timeout(std::time::Duration::from_secs(30)) + .timeout(timeout) + .redirect(reqwest::redirect::Policy::custom(same_origin_redirect)) .user_agent(format!("gl/{} gitlawb-cli", env!("CARGO_PKG_VERSION"))) .build() .expect("failed to build HTTP client"); @@ -203,24 +256,64 @@ async fn obtain_proof(cfg: IcaptchaCfg) -> Result { .context("iCaptcha solver task panicked")? } +/// What a capped body read produced. `text` is the bytes that arrived; the two flags +/// say why the read stopped where it did, which a caller that CLASSIFIES on the body +/// cannot work out from the text alone. +pub(crate) struct CappedBody { + /// The bytes read, lossily decoded. + pub(crate) text: String, + /// The cap cut the body short. + pub(crate) truncated: bool, + /// A chunk read FAILED part-way through, so the body is not merely short, it is + /// unfinished and the node may have had more to say. + pub(crate) read_failed: bool, +} + /// Read at most `cap` bytes of a response body. Bounds the allocation from a /// hostile or broken node returning a huge error body — the display is capped /// separately, but the read itself must not be unbounded (INV-6, read half). -pub(crate) async fn read_body_capped(mut resp: reqwest::Response, cap: usize) -> String { +/// +/// `truncated` reports whether the cap cut the body short. A caller that CLASSIFIES +/// on the body needs it: a cut body fails JSON parse, and a parse failure is +/// indistinguishable from a node that sent no code at all, so without this flag an +/// oversized body silently picks a different arm. +/// +/// `read_failed` reports the other way a body can end early. A mid-body read error +/// used to leave through the same exit as a clean end of stream, so a 500 whose body +/// died in transit surfaced as an empty message and the caller was told +/// `node returned 500: ` with nothing after the colon. That is a report of what the +/// node said, and the node never got to say it. +pub(crate) async fn read_body_capped(mut resp: reqwest::Response, cap: usize) -> CappedBody { let mut buf: Vec = Vec::new(); + let mut truncated = false; + let mut read_failed = false; while buf.len() < cap { match resp.chunk().await { Ok(Some(chunk)) => { let take = (cap - buf.len()).min(chunk.len()); buf.extend_from_slice(&chunk[..take]); if take < chunk.len() { + truncated = true; break; // hit the cap mid-chunk } } - _ => break, // end of body or read error — return what we have + Ok(None) => break, // clean end of body + Err(_) => { + read_failed = true; + break; + } } } - String::from_utf8_lossy(&buf).into_owned() + // A body that lands exactly on the cap may or may not have more behind it; + // report it as cut, since the classification that follows cannot tell either. + if buf.len() >= cap { + truncated = true; + } + CappedBody { + text: String::from_utf8_lossy(&buf).into_owned(), + truncated, + read_failed, + } } /// Strip terminal-dangerous characters from (and cap the length of) a @@ -621,6 +714,530 @@ mod tests { ic.answer.assert(); } + // ── redirect policy ───────────────────────────────────────────────── + + /// The signed headers must not survive a redirect off the node's origin. + /// + /// reqwest strips only `Authorization`, `Cookie`, `Proxy-Authorization` and + /// `WWW-Authenticate` across hosts, so `Signature` and `Signature-Input` used to + /// ride a 302 straight to whatever origin the node named. The signature binds + /// `@method`, `@path` and `content-digest` and nothing about the authority, so the + /// receiving host holds a credential that reads path-scoped objects as the caller + /// at any node until the clock-skew window closes. + /// + /// Two mockito servers are two ports on one host, which is exactly the boundary + /// this policy draws (and the one reqwest's own header stripping draws). The + /// second server answers everything and expects nothing: a followed redirect + /// fails the expectation whether or not the signature came with it. MUTATION + /// (RED): drop the `.redirect(...)` line and the second server is hit. + #[tokio::test] + async fn cross_origin_redirect_is_not_followed() { + let mut elsewhere = Server::new_async().await; + let never = elsewhere + .mock("GET", mockito::Matcher::Any) + .with_status(200) + .with_body("bytes from the redirect target") + .expect(0) + .create_async() + .await; + let signature_seen = elsewhere + .mock("GET", mockito::Matcher::Any) + .match_header("signature", mockito::Matcher::Any) + .with_status(200) + .expect(0) + .create_async() + .await; + + let mut node = Server::new_async().await; + let bounce = node + .mock("GET", "/api/v1/thing") + .with_status(302) + .with_header("location", &format!("{}/api/v1/thing", elsewhere.url())) + .expect(1) + .create_async() + .await; + + let client = NodeClient::new(node.url(), Some(test_keypair())); + let resp = client.get_signed("/api/v1/thing").await.unwrap(); + + assert_eq!( + resp.status(), + 302, + "a refused redirect stops rather than errors, so the caller sees the 3xx \ + and reports it through the status path it already has" + ); + let body = resp.text().await.unwrap(); + assert!( + !body.contains("bytes from the redirect target"), + "the redirect target's bytes must never reach the caller, got: {body}" + ); + bounce.assert_async().await; + never.assert_async().await; + signature_seen.assert_async().await; + } + + /// A node redirecting to itself is same-origin, so the origin predicate follows it + /// every time and only the chain bound ends the loop. Deleting the bound left the + /// whole suite green, because nothing here had ever built a cycle. + /// + /// It has a second job now. A redirect back to the identical path and query is the + /// only same-origin hop the predicate still follows, so the eleven hits below are + /// also this crate's executed proof that such a hop IS followed. The old positive + /// fixture drove a trailing-slash rewrite, which the request-target rule refuses, + /// and an http-to-https upgrade cannot be mocked over mockito's plain http. + /// + /// The route answers 301 pointing back at itself. Bounded, the handler is hit + /// once for the original request plus `MAX_REDIRECTS` follows and the call returns + /// the 301 (a refused redirect stops rather than errors). Unbounded, it runs until + /// the client's total request timeout cuts it off, which is a 30 second spin at the + /// shipped value and a request per round trip for the node. + /// + /// MUTATION (RED): delete the `previous().len()` check. + #[tokio::test] + async fn a_self_redirect_stops_at_the_chain_bound() { + let mut node = Server::new_async().await; + let hits = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let h = hits.clone(); + let loop_route = node + .mock("GET", "/api/v1/loop") + .with_status(301) + .with_header("location", "/api/v1/loop") + .with_body_from_request(move |_req| { + h.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Vec::new() + }) + .expect_at_least(1) + .create_async() + .await; + + let client = NodeClient::with_timeout( + node.url(), + Some(test_keypair()), + std::time::Duration::from_secs(5), + ); + let resp = client + .get_signed("/api/v1/loop") + .await + .expect("the bound must end the chain, not the timeout"); + + assert_eq!( + resp.status(), + 301, + "the chain ends by refusing the next hop, so the last 3xx is what comes back" + ); + assert_eq!( + hits.load(std::sync::atomic::Ordering::SeqCst), + gitlawb_core::redirect::MAX_REDIRECTS + 1, + "one original request plus MAX_REDIRECTS follows, matching what \ + Policy::limited(MAX_REDIRECTS) would have permitted" + ); + loop_route.assert_async().await; + } + + /// A same-origin hop that REWRITES the request-target is refused, even though the + /// origin never changes. + /// + /// The signature binds `@path` as the literal path-and-query the client signed, and + /// the node rebuilds it from the URI it received. A `/api/v1/thing` to + /// `/api/v1/thing/` bounce therefore presents a signature over a target the node + /// never saw, and every signed read behind such a proxy 401s. Refusing the hop + /// hands the caller the 3xx that names what happened instead. + /// + /// The target mock expects zero hits and is asserted: mockito only checks + /// `.expect(N)` when `.assert()` runs, so an unbound or unasserted mock passes + /// vacuously. + #[tokio::test] + async fn same_origin_path_changing_redirect_is_refused() { + let mut node = Server::new_async().await; + let bounce = node + .mock("GET", "/api/v1/thing") + .with_status(301) + .with_header("location", "/api/v1/thing/") + .expect(1) + .create_async() + .await; + let target = node + .mock("GET", "/api/v1/thing/") + .with_status(200) + .with_body("bytes from the rewritten target") + .expect(0) + .create_async() + .await; + + let client = NodeClient::new(node.url(), Some(test_keypair())); + let resp = client.get_signed("/api/v1/thing").await.unwrap(); + + assert_eq!( + resp.status(), + 301, + "a refused redirect stops rather than errors, so the caller sees the 3xx \ + and reports it through the status path it already has" + ); + let body = resp.text().await.unwrap(); + assert!( + !body.contains("bytes from the rewritten target"), + "the rewritten target's bytes must never reach the caller, got: {body}" + ); + bounce.assert_async().await; + target.assert_async().await; + } + + // ── the node's own verification, run over what the client actually sent ── + + /// What the verifying mock made of a request that reached it. + /// + /// The empty slot (`None`) is its own state and means the mock was never hit at + /// all, which is what the refusal test asserts. It must stay distinguishable from + /// [`Verdict::WrongIdentity`], because "nobody verified anything" and "something + /// verified against the wrong key" are opposite findings. + /// + /// The payloads are read through `Debug` in the assertion messages and nowhere + /// else, which the dead-code pass does not count; they carry the detail that makes + /// a failure legible, so they stay. + #[derive(Debug)] + #[allow(dead_code)] + enum Verdict { + /// The chain accepted the signature AND the key it resolved is the test's DID. + Accepted, + /// The chain refused it. Carries the error so a failure reads as the actual + /// rejection rather than a bare hit count. + Rejected(String), + /// The chain accepted a signature made by somebody else. A key resolved from + /// the parsed `key_id` is read out of the artifact under verification, so an + /// accept on it alone proves consistency, never authenticity. + WrongIdentity { expected: String, got: String }, + } + + /// The node's `require_signature` verification, over a request this crate did not + /// build: parse the headers, recompute the content-digest from the body, rebuild + /// the signing string over `@method`/`@path`/`content-digest`, Ed25519-verify. + /// Returns the DID the signature resolved to, so a caller can pin the identity. + /// + /// A hand-copy of its twin in `crates/git-remote-gitlawb/src/main.rs`, which gl + /// cannot import (`git-remote-gitlawb` is a binary crate and this is its test + /// module). Keep the two textually identical apart from the mockito seam around + /// them, so an edit to one is visibly an edit to both. + /// + /// The production verifier both copies mirror is `crate::auth::require_signature` in + /// `crates/gitlawb-node/src/auth/mod.rs`. This is a re-implementation, not a call, so + /// an edit to that middleware has to land here too: otherwise the copies drift and + /// this test keeps passing against a rule the node has stopped applying. + /// + /// It asserts internally, which is deliberate but constrains its callers: inside + /// `with_body_from_request` those assertions fire on the server thread and reach + /// the client as a transport error, not as a recorded verdict. So the identity + /// check lives in the caller as a [`Verdict`] variant, never as an assert in here. + fn node_verifies( + method: &str, + path_and_query: &str, + body: &[u8], + sig_input: &str, + sig_header: &str, + content_digest: &str, + ) -> anyhow::Result { + use gitlawb_core::http_sig::{build_signing_string, compute_content_digest, HttpSignature}; + use gitlawb_core::identity::verify; + use std::collections::HashMap; + + let sig = HttpSignature::parse(sig_input, sig_header)?; + sig.check_created()?; + assert!( + sig.missing_components().is_empty(), + "signature must cover all required components" + ); + assert_eq!(sig.alg, "ed25519"); + assert_eq!( + content_digest, + compute_content_digest(body), + "content-digest must match the body" + ); + let vk = sig.key_id.to_verifying_key()?; + let mut values = HashMap::new(); + values.insert("@method".to_string(), method.to_uppercase()); + values.insert("@path".to_string(), path_and_query.to_string()); + values.insert("content-digest".to_string(), content_digest.to_string()); + let sig_params_value = sig_input.strip_prefix("sig1=").unwrap_or(sig_input); + let components: Vec<&str> = sig.components.iter().map(String::as_str).collect(); + let signing_string = build_signing_string(&components, sig_params_value, &values)?; + let sig_array: [u8; 64] = sig.signature_bytes.as_slice().try_into()?; + verify(&vk, signing_string.as_bytes(), &sig_array)?; + Ok(sig.key_id.to_string()) + } + + /// Pull a header value off a received mockito request, or explain which one the + /// client failed to send. + fn received_header(req: &mockito::Request, name: &str) -> String { + req.header(name) + .first() + .unwrap_or_else(|| panic!("the client sent no {name} header")) + .to_str() + .unwrap() + .to_string() + } + + /// Run [`node_verifies`] over a GET that arrived at the mock and record what the + /// node would have made of it, pinned to `expected_did`. + fn record_get_verdict( + req: &mockito::Request, + expected_did: &str, + slot: &std::sync::Arc>>, + ) { + let verdict = match node_verifies( + "GET", + req.path_and_query(), + b"", + &received_header(req, "signature-input"), + &received_header(req, "signature"), + &received_header(req, "content-digest"), + ) { + Ok(did) if did == expected_did => Verdict::Accepted, + Ok(did) => Verdict::WrongIdentity { + expected: expected_did.to_string(), + got: did, + }, + Err(e) => Verdict::Rejected(e.to_string()), + }; + *slot.lock().unwrap() = Some(verdict); + } + + /// The finding's repro, now a guard: a rewritten same-origin target must never + /// receive the signature, and the proof is the node's own verification, not a hit + /// count. + /// + /// The target mock runs the full `require_signature` chain over the request it + /// receives. Post-fix the hop is refused, so the slot stays empty. Pre-fix the hop + /// is followed and the slot records the Ed25519 rejection of a signature made over + /// `/api/v1/thing` and presented at `/api/v1/thing/`, which is the 401 an operator + /// behind such a proxy actually sees. The verdict is asserted first, so a failure + /// speaks about verification rather than about reachability. + /// + /// Its paired positive control is + /// `a_direct_signed_get_verifies_under_the_node_verifier`: without it, an empty + /// slot would be satisfied just as well by a harness that can never record + /// anything. + #[tokio::test] + async fn a_rewritten_target_never_receives_the_signature() { + let kp = test_keypair(); + let expected_did = kp.did().to_string(); + let slot = std::sync::Arc::new(std::sync::Mutex::new(None::)); + + let mut node = Server::new_async().await; + let bounce = node + .mock("GET", "/api/v1/thing") + .with_status(301) + .with_header("location", "/api/v1/thing/") + .expect(1) + .create_async() + .await; + let recorder = slot.clone(); + let did_for_target = expected_did.clone(); + let target = node + .mock("GET", "/api/v1/thing/") + .with_status(200) + .with_body_from_request(move |req| { + record_get_verdict(req, &did_for_target, &recorder); + Vec::new() + }) + .expect(0) + .create_async() + .await; + + let client = NodeClient::new(node.url(), Some(kp)); + let resp = client.get_signed("/api/v1/thing").await.unwrap(); + + let verdict = slot.lock().unwrap().take(); + assert!( + verdict.is_none(), + "the node's own verifier must never see this request: the signature covers \ + /api/v1/thing and the rewritten target is /api/v1/thing/, so what arrives \ + there is a stale request-target; recorded verdict: {verdict:?}" + ); + assert_eq!( + resp.status(), + 301, + "the caller sees the 3xx, not the rewritten target's answer" + ); + bounce.assert_async().await; + target.assert_async().await; + } + + /// The positive control for the test above, and the proof that the client signs + /// the query it sends. + /// + /// A direct signed GET, no redirect anywhere, through the same verifying mock. The + /// verdict must be `Accepted`, which is what makes the refusal test's empty slot + /// attributable to the refusal rather than to a harness that cannot record. The + /// path carries a query, so a client that signed the bare path would land here as + /// `Rejected`. + #[tokio::test] + async fn a_direct_signed_get_verifies_under_the_node_verifier() { + let kp = test_keypair(); + let expected_did = kp.did().to_string(); + let slot = std::sync::Arc::new(std::sync::Mutex::new(None::)); + + let mut node = Server::new_async().await; + let recorder = slot.clone(); + let did_for_target = expected_did.clone(); + let route = node + .mock("GET", "/api/v1/thing?x=1") + .with_status(200) + .with_body_from_request(move |req| { + record_get_verdict(req, &did_for_target, &recorder); + Vec::new() + }) + .expect(1) + .create_async() + .await; + + let client = NodeClient::new(node.url(), Some(kp)); + let resp = client.get_signed("/api/v1/thing?x=1").await.unwrap(); + assert_eq!(resp.status(), 200); + + let verdict = slot.lock().unwrap().take(); + assert!( + matches!(verdict, Some(Verdict::Accepted)), + "a direct signed GET must verify under the node's own chain and resolve to \ + {expected_did}, or the refusal test's empty slot proves nothing; recorded \ + verdict: {verdict:?}" + ); + route.assert_async().await; + } + + // ── read_body_capped ──────────────────────────────────────────────── + + /// A body whose read FAILS mid-stream must be distinguishable from a body that + /// ended. Both used to leave through the same `_ => break`, so a 500 whose body + /// died in transit produced an empty message and the caller was told + /// `node returned 500: ` with nothing after the colon. + /// + /// The fixture is a raw listener that promises 64 bytes in `Content-Length`, + /// writes 5, and closes. mockito cannot express that: it always completes the + /// response it advertises. MUTATION (RED): restore the single `_ => break` arm + /// (or hard-code `read_failed: false`) and the flag reads false. + #[tokio::test] + async fn read_body_capped_flags_a_mid_body_read_failure() { + let addr = spawn_short_body_listener().await; + let resp = reqwest::get(format!("http://{addr}/truncated")) + .await + .expect("headers arrive before the body is cut"); + let body = read_body_capped(resp, 8192).await; + + assert!( + body.read_failed, + "a body cut off mid-stream must be reported as a failed read, not as a \ + body that ended: got {:?}", + body.text + ); + assert!( + !body.truncated, + "the cap did not cut this one; 5 bytes are nowhere near 8 KiB" + ); + } + + /// The must-not half: a body that ends cleanly must NOT be flagged, or the flag + /// means nothing and every terminal starts claiming the node went quiet. + #[tokio::test] + async fn read_body_capped_does_not_flag_a_clean_body() { + let mut server = Server::new_async().await; + let _m = server + .mock("GET", "/ok") + .with_status(500) + .with_body("node said this") + .create_async() + .await; + let resp = reqwest::get(format!("{}/ok", server.url())).await.unwrap(); + let body = read_body_capped(resp, 8192).await; + + assert_eq!(body.text, "node said this"); + assert!(!body.read_failed, "a complete body is not a failed read"); + assert!(!body.truncated, "a complete body is not a truncated one"); + } + + /// Answer one request with headers promising more body than gets written, then + /// close the connection. Returns the listener's address. + async fn spawn_short_body_listener() -> std::net::SocketAddr { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + let (mut sock, _) = listener.accept().await.unwrap(); + let mut scratch = [0u8; 1024]; + let _ = sock.read(&mut scratch).await; + let _ = sock + .write_all(b"HTTP/1.1 500 Internal Server Error\r\nContent-Length: 64\r\n\r\nshort") + .await; + let _ = sock.flush().await; + // Drop closes the socket with 59 of the promised bytes never sent. + }); + addr + } + + // ── total request timeout ─────────────────────────────────────────── + + /// The client's timeout is a TOTAL request timeout, so it bounds a download and + /// not just the handshake. `gl ipfs get`'s documentation leans on exactly that: + /// the wall-clock deadline covers the search and deliberately stops at the + /// response headers, and this timeout is the only thing left bounding the body. + /// + /// Driven at 250ms through `with_timeout`, the seam `new` itself calls with + /// `TOTAL_REQUEST_TIMEOUT`, because a test at the shipped 30s has no place in this + /// suite. What that costs is the value; what it proves is the SHAPE, which is the + /// part in doubt: that the deadline keeps running once the headers have landed. A + /// timeout that covered only the handshake would let this request hang until the + /// listener gives up. + #[tokio::test] + async fn total_timeout_cuts_off_a_body_that_outruns_it() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + let (mut sock, _) = listener.accept().await.unwrap(); + let mut scratch = [0u8; 1024]; + let _ = sock.read(&mut scratch).await; + // Headers land immediately, then the body stalls indefinitely. + let _ = sock + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 1024\r\n\r\nfirst") + .await; + let _ = sock.flush().await; + tokio::time::sleep(std::time::Duration::from_secs(30)).await; + }); + + let client = NodeClient::with_timeout( + format!("http://{addr}"), + None, + std::time::Duration::from_millis(250), + ); + let started = std::time::Instant::now(); + let resp = client.get("/slow").await.expect("headers arrive promptly"); + assert_eq!( + resp.status(), + 200, + "the stall is in the body, not the status" + ); + let err = resp + .bytes() + .await + .expect_err("a body still arriving past the total timeout must be cut off"); + let elapsed = started.elapsed(); + + assert!( + err.is_timeout(), + "the body read must end in a timeout, got: {err}" + ); + assert!( + elapsed < std::time::Duration::from_secs(5), + "the timeout must fire on its own schedule, not wait out the listener; \ + took {elapsed:?}" + ); + } + + #[test] + fn shipped_client_uses_the_documented_total_timeout() { + // The scaled-down test above proves the shape at 250ms. This pins the value + // `new` actually ships, so the two together cover the documented behaviour. + assert_eq!(TOTAL_REQUEST_TIMEOUT, std::time::Duration::from_secs(30)); + } + #[test] fn sanitize_strips_controls_bidi_and_caps_length() { // C0 (ESC/BEL) and the Cf bidi override (U+202E) are both removed; the diff --git a/crates/gl/src/ipfs_cmd.rs b/crates/gl/src/ipfs_cmd.rs index d4cb64fc..93ca5511 100644 --- a/crates/gl/src/ipfs_cmd.rs +++ b/crates/gl/src/ipfs_cmd.rs @@ -4,12 +4,13 @@ //! objects by their content-addressed CID. use std::path::PathBuf; +use std::time::Duration; use anyhow::{Context, Result}; use clap::{Args, Subcommand}; use serde_json::Value; -use crate::http::NodeClient; +use crate::http::{read_body_capped, sanitize_node_msg, NodeClient}; #[derive(Args)] pub struct IpfsArgs { @@ -28,18 +29,67 @@ pub enum IpfsCmd { dir: Option, }, /// Retrieve and display a git object from the node by its CIDv1 + /// + /// Object bytes go to stdout so the command pipes; diagnostics go to stderr. + /// + /// Objects pinned before the node started recording which repo they came from + /// are found by scanning its repo inventory, and that scan stops at the node's + /// per-request ceilings. When it stops the node answers 503 with a resume token + /// rather than a false "not found", and this command follows it automatically: + /// up to 8 resumes after the first request, so at most 9 calls to the node, + /// waiting between attempts for as long as the node's Retry-After asks and never + /// longer than 5 seconds. + /// + /// The whole ladder runs under a 60 second wall-clock deadline. That deadline + /// bounds the search: each attempt gets only the time left on it to produce + /// response headers, and it deliberately does not cover the download of an + /// object once found. The download is not unbounded, though. The client's 30 + /// second HTTP timeout is a TOTAL request timeout, running from the moment a + /// request starts connecting until its body has finished, so a transfer still + /// going 30 seconds after its own request began is cut off. Waits between + /// attempts are bounded by the time left on the deadline as well as by the 5 + /// second clamp, so the longest a run can spend on the network is about 90 + /// seconds: the deadline, plus the 30 second timeout covering the last attempt. + /// Writing the object out is not covered by either bound, so piping into a reader + /// that stops reading can hold the command open past that. + /// + /// A 429 ends the ladder immediately: the node's rate-limit window is an hour, + /// so the wait it asks for cannot be honored inside one invocation. A transient + /// overload (a 503 that carries no incomplete-scan code) is retried on the token + /// already held, under the same cap, clamp and deadline. The node's per-IP + /// fanout brake can also end a ladder well short of the cap, so automatic + /// resumption is not a guarantee that the object will be reached. + /// + /// Whenever one of those bounds stops the ladder with a usable token still in + /// hand, the command prints the token and the exact invocation that continues + /// from it, `gl ipfs get --scan `, and exits nonzero. Re-running + /// without the token restarts the scan at the first row, reproduces the same + /// truncation and spends the node's per-IP budget again, so the token is the + /// only thing that makes progress. Tokens are valid for an hour. Get { /// The CIDv1 string (e.g. bafkrei...) cid: String, #[arg(long, default_value = "https://node.gitlawb.com", env = "GITLAWB_NODE")] node: String, + /// Identity directory (default: ~/.gitlawb) + #[arg(long)] + dir: Option, + /// Resume token from a scan that stopped at a bound, as printed by a + /// previous run that gave up with the result incomplete + #[arg(long, value_name = "TOKEN")] + scan: Option, }, } pub async fn run(args: IpfsArgs) -> Result<()> { match args.cmd { IpfsCmd::List { node, dir } => cmd_list(node, dir).await, - IpfsCmd::Get { cid, node } => cmd_get(cid, node).await, + IpfsCmd::Get { + cid, + node, + dir, + scan, + } => cmd_get(cid, node, dir, scan).await, } } @@ -86,42 +136,485 @@ async fn cmd_list(node: String, dir: Option) -> Result<()> { Ok(()) } -async fn cmd_get(cid: String, node: String) -> Result<()> { - let client = NodeClient::new(&node, None); - let path = format!("/ipfs/{cid}"); - let resp = client - .get(&path) - .await - .with_context(|| format!("failed to fetch CID {cid} from {node}"))?; +/// Automatic resumes attempted after the initial request when the node reports a +/// truncated legacy scan, so at most `MAX_SCAN_RESUMES + 1` node calls per invocation. +const MAX_SCAN_RESUMES: usize = 8; - let status = resp.status(); - if !status.is_success() { - let body = resp.text().await.unwrap_or_default(); - anyhow::bail!("node returned {status}: {body}"); +/// Wall-clock budget for a whole `gl ipfs get`, resumes included. +const SCAN_DEADLINE: Duration = Duration::from_secs(60); + +/// Longest single wait honored between attempts, whatever `Retry-After` asks for. +/// The node picks that number, so an unclamped sleep would let a hostile one stall +/// the client for as long as it likes. +const MAX_RETRY_AFTER: Duration = Duration::from_secs(5); + +/// Wait used when a retryable response carries no usable `Retry-After`. +const DEFAULT_RETRY_AFTER: Duration = Duration::from_secs(1); + +/// Generous ceiling on a continuation token. Real tokens are fixed-width (756 +/// base64url characters today), but the sealed layout has already changed once and +/// a rejected token is terminal, so a tight bound would silently kill resume on a +/// future version bump. +const MAX_CONTINUATION_LEN: usize = 2048; + +/// Mirror a stderr diagnostic into a per-thread buffer under `cfg(test)` so the +/// command-level tests can assert on what the caller is actually told. Callers +/// see the same line either way; only the test-visible copy is conditional. +fn diag(msg: &str) { + eprintln!("{msg}"); + #[cfg(test)] + tests::record_diag(msg); +} + +async fn cmd_get( + cid: String, + node: String, + dir: Option, + scan: Option, +) -> Result<()> { + cmd_get_inner(cid, node, dir, scan, SCAN_DEADLINE, MAX_SCAN_RESUMES).await +} + +/// The body of `gl ipfs get`, with the bounds and the starting continuation as +/// parameters so tests can drive the resume ladder without waiting out the shipped +/// defaults. `cmd_get` supplies those defaults. +async fn cmd_get_inner( + cid: String, + node: String, + dir: Option, + continuation: Option, + deadline: Duration, + cap: usize, +) -> Result<()> { + // #173 (F5): the resolver now serves path-scoped objects to authorized readers, + // so sign with an available identity like `gl ipfs list` — otherwise an owner or + // listed reader gets the opaque anonymous 404 for content they can read. + // `get_authed` signs when a keypair is present and falls back to unsigned. + // + // An explicit `--dir` is a request to use THAT identity: propagate a + // missing/corrupt-keystore error (like `list`) instead of silently sending an + // anonymous request the authorized reader would see as the node's opaque 404 + // (#173 review). Only the default (no `--dir`) keeps the best-effort unsigned + // fallback, so `get` stays usable for genuinely public content. + let keypair = match dir.as_deref() { + Some(dir) => Some(crate::identity::load_keypair_from_dir(Some(dir))?), + None => crate::identity::load_keypair_from_dir(None).ok(), + }; + let client = NodeClient::new(&node, keypair); + // #173 review (F1): the node now accepts equivalent multibase spellings, + // including base64 CIDs (prefix 'm'), whose alphabet contains '/', '+', '='. + // Interpolating the CID raw would make the client request (and sign) + // `/ipfs//`, which neither matches the single-segment Axum + // route nor points at the intended target. Percent-encode the CID as exactly + // one path segment so the signed and sent target agree and the server's + // `Path` extractor decodes it back to the original CID. + let encoded_cid = encode_cid_segment(&cid); + + // A caller-supplied continuation reaches the same signed target as a node-chosen + // one, so it clears the same bar before the first request. + let mut token = match continuation { + Some(t) if valid_continuation(&t) => Some(t), + Some(_) => anyhow::bail!( + "the supplied continuation is not a resume token: \ + expected 1 to {MAX_CONTINUATION_LEN} base64url characters" + ), + None => None, + }; + + // One deadline for the whole ladder, captured before the first request and used + // both as the loop's bound and as each attempt's own timeout, so no attempt can + // start just under the deadline and then run a fresh unbounded 30s of its own. + // That wrap covers `get_authed` only, which resolves on the response HEADERS: the + // deadline is here to stop a slow legacy SEARCH, and extending it over the body + // read would abort a legitimate large download whose bytes are already flowing. + // reqwest's blanket 30s is what bounds the download instead; it is a TOTAL request + // timeout, from the start of the request through the end of its body, so a transfer + // slower than that from its own request's start IS cut off. + // The composed bound that follows. The last attempt of any run starts strictly + // before the deadline, since both checks above run first, and that same blanket 30s + // covers its whole request, so it is over by deadline + 30s. Two reads sit under the + // 30s and not under the deadline: `write_object`'s success read, which ends the run, + // and `read_body_capped`'s error read, which on a retryable arm is followed by one + // wait. That wait adds no term of its own, because it is bounded by the time LEFT on + // the deadline as well as by the clamp. So the worst case ON THE NETWORK is + // deadline + 30s, about 90s at the shipped defaults. `write_object`'s writes to + // stdout are blocking and under neither bound, so a stalled consumer on the other + // end of the pipe can outlast that; nothing here can bound a caller's own reader. + let start = tokio::time::Instant::now(); + let mut requests = 0usize; + loop { + if requests > cap { + diag(&format!( + "warning: the node's legacy scan is still incomplete after {cap} automatic \ + resumes; the object may sit beyond the rows scanned so far" + )); + surface_resume(&cid, token.as_deref()); + anyhow::bail!( + "gave up on an incomplete scan for CID {cid} after {requests} node calls" + ); + } + let remaining = deadline.saturating_sub(start.elapsed()); + if remaining.is_zero() { + return Err(deadline_reached(&cid, token.as_deref(), deadline)); + } + + // The token joins the single `path` binding BEFORE `get_authed` signs, so the + // signature covers the query string and the bytes signed are the bytes sent. + // Percent-encoding is the identity over the accepted alphabet; it is here for + // the value that is not. + let path = match &token { + Some(t) => format!("/ipfs/{encoded_cid}?scan={}", urlencoding::encode(t)), + None => format!("/ipfs/{encoded_cid}"), + }; + let resp = match tokio::time::timeout(remaining, client.get_authed(&path)).await { + Ok(Ok(r)) => r, + Ok(Err(e)) => { + // A transport failure ends the ladder with the held token still + // pointing at a real position, so hand it back before propagating: + // otherwise a connection reset mid-ladder loses the only thing that + // makes progress on a re-run. + surface_resume(&cid, token.as_deref()); + return Err(e).with_context(|| format!("failed to fetch CID {cid} from {node}")); + } + Err(_) => return Err(deadline_reached(&cid, token.as_deref(), deadline)), + }; + requests += 1; + + // Status first, before any body read. + let status = resp.status(); + if status.is_success() { + return write_object(resp).await; + } + if status == reqwest::StatusCode::TOO_MANY_REQUESTS { + // Terminal on the status alone: the fanout limiter's window is an hour, so + // the wait it advertises cannot be honored inside one invocation, and + // retrying only deepens the shedding the ladder itself caused. + diag( + "warning: the node is rate limiting this scan, so the result is incomplete; \ + its limit window outlasts a single invocation", + ); + surface_resume(&cid, token.as_deref()); + anyhow::bail!("node returned {status}: rate limited"); + } + + let retry_after = parse_retry_after(resp.headers()); + let body = read_body_capped(resp, 8 * 1024).await; + let (raw, truncated) = (body.text, body.truncated); + let read_failed = body.read_failed; + let parsed = serde_json::from_str::(&raw).ok(); + let code = parsed.as_ref().and_then(|v| v["error"].as_str()); + let node_msg = parsed + .as_ref() + .and_then(|v| v["message"].as_str()) + .unwrap_or(raw.as_str()); + let offered = parsed.as_ref().and_then(|v| v["continuation"].as_str()); + + // The one classification site. The node's error code picks the arm and the + // default is terminal, so an unrecognized code can never resolve to a retry. + let resume_with = match code { + Some("search_incomplete") => offered + .filter(|t| valid_continuation(t)) + .map(str::to_string), + // A mid-ladder overload sheds a request whose permit is released at request + // end and asks the caller back shortly, so the ladder continues on the token + // it already holds. With no token there is nothing to resume, which falls to + // the default arm below. + // + // A body that did not arrive whole is excluded from this arm, whether the + // cap CUT it short or the read FAILED part-way. Either way it cannot parse, + // so its code reads as absent and an oversized or broken `search_incomplete` + // would land here and be retried on the OLD token, replaying one position + // for every rung while the fresh continuation it offered goes unread. + // Unclassifiable is terminal, like any unrecognized code. + Some(_) | None + if status == reqwest::StatusCode::SERVICE_UNAVAILABLE + && !truncated + && !read_failed => + { + token.clone() + } + _ => None, + }; + + let Some(next) = resume_with else { + let msg = node_tail(node_msg, read_failed); + if code == Some("search_incomplete") { + // Naming the bound, never echoing the value: a rejected token is + // node-chosen text and has no business in a terminal message. + let why = if offered.is_some() { + // The OFFERED token is unusable, but the one already held still + // points at a real position, so the ladder ends with something to + // resume from. Surface ours, never theirs. + surface_resume(&cid, token.as_deref()); + format!( + "the continuation it offered is not a resume token \ + (expected 1 to {MAX_CONTINUATION_LEN} base64url characters)" + ) + } else { + // No continuation at all is the node's deliberate "the scan wrapped + // and finished" signal, so a resume hint here would invite a re-run + // that cannot find more than this one did. + "it offered no continuation token".to_string() + }; + anyhow::bail!("node returned {status} with the scan incomplete and {why}: {msg}"); + } + // Anything else stops the ladder with the held token still usable, so hand + // it back. The exception is a definitive 404: that is an answer, and a + // resume hint beside it would contradict it. + if status != reqwest::StatusCode::NOT_FOUND { + surface_resume(&cid, token.as_deref()); + } + anyhow::bail!("node returned {status}: {msg}"); + }; + + // Bounded three ways, and the deadline is the term that stops the give-up from + // overshooting: the loop only re-checks it at the top, so a wait longer than + // what is left would run past the deadline before anything noticed. + let left = deadline.saturating_sub(start.elapsed()); + tokio::time::sleep(retry_after.min(MAX_RETRY_AFTER).min(left)).await; + token = Some(next); } +} + +/// Write a successful response: diagnostics to stderr, raw bytes to stdout so the +/// output stays pipeable. +async fn write_object(resp: reqwest::Response) -> Result<()> { + write_object_to(resp, &mut std::io::stdout()).await +} - // Print headers for diagnostics +/// `write_object` with the sink as a parameter, so a test can read back what a +/// caller would have received on stdout. `write_object` supplies the real one. +/// +/// The body is STREAMED. `resp.bytes()` buffers the whole object first, so a node +/// answering 200 with a very large body delivered fast made the client allocate all +/// of it before a byte reached stdout; the 30 second client timeout bounds how long +/// that takes, not how much it costs. Chunk-at-a-time the peak is one chunk, and the +/// sibling error read is already capped at 8 KiB. +async fn write_object_to( + mut resp: reqwest::Response, + out: &mut W, +) -> Result<()> { let headers = resp.headers().clone(); if let Some(git_hash) = headers.get("x-git-hash") { - eprintln!("x-git-hash: {}", git_hash.to_str().unwrap_or("?")); + diag(&format!( + "x-git-hash: {}", + git_hash.to_str().unwrap_or("?") + )); } if let Some(content_cid) = headers.get("x-content-cid") { - eprintln!("x-content-cid: {}", content_cid.to_str().unwrap_or("?")); + diag(&format!( + "x-content-cid: {}", + content_cid.to_str().unwrap_or("?") + )); } - // Write raw bytes to stdout (allows piping to files or other tools) - let bytes = resp.bytes().await.context("failed to read response body")?; - use std::io::Write; - std::io::stdout() - .write_all(&bytes) - .context("failed to write to stdout")?; + while let Some(chunk) = resp.chunk().await.context("failed to read response body")? { + out.write_all(&chunk).context("failed to write to stdout")?; + } + // Flush explicitly rather than leaving the tail to the process-exit flush, which + // discards its error: `gl ipfs get > object.bin` onto a full disk or a + // closed pipe would otherwise leave a TRUNCATED file behind exit status 0, and on + // a content-addressed fetch a silently short object is the worst possible answer. + out.flush().context("failed to flush stdout")?; Ok(()) } +/// Report the wall-clock give-up and hand the caller their token back. +fn deadline_reached(cid: &str, token: Option<&str>, deadline: Duration) -> anyhow::Error { + diag(&format!( + "warning: the node's legacy scan is still incomplete at the {}s deadline; \ + the object may sit beyond the rows scanned so far", + deadline.as_secs_f32() + )); + surface_resume(cid, token); + anyhow::anyhow!("gave up on an incomplete scan for CID {cid} at the wall-clock deadline") +} + +/// Hand back the token that still points at where the scan stopped, together with +/// the invocation that resumes from it. A bare re-run restarts at row 0, reproduces +/// the same truncation, and re-spends the caller's per-IP budget, so the token is +/// the only thing that makes progress. +fn surface_resume(cid: &str, token: Option<&str>) { + if let Some(t) = token { + diag(&format!( + "resume from where this stopped: gl ipfs get {cid} --scan {t}" + )); + } +} + +/// Render the tail of a terminal message: what the node said, sanitized, plus the +/// fact that its body did not finish arriving when that is what happened. +/// +/// A read that fails mid-body is not the same as a node with nothing to say, and the +/// two used to render identically. A 500 whose body died in transit produced an empty +/// message and the terminal read `node returned 500: `, which reports the node as +/// silent when the truth is that the connection broke before it could be heard. +fn node_tail(node_msg: &str, read_failed: bool) -> String { + let msg = sanitize_node_msg(node_msg); + match (read_failed, msg.is_empty()) { + (false, _) => msg, + (true, true) => "the response body could not be read".to_string(), + (true, false) => format!("{msg} (the response body could not be read in full)"), + } +} + +/// `Retry-After` in delta-seconds. Absent, non-numeric, or an HTTP-date all fall +/// back to one second; the caller clamps whatever comes back. +fn parse_retry_after(headers: &reqwest::header::HeaderMap) -> Duration { + headers + .get(reqwest::header::RETRY_AFTER) + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.trim().parse::().ok()) + .map(Duration::from_secs) + .unwrap_or(DEFAULT_RETRY_AFTER) +} + +/// A continuation is node-chosen and goes straight into a signed request target, so +/// accept only the alphabet the node's sealer emits. A value carrying `#`, `&`, `?`, +/// `/`, whitespace, or control bytes could make the URL reqwest parses differ from +/// the bytes signed. +fn valid_continuation(token: &str) -> bool { + !token.is_empty() + && token.len() <= MAX_CONTINUATION_LEN + && token + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_') +} + +/// Percent-encode a CID so it occupies exactly one path segment of `/ipfs/`. +/// `urlencoding::encode` escapes every byte outside the RFC 3986 unreserved set +/// (ALPHA / DIGIT / `-._~`), so the base64-CID characters that would otherwise +/// break the single-segment route — `/`, `+`, `=` — are all escaped, and the +/// server's `Path` extractor decodes the result back to the original CID (#173 +/// review, F1). +fn encode_cid_segment(cid: &str) -> String { + urlencoding::encode(cid).into_owned() +} + #[cfg(test)] mod tests { use super::*; + use std::cell::RefCell; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::{Arc, Mutex}; + use std::time::Instant; + + thread_local! { + /// Test-visible copy of the stderr diagnostics `diag` emits. `#[tokio::test]` + /// runs the future on the test's own thread, so a thread-local is enough. + static DIAG: RefCell = const { RefCell::new(String::new()) }; + } + + pub(super) fn record_diag(msg: &str) { + DIAG.with(|d| { + let mut d = d.borrow_mut(); + d.push_str(msg); + // A space, not a newline: the `has_control_or_bidi` assertions run over the + // whole telling, so a newline the harness inserts itself would make them + // fire on ANY stderr diagnostic and turn "node text is sanitized" into + // "nothing was printed to stderr", which R21 requires. A node-supplied + // control character is still caught. + d.push(' '); + }); + } + + fn reset_diag() { + DIAG.with(|d| d.borrow_mut().clear()); + } + + fn diag_text() -> String { + DIAG.with(|d| d.borrow().clone()) + } + + /// Everything the caller is told about a failed get: the stderr diagnostics plus + /// the error itself (main renders both). `{:#}` flattens the anyhow context chain + /// onto one line, so a message carried in a context layer is still covered. + fn told(err: &anyhow::Error) -> String { + format!("{}{err:#}", diag_text()) + } + + /// Width of a real continuation today (see the node's scan_token module): 756 + /// base64url-no-pad characters. The tests build tokens of that width so the + /// fixtures look like the wire, not like a placeholder. Nothing in this crate + /// seals a real token, so the number is pinned on the other side by + /// `token_length_is_invariant_across_oid_widths` in gitlawb-core's scan_token. + const TOKEN_LEN: usize = 756; + + fn token_of_len(seed: &str, len: usize) -> String { + let mut t = String::from(seed); + while t.len() < len { + t.push('A'); + } + t.truncate(len); + t + } + + fn make_token(seed: &str) -> String { + token_of_len(seed, TOKEN_LEN) + } + + /// The `scan` query value of an incoming request, if it carries one. + fn scan_of(path_and_query: &str) -> Option { + let (_, query) = path_and_query.split_once('?')?; + query + .split('&') + .find_map(|kv| kv.strip_prefix("scan=")) + .map(str::to_string) + } + + /// Derive the NEXT continuation from the one the client just echoed, so every + /// response on a ladder carries a different token (a real node re-seals with a + /// fresh nonce every time, and a fixed replayed body would hide that). + fn next_token(echoed: Option<&str>) -> String { + let n = echoed + .map(|t| { + t.trim_end_matches('A') + .trim_start_matches('t') + .parse::() + .unwrap_or(0) + }) + .unwrap_or(0); + make_token(&format!("t{}", n + 1)) + } + + /// A node message crafted to reach the terminal: a raw DEL (a control character + /// JSON permits unescaped), a JSON-escaped ESC/CSI sequence, a raw bidi override, + /// and a long tail so a missing length cap shows up. + fn hostile_msg() -> String { + format!("boom \u{7f} \\u001b[31m \u{202e} {}", "x".repeat(5000)) + } + + fn incomplete_body(continuation: Option<&str>, msg: &str) -> String { + match continuation { + Some(t) => { + format!(r#"{{"error":"search_incomplete","message":"{msg}","continuation":"{t}"}}"#) + } + None => format!(r#"{{"error":"search_incomplete","message":"{msg}"}}"#), + } + } + + fn has_control_or_bidi(s: &str) -> bool { + s.chars() + .any(|c| c.is_control() || gitlawb_core::sanitize::is_bidi_format(c)) + } + + /// A bare re-run restarts the scan at row 0 and re-spends the caller's per-IP + /// budget, so any "run it again" phrasing is only honest when the token that + /// makes progress is right there with it. + fn implies_bare_rerun(text: &str, token: &str) -> bool { + let lower = text.to_lowercase(); + [ + "try again", + "re-run", + "rerun", + "run it again", + "retry the command", + ] + .iter() + .any(|p| lower.contains(p)) + && !text.contains(token) + } /// Seed a keypair into a temp dir the way `load_keypair_from_dir` expects, /// then return the dir handle (keeps it alive for the test's duration). @@ -235,4 +728,1600 @@ mod tests { m.assert_async().await; } + + /// #173 (F5): `gl ipfs get` must SIGN with an available identity, like + /// `gl ipfs list`, so an owner/reader can retrieve a path-scoped object the node + /// now resolves by CID. RED before the fix: cmd_get ignores the identity dir and + /// sends an unsigned request, so the signature-matching mock is never hit + /// (cmd_get errors on the unmatched 501, and m.assert fails). GREEN after: the + /// signed request carries the RFC 9421 headers and is served 200. + #[tokio::test] + async fn test_cmd_get_signs_when_identity_present() { + let mut server = mockito::Server::new_async().await; + let keystore = seed_keystore(); + + let m = server + .mock("GET", "/ipfs/bafkreitestcid") + .match_header("signature", mockito::Matcher::Any) + .match_header("signature-input", mockito::Matcher::Any) + .with_status(200) + .with_header("content-type", "application/octet-stream") + .with_header("x-git-hash", "abc123") + .with_body("object bytes") + .create_async() + .await; + + cmd_get( + "bafkreitestcid".to_string(), + server.url(), + Some(keystore.path().to_path_buf()), + None, + ) + .await + .expect("signed get of a resolvable object should succeed"); + + m.assert_async().await; + } + + /// #173 (F5) must-not: a genuine anonymous denial must surface as an error, not + /// be masked as success. With no identity dir the request is unsigned; a 404 + /// from the node must produce an Err mentioning the status. + #[tokio::test] + async fn test_cmd_get_anonymous_denial_is_error() { + let mut server = mockito::Server::new_async().await; + + let m = server + .mock("GET", "/ipfs/bafkreidenied") + .with_status(404) + .with_header("content-type", "text/plain") + .with_body("no git object found") + .create_async() + .await; + + let err = cmd_get("bafkreidenied".to_string(), server.url(), None, None) + .await + .expect_err("a 404 denial must be an error, not masked success"); + assert!( + err.to_string().contains("404"), + "error should mention the status, got: {err}" + ); + + m.assert_async().await; + } + + // #173 (F3): a truncated legacy scan comes back as 503 `search_incomplete` with a + // sealed continuation token. The command must follow that token instead of + // dead-ending, under an attempt cap, a wall-clock deadline, and a clamped + // Retry-After, and every terminal that still holds a token must hand it back with + // the invocation that resumes from it. The ladder fixtures answer with + // `Retry-After: 0` so the clamped sleeps are zero and a nine-call ladder stays + // sub-second in real time. + + /// Scenario 1. A `search_incomplete` 503 carrying a valid continuation is resumed: + /// the second request repeats the CID with `?scan=` (percent-encoding is the + /// identity over the base64url alphabet, so the echo is byte-identical) and the + /// content it returns is written. + #[tokio::test] + async fn test_cmd_get_resumes_search_incomplete_with_continuation() { + reset_diag(); + let mut server = mockito::Server::new_async().await; + let t = make_token("t1"); + + let m1 = server + .mock("GET", "/ipfs/bafkreiresume") + .with_status(503) + .with_header("content-type", "application/json") + .with_header("retry-after", "0") + .with_body(incomplete_body(Some(&t), "scan truncated")) + .expect(1) + .create_async() + .await; + let m2 = server + .mock("GET", "/ipfs/bafkreiresume") + .match_query(mockito::Matcher::Exact(format!("scan={t}"))) + .with_status(200) + .with_header("content-type", "application/octet-stream") + .with_body("object bytes") + .expect(1) + .create_async() + .await; + + cmd_get("bafkreiresume".to_string(), server.url(), None, None) + .await + .expect("a search_incomplete 503 carrying a continuation must resume, not bail"); + + m1.assert_async().await; + m2.assert_async().await; + } + + /// Scenario 2. A node that keeps truncating stops at the attempt cap: 8 automatic + /// resumes after the initial request, 9 node calls in all. The give-up names the + /// incomplete result and the cap, and hands back the token still held with the + /// invocation that resumes from it. + #[tokio::test] + async fn test_cmd_get_resume_ladder_stops_at_attempt_cap() { + reset_diag(); + let mut server = mockito::Server::new_async().await; + let calls = Arc::new(AtomicUsize::new(0)); + let c = calls.clone(); + + let m = server + .mock("GET", mockito::Matcher::Any) + .with_status(503) + .with_header("content-type", "application/json") + .with_header("retry-after", "0") + .with_body_from_request(move |req| { + c.fetch_add(1, Ordering::SeqCst); + let next = next_token(scan_of(req.path_and_query()).as_deref()); + incomplete_body(Some(&next), "scan truncated").into_bytes() + }) + .expect(9) + .create_async() + .await; + + let err = cmd_get("bafkreicap".to_string(), server.url(), None, None) + .await + .expect_err("a ladder that never completes must end in an error"); + let told = told(&err); + let held = make_token("t9"); + + assert_eq!( + calls.load(Ordering::SeqCst), + 9, + "cap is 8 resumes after the initial request, so exactly 9 node calls" + ); + assert!( + told.to_lowercase().contains("incomplete"), + "the give-up must name the incomplete result, got: {told}" + ); + assert!( + told.contains(&format!("after {MAX_SCAN_RESUMES} automatic resumes")), + "the give-up must name the resume cap in words, got: {told}" + ); + assert!( + told.contains(&held), + "the still-held continuation must be surfaced, got: {told}" + ); + assert!( + told.contains(&format!("--scan {held}")), + "the exact resuming invocation must be surfaced, got: {told}" + ); + assert!( + !implies_bare_rerun(&told, &held), + "a bare re-run restarts at row 0, so the wording must not imply it helps: {told}" + ); + + m.assert_async().await; + } + + /// Scenario 3. A wedged node (every response a fresh token for the same position) + /// is indistinguishable from slow progress at the client, because tokens are + /// nonce-randomized ciphertext. The cap is what ends it, and that is the whole + /// assertion: the ladder stops at 9 calls with the explicit incomplete report. + #[tokio::test] + async fn test_cmd_get_wedged_ladder_still_stops_at_attempt_cap() { + reset_diag(); + let mut server = mockito::Server::new_async().await; + let calls = Arc::new(AtomicUsize::new(0)); + let c = calls.clone(); + + let m = server + .mock("GET", mockito::Matcher::Any) + .with_status(503) + .with_header("content-type", "application/json") + .with_header("retry-after", "0") + .with_body_from_request(move |_req| { + let n = c.fetch_add(1, Ordering::SeqCst) + 1; + // A distinct token every time, none of them advancing the cursor. + incomplete_body(Some(&make_token(&format!("w{n}"))), "scan truncated").into_bytes() + }) + .expect(9) + .create_async() + .await; + + let err = cmd_get("bafkreiwedged".to_string(), server.url(), None, None) + .await + .expect_err("a wedged ladder must end in an error"); + let told = told(&err); + + assert_eq!( + calls.load(Ordering::SeqCst), + 9, + "the cap is the only bound on a wedged ladder, so exactly 9 node calls" + ); + assert!( + told.to_lowercase().contains("incomplete"), + "the give-up must name the incomplete result, got: {told}" + ); + + m.assert_async().await; + } + + /// Scenario 4. A `search_incomplete` 503 with no continuation is terminal: there is + /// nothing to resume from, and the message says so rather than reporting a bare + /// status. Exactly one node call. + #[tokio::test] + async fn test_cmd_get_search_incomplete_without_continuation_is_terminal() { + reset_diag(); + let mut server = mockito::Server::new_async().await; + + let m1 = server + .mock("GET", "/ipfs/bafkreinotoken") + .with_status(503) + .with_header("content-type", "application/json") + .with_body(incomplete_body(None, "scan truncated")) + .expect(1) + .create_async() + .await; + let m2 = server + .mock("GET", mockito::Matcher::Regex("scan=".to_string())) + .expect(0) + .create_async() + .await; + + let err = cmd_get("bafkreinotoken".to_string(), server.url(), None, None) + .await + .expect_err("a truncation with no continuation must be an error"); + let told = told(&err); + + assert!( + told.to_lowercase().contains("continuation"), + "the terminal must name the missing continuation, got: {told}" + ); + assert!( + told.contains("503"), + "the terminal must still name the status, got: {told}" + ); + + m1.assert_async().await; + m2.assert_async().await; + } + + /// Scenario 5. An overload 503 on the FIRST request holds no token, so there is + /// nothing to resume: terminal, one call, and the node's text reaches the terminal + /// sanitized and length-capped rather than verbatim. + #[tokio::test] + async fn test_cmd_get_first_request_overload_503_is_terminal_and_sanitized() { + reset_diag(); + let mut server = mockito::Server::new_async().await; + + let m1 = server + .mock("GET", "/ipfs/bafkreioverload") + .with_status(503) + .with_header("content-type", "application/json") + .with_header("retry-after", "0") + .with_body(format!( + r#"{{"error":"overloaded","message":"{}"}}"#, + hostile_msg() + )) + .expect(1) + .create_async() + .await; + let m2 = server + .mock("GET", mockito::Matcher::Regex("scan=".to_string())) + .expect(0) + .create_async() + .await; + + let err = cmd_get("bafkreioverload".to_string(), server.url(), None, None) + .await + .expect_err("a first-request overload must be an error"); + let told = told(&err); + + assert!( + !has_control_or_bidi(&told), + "node text must be sanitized before it reaches the terminal, got: {told:?}" + ); + assert!( + told.chars().count() < 600, + "node text must be length-capped, got {} chars", + told.chars().count() + ); + assert!( + told.contains("503"), + "the terminal must name the status, got: {told}" + ); + + m1.assert_async().await; + m2.assert_async().await; + } + + /// Scenario 6. The resumed request is signed like the first one, and the signature + /// covers the query: the token joins the path binding before signing, so the mock + /// matching both the `scan=` query and the RFC 9421 headers is the one served. + #[tokio::test] + async fn test_cmd_get_resumed_request_is_signed() { + reset_diag(); + let mut server = mockito::Server::new_async().await; + let keystore = seed_keystore(); + let t = make_token("t1"); + + let m1 = server + .mock("GET", "/ipfs/bafkreisigned") + .match_header("signature", mockito::Matcher::Any) + .with_status(503) + .with_header("content-type", "application/json") + .with_header("retry-after", "0") + .with_body(incomplete_body(Some(&t), "scan truncated")) + .expect(1) + .create_async() + .await; + let m2 = server + .mock("GET", "/ipfs/bafkreisigned") + .match_query(mockito::Matcher::Exact(format!("scan={t}"))) + .match_header("signature", mockito::Matcher::Any) + .match_header("signature-input", mockito::Matcher::Any) + .with_status(200) + .with_header("content-type", "application/octet-stream") + .with_body("object bytes") + .expect(1) + .create_async() + .await; + + cmd_get( + "bafkreisigned".to_string(), + server.url(), + Some(keystore.path().to_path_buf()), + None, + ) + .await + .expect("the resumed request must be signed and served"); + + m1.assert_async().await; + m2.assert_async().await; + } + + /// Scenario 7. An oversized, hostile `search_incomplete` body still terminates + /// cleanly: the surfaced message is capped and free of control and bidi characters. + #[tokio::test] + async fn test_cmd_get_hostile_incomplete_body_is_capped_and_sanitized() { + reset_diag(); + let mut server = mockito::Server::new_async().await; + + let m = server + .mock("GET", "/ipfs/bafkreihostilebody") + .with_status(503) + .with_header("content-type", "application/json") + .with_body(incomplete_body(None, &hostile_msg())) + .expect(1) + .create_async() + .await; + + let err = cmd_get("bafkreihostilebody".to_string(), server.url(), None, None) + .await + .expect_err("a hostile truncation body must still be an error"); + let told = told(&err); + + assert!( + !has_control_or_bidi(&told), + "node text must be sanitized before it reaches the terminal, got: {told:?}" + ); + assert!( + told.chars().count() < 600, + "node text must be length-capped, got {} chars", + told.chars().count() + ); + + m.assert_async().await; + } + + /// Scenario 8. A continuation the node chose but that fails validation (`#`, a + /// newline, `&`) never enters the signed path: terminal exactly like a missing + /// token, no second request, and the rejected token is never echoed into the + /// message (the bound is named instead). + #[tokio::test] + async fn test_cmd_get_hostile_continuation_token_is_rejected() { + reset_diag(); + let mut server = mockito::Server::new_async().await; + + let m1 = server + .mock("GET", "/ipfs/bafkreihostiletoken") + .with_status(503) + .with_header("content-type", "application/json") + .with_body(format!( + r#"{{"error":"search_incomplete","message":"{}","continuation":"abc#\ndef&ghi"}}"#, + hostile_msg() + )) + .expect(1) + .create_async() + .await; + let m2 = server + .mock("GET", mockito::Matcher::Regex("scan=".to_string())) + .expect(0) + .create_async() + .await; + + let err = cmd_get("bafkreihostiletoken".to_string(), server.url(), None, None) + .await + .expect_err("a malformed continuation must be terminal"); + let told = told(&err); + + assert!( + !told.contains("abc#") && !told.contains("def&ghi"), + "a rejected token must never be echoed into the message, got: {told}" + ); + assert!( + !has_control_or_bidi(&told), + "node text must be sanitized before it reaches the terminal, got: {told:?}" + ); + assert!( + told.chars().count() < 600, + "node text must be length-capped, got {} chars", + told.chars().count() + ); + + m1.assert_async().await; + m2.assert_async().await; + } + + /// Scenario 9. A mid-ladder 429 is terminal: the fanout limiter's window is an + /// hour, so its Retry-After cannot be honored inside one invocation. The message + /// names rate limiting (not truncation), is sanitized and capped, and the token + /// still held comes back with the invocation that resumes from it. + #[tokio::test] + async fn test_cmd_get_mid_ladder_429_is_terminal_and_surfaces_token() { + reset_diag(); + let mut server = mockito::Server::new_async().await; + let t = make_token("t1"); + + let m1 = server + .mock("GET", "/ipfs/bafkreithrottled") + .with_status(503) + .with_header("content-type", "application/json") + .with_header("retry-after", "0") + .with_body(incomplete_body(Some(&t), "scan truncated")) + .expect(1) + .create_async() + .await; + let m2 = server + .mock("GET", "/ipfs/bafkreithrottled") + .match_query(mockito::Matcher::Exact(format!("scan={t}"))) + .with_status(429) + .with_header("content-type", "application/json") + .with_header("retry-after", "3600") + .with_body(format!( + r#"{{"error":"rate_limited","message":"{}"}}"#, + hostile_msg() + )) + .expect(1) + .create_async() + .await; + + let err = cmd_get("bafkreithrottled".to_string(), server.url(), None, None) + .await + .expect_err("a mid-ladder 429 must be an error"); + let told = told(&err); + + assert!( + told.to_lowercase().contains("rate limit"), + "the terminal must name rate limiting, distinct from the truncation wording, got: {told}" + ); + assert!( + !has_control_or_bidi(&told), + "node text must be sanitized before it reaches the terminal, got: {told:?}" + ); + // The length bound is scoped to the error text, not to `told`: R21 requires a + // stderr line carrying the still-held 756-character token, so no implementation + // can keep the whole telling under 600 characters. The error text is where an + // uncapped node body would land on this path, so the property still binds. + let reported = format!("{err:#}"); + assert!( + reported.chars().count() < 600, + "node text must be length-capped, got {} chars", + reported.chars().count() + ); + assert!( + told.contains(&t) && told.contains(&format!("--scan {t}")), + "the still-held continuation and its resuming invocation must be surfaced, got: {told}" + ); + + m1.assert_async().await; + m2.assert_async().await; + } + + /// Scenario 10. A mid-ladder overload 503 (no `search_incomplete` code, token still + /// held) is retried, not terminal: its three sources are transient, the node itself + /// says to retry shortly, and nothing accumulates per IP on that path. The ladder + /// continues on the same token and completes in three calls. + #[tokio::test] + async fn test_cmd_get_mid_ladder_overload_503_is_retried() { + reset_diag(); + let mut server = mockito::Server::new_async().await; + let t = make_token("t1"); + + let m1 = server + .mock("GET", "/ipfs/bafkreimidoverload") + .with_status(503) + .with_header("content-type", "application/json") + .with_header("retry-after", "0") + .with_body(incomplete_body(Some(&t), "scan truncated")) + .expect(1) + .create_async() + .await; + // Both of the next two match `scan=T`; mockito serves the first one that still + // has hits outstanding, so registration order sequences the overload then the + // success. + let m2 = server + .mock("GET", "/ipfs/bafkreimidoverload") + .match_query(mockito::Matcher::Exact(format!("scan={t}"))) + .with_status(503) + .with_header("content-type", "application/json") + .with_header("retry-after", "0") + .with_body(r#"{"error":"overloaded","message":"busy, retry shortly"}"#) + .expect(1) + .create_async() + .await; + let m3 = server + .mock("GET", "/ipfs/bafkreimidoverload") + .match_query(mockito::Matcher::Exact(format!("scan={t}"))) + .with_status(200) + .with_header("content-type", "application/octet-stream") + .with_body("object bytes") + .expect(1) + .create_async() + .await; + + cmd_get("bafkreimidoverload".to_string(), server.url(), None, None) + .await + .expect("a mid-ladder overload must be retried on the held token, not terminal"); + + m1.assert_async().await; + m2.assert_async().await; + m3.assert_async().await; + } + + /// Scenario 10b. The classification default arm with a token ALREADY HELD. Every + /// other fixture reaches that arm token-less (scenario 5's first-request overload) + /// or never reaches it at all (scenario 9's 429 short-circuits on the status), so + /// the arm's terminality was certified only for the case where there was nothing + /// to resume with anyway. Here an unknown code arrives mid-ladder on a 500, which + /// is not the overload status, with a valid token in hand: it must still be + /// terminal. Two calls, and the fixture stops well short of the cap so a + /// misclassified retry shows up as a call count rather than a cap give-up. + #[tokio::test] + async fn test_cmd_get_mid_ladder_unknown_code_is_terminal_with_token_held() { + reset_diag(); + let mut server = mockito::Server::new_async().await; + let t = make_token("t1"); + let calls = Arc::new(AtomicUsize::new(0)); + let c1 = calls.clone(); + let c2 = calls.clone(); + + let m1 = server + .mock("GET", "/ipfs/bafkreiunknowncode") + .with_status(503) + .with_header("content-type", "application/json") + .with_header("retry-after", "0") + .with_body_from_request({ + let t = t.clone(); + move |_req| { + c1.fetch_add(1, Ordering::SeqCst); + incomplete_body(Some(&t), "scan truncated").into_bytes() + } + }) + .expect(1) + .create_async() + .await; + let m2 = server + .mock("GET", "/ipfs/bafkreiunknowncode") + .match_query(mockito::Matcher::Exact(format!("scan={t}"))) + .with_status(500) + .with_header("content-type", "application/json") + .with_body_from_request(move |_req| { + c2.fetch_add(1, Ordering::SeqCst); + br#"{"error":"index_corrupt","message":"scan index unreadable"}"#.to_vec() + }) + .expect(1) + .create_async() + .await; + + let err = cmd_get("bafkreiunknowncode".to_string(), server.url(), None, None) + .await + .expect_err("an unknown code mid-ladder must be an error"); + let told = told(&err); + + assert_eq!( + calls.load(Ordering::SeqCst), + 2, + "an unrecognized code with a token held must be terminal, not retried" + ); + assert!( + told.contains("500"), + "the terminal must name the status, got: {told}" + ); + // Terminal is only half of it. The ladder stopped holding a token that still + // points at a real position, and without it the caller's only recourse is a + // bare re-run that restarts at row 0 and re-spends the per-IP budget. Every + // terminal that holds a usable token must hand it back. + assert!( + told.contains(&t) && told.contains(&format!("--scan {t}")), + "the still-held continuation and its resuming invocation must be surfaced \ + on a mid-ladder terminal, got: {told}" + ); + + m1.assert_async().await; + m2.assert_async().await; + } + + /// Scenario 10c, the other reachable terminal that holds a token: rung 1 offers a + /// valid continuation, rung 2 answers `search_incomplete` with a MALFORMED one. + /// + /// The offered token is unusable and must never be echoed, but the token the client + /// already HOLDS is untouched by that rejection and still points at where the scan + /// stopped, so it is what must come back. Distinct from scenario 4, where the node + /// offers nothing at all: that is its deliberate "the scan wrapped and finished" + /// signal and carries no resume hint. + #[tokio::test] + async fn test_cmd_get_rejected_offered_token_still_surfaces_the_held_one() { + reset_diag(); + let mut server = mockito::Server::new_async().await; + let t = make_token("t1"); + let malformed = "abc#def&ghi"; + let calls = Arc::new(AtomicUsize::new(0)); + let c1 = calls.clone(); + let c2 = calls.clone(); + + let m1 = server + .mock("GET", "/ipfs/bafkreirejectedoffer") + .with_status(503) + .with_header("content-type", "application/json") + .with_header("retry-after", "0") + .with_body_from_request({ + let t = t.clone(); + move |_req| { + c1.fetch_add(1, Ordering::SeqCst); + incomplete_body(Some(&t), "scan truncated").into_bytes() + } + }) + .expect(1) + .create_async() + .await; + let m2 = server + .mock("GET", "/ipfs/bafkreirejectedoffer") + .match_query(mockito::Matcher::Exact(format!("scan={t}"))) + .with_status(503) + .with_header("content-type", "application/json") + .with_header("retry-after", "0") + .with_body_from_request(move |_req| { + c2.fetch_add(1, Ordering::SeqCst); + incomplete_body(Some(malformed), "scan truncated").into_bytes() + }) + .expect(1) + .create_async() + .await; + + let err = cmd_get("bafkreirejectedoffer".to_string(), server.url(), None, None) + .await + .expect_err("a malformed offered continuation must be terminal"); + let told = told(&err); + + assert_eq!( + calls.load(Ordering::SeqCst), + 2, + "a rejected offer is terminal, so exactly two node calls" + ); + assert!( + told.contains(&t) && told.contains(&format!("--scan {t}")), + "the still-held continuation and its resuming invocation must be surfaced, \ + got: {told}" + ); + assert!( + !told.contains("abc#") && !told.contains("def&ghi"), + "a rejected token must never be echoed into the message, got: {told}" + ); + + m1.assert_async().await; + m2.assert_async().await; + } + + /// Scenario 11. The wall-clock deadline bounds the whole loop, and it bounds each + /// attempt's own timeout. This drives the give-up tail, where the ladder never + /// reaches a body read, so its bound is the deadline plus one clamped wait (a + /// stalled body composes differently; see the note in `cmd_get_inner`). Injected + /// through the seam because the shipped 60s is unreachable + /// under the 5s clamp and 8 resumes. + #[tokio::test] + async fn test_cmd_get_resume_ladder_stops_at_wall_clock_deadline() { + reset_diag(); + let mut server = mockito::Server::new_async().await; + let calls = Arc::new(AtomicUsize::new(0)); + let last = Arc::new(Mutex::new(String::new())); + let c = calls.clone(); + let l = last.clone(); + + let m = server + .mock("GET", mockito::Matcher::Any) + .with_status(503) + .with_header("content-type", "application/json") + .with_header("retry-after", "1") + .with_body_from_request(move |req| { + c.fetch_add(1, Ordering::SeqCst); + let next = next_token(scan_of(req.path_and_query()).as_deref()); + *l.lock().unwrap() = next.clone(); + incomplete_body(Some(&next), "scan truncated").into_bytes() + }) + .expect_at_least(2) + .create_async() + .await; + + let started = Instant::now(); + let err = cmd_get_inner( + "bafkreislowscan".to_string(), + server.url(), + None, + None, + Duration::from_millis(2500), + MAX_SCAN_RESUMES, + ) + .await + .expect_err("a ladder that outruns the deadline must end in an error"); + let elapsed = started.elapsed(); + let told = told(&err); + let held = last.lock().unwrap().clone(); + + assert!( + told.to_lowercase().contains("deadline"), + "the give-up must name the deadline, not the cap, got: {told}" + ); + let calls = calls.load(Ordering::SeqCst); + // The lower bound is 1, not 2. What this scenario proves is that the DEADLINE, + // not the cap, is what ends the ladder, and one call satisfies that as well as + // three do; requiring a resume as well made the test depend on a loaded runner + // fitting two round trips inside 2.5s, which is the likeliest flake in the + // suite. That a valid continuation is actually resumed with is scenario 1's job. + assert!( + (1..9).contains(&calls), + "the deadline must stop the ladder before the cap, made {calls} calls" + ); + assert!( + elapsed < Duration::from_secs(9), + "the ladder never reaches a body read here, and every wait is bounded by the \ + time left on the 2.5s deadline, so the run is over near the deadline itself; \ + took {elapsed:?}" + ); + assert!( + told.contains(&held) && told.contains(&format!("--scan {held}")), + "the still-held continuation and its resuming invocation must be surfaced, got: {told}" + ); + + m.assert_async().await; + } + + /// Scenario 12, lower half of the boundary pair: a 2048-character token is inside + /// the accepted bound and is resumed with. + #[tokio::test] + async fn test_cmd_get_accepts_2048_char_continuation() { + reset_diag(); + let mut server = mockito::Server::new_async().await; + let t = token_of_len("t1", 2048); + + let m1 = server + .mock("GET", "/ipfs/bafkreibound") + .with_status(503) + .with_header("content-type", "application/json") + .with_header("retry-after", "0") + .with_body(incomplete_body(Some(&t), "scan truncated")) + .expect(1) + .create_async() + .await; + let m2 = server + .mock("GET", "/ipfs/bafkreibound") + .match_query(mockito::Matcher::Exact(format!("scan={t}"))) + .with_status(200) + .with_header("content-type", "application/octet-stream") + .with_body("object bytes") + .expect(1) + .create_async() + .await; + + cmd_get("bafkreibound".to_string(), server.url(), None, None) + .await + .expect("a 2048-character token is within the bound and must be resumed with"); + + m1.assert_async().await; + m2.assert_async().await; + } + + /// Scenario 12, upper half: one character past the bound is rejected, terminal + /// exactly like a missing token, and never echoed back. + #[tokio::test] + async fn test_cmd_get_rejects_2049_char_continuation() { + reset_diag(); + let mut server = mockito::Server::new_async().await; + let t = token_of_len("t1", 2049); + + let m1 = server + .mock("GET", "/ipfs/bafkreioverbound") + .with_status(503) + .with_header("content-type", "application/json") + .with_body(incomplete_body(Some(&t), "scan truncated")) + .expect(1) + .create_async() + .await; + let m2 = server + .mock("GET", mockito::Matcher::Regex("scan=".to_string())) + .expect(0) + .create_async() + .await; + + let err = cmd_get("bafkreioverbound".to_string(), server.url(), None, None) + .await + .expect_err("an over-bound token must be terminal"); + let told = told(&err); + + assert!( + !told.contains(&t), + "a rejected token must never be echoed into the message, got: {told}" + ); + assert!( + told.chars().count() < 600, + "node text must be length-capped, got {} chars", + told.chars().count() + ); + + m1.assert_async().await; + m2.assert_async().await; + } + + /// Scenario 13. A caller-supplied continuation is a resume INPUT: the very first + /// request carries `?scan=`, so an invocation picked up from a previous + /// terminal starts where that one stopped instead of walking from row 0 again. + #[tokio::test] + async fn test_cmd_get_caller_supplied_continuation_starts_from_token() { + reset_diag(); + let mut server = mockito::Server::new_async().await; + let t = make_token("t7"); + + let front = server + .mock("GET", "/ipfs/bafkreisupplied") + .expect(0) + .create_async() + .await; + let resumed = server + .mock("GET", "/ipfs/bafkreisupplied") + .match_query(mockito::Matcher::Exact(format!("scan={t}"))) + .with_status(200) + .with_header("content-type", "application/octet-stream") + .with_body("object bytes") + .expect(1) + .create_async() + .await; + + let res = cmd_get_inner( + "bafkreisupplied".to_string(), + server.url(), + None, + Some(t.clone()), + SCAN_DEADLINE, + MAX_SCAN_RESUMES, + ) + .await; + + front.assert_async().await; + resumed.assert_async().await; + res.expect("a supplied continuation must be used, not ignored"); + } + + /// R21, the wired half. The scenario above drives `cmd_get_inner` directly, so + /// it proves the resume INPUT works but says nothing about the `--scan` arg + /// reaching it. This one goes through `cmd_get`, the function clap dispatches + /// to, so a rewiring that drops the argument on the floor turns it red. Without + /// it the flag can be silently disconnected while every other resume test stays + /// green, and the invocation this command prints at a bound would be advice the + /// binary does not honor. + #[tokio::test] + async fn test_cmd_get_passes_the_scan_arg_through_to_the_resume_input() { + reset_diag(); + let mut server = mockito::Server::new_async().await; + let t = make_token("t14"); + + let front = server + .mock("GET", "/ipfs/bafkreiwired") + .expect(0) + .create_async() + .await; + let resumed = server + .mock("GET", "/ipfs/bafkreiwired") + .match_query(mockito::Matcher::Exact(format!("scan={t}"))) + .with_status(200) + .with_header("content-type", "application/octet-stream") + .with_body("object bytes") + .expect(1) + .create_async() + .await; + + let res = cmd_get( + "bafkreiwired".to_string(), + server.url(), + None, + Some(t.clone()), + ) + .await; + + front.assert_async().await; + resumed.assert_async().await; + res.expect("the --scan argument must reach the resume input through cmd_get"); + } + + /// #173 review (F1): a base64 CID (multibase prefix 'm') can contain '/', '+', + /// and '='. The client must percent-encode it into ONE path segment before + /// building and signing `/ipfs/`; otherwise the '/' splits the target so + /// it misses the single-segment Axum route and the signature covers the wrong + /// path. Assert the encoded segment carries no raw '/', '+', or '=', and that + /// it decodes back to the original CID (the server's `Path` extractor performs + /// that same decode). RED with the old raw `format!("/ipfs/{cid}")`: the + /// segment still contains '/'. + #[test] + fn test_encode_cid_segment_escapes_base64_alphabet() { + let cid = "mFoo/Bar+baz=="; + let encoded = encode_cid_segment(cid); + + assert!( + !encoded.contains('/'), + "encoded CID must be a single path segment (no raw '/'), got: {encoded}" + ); + assert!( + !encoded.contains('+'), + "encoded CID must escape '+', got: {encoded}" + ); + assert!( + !encoded.contains('='), + "encoded CID must escape '=', got: {encoded}" + ); + + let decoded = urlencoding::decode(&encoded).expect("encoded CID must decode"); + assert_eq!( + decoded, cid, + "encoding must round-trip back to the original CID" + ); + } + + /// #173 review: `gl ipfs get --dir ` must PROPAGATE a missing/corrupt + /// identity-load error like `gl ipfs list`, not silently fall back to an anonymous + /// request — otherwise an authorized reader pointing `--dir` at a broken keystore + /// gets the node's opaque 404 instead of the actionable key-load error. The + /// unsigned fallback is preserved only when NO `--dir` is given (covered by + /// `test_cmd_get_anonymous_denial_is_error`). RED before the fix (`.ok()` swallows + /// the error, an anonymous request is sent, and the `.expect(0)` mock is hit), + /// GREEN after. + #[tokio::test] + async fn test_cmd_get_explicit_dir_no_identity_errors_without_request() { + let mut server = mockito::Server::new_async().await; + // Empty keystore dir passed explicitly via --dir: no identity.pem present. + let empty = tempfile::TempDir::new().unwrap(); + + // The endpoint must never be hit when an explicit --dir fails to load. + let m = server + .mock("GET", "/ipfs/bafkreitestcid") + .expect(0) + .create_async() + .await; + + let err = cmd_get( + "bafkreitestcid".to_string(), + server.url(), + Some(empty.path().to_path_buf()), + None, + ) + .await + .expect_err("an explicit --dir that fails to load must be an error"); + assert!( + err.to_string().contains("gl identity new") + || err.to_string().contains("no identity found") + || err.to_string().contains("failed to load keypair"), + "error should name the key-load failure, got: {err}" + ); + + m.assert_async().await; + } + + /// #173 review (F4): a caller-supplied `--scan` value clears the same bar as a + /// node-offered one, BEFORE any request is signed. Both existing caller-supplied + /// scenarios pass a valid token, so the reject arm of that match was uncovered and + /// deleting the check left every test green even though the identical property is + /// covered on the node-offered side. A malformed value must fail with no node call + /// at all, and the rejection names the bound rather than echoing the value. + #[tokio::test] + async fn test_cmd_get_rejects_a_malformed_caller_supplied_continuation() { + reset_diag(); + let mut server = mockito::Server::new_async().await; + let bad = "abc#def&ghi"; + + // Nothing may be sent: the value would otherwise reach a signed target. + let m = server + .mock("GET", mockito::Matcher::Any) + .expect(0) + .create_async() + .await; + + let err = cmd_get_inner( + "bafkreibadinput".to_string(), + server.url(), + None, + Some(bad.to_string()), + SCAN_DEADLINE, + MAX_SCAN_RESUMES, + ) + .await + .expect_err("a malformed --scan value must be rejected"); + let told = told(&err); + + assert!( + told.contains(&MAX_CONTINUATION_LEN.to_string()) + && told.to_lowercase().contains("base64url"), + "the rejection must name the bound, got: {told}" + ); + assert!( + !told.contains("abc#") && !told.contains("def&ghi"), + "a rejected value must never be echoed back, got: {told}" + ); + + m.assert_async().await; + } + + /// #173 review (F3): the `Retry-After` clamp must actually bind somewhere. Every + /// other retryable fixture answers `Retry-After: 0` or `1`, both already under the + /// 5 second clamp, and the one 3600 in the suite rides a 429 that returns before + /// the header is ever parsed. So deleting `.min(MAX_RETRY_AFTER)` left the whole + /// suite green. + /// + /// Here a retryable 503 asks for an hour, with a valid continuation, under a + /// deadline set a little wider than the clamp. Clamped, the first wait is 5 + /// seconds and the deadline still has room for a second attempt. Unclamped, that + /// one wait consumes the whole deadline and the run ends after a single call. The + /// call count is what separates them, and it fails fast rather than hanging, + /// because the wait is also bounded by the time left on the deadline. + #[tokio::test] + async fn test_cmd_get_clamps_a_hostile_retry_after_below_the_deadline() { + reset_diag(); + let mut server = mockito::Server::new_async().await; + let calls = Arc::new(AtomicUsize::new(0)); + let c = calls.clone(); + + let m = server + .mock("GET", mockito::Matcher::Any) + .with_status(503) + .with_header("content-type", "application/json") + .with_header("retry-after", "3600") + .with_body_from_request(move |req| { + c.fetch_add(1, Ordering::SeqCst); + let next = next_token(scan_of(req.path_and_query()).as_deref()); + incomplete_body(Some(&next), "scan truncated").into_bytes() + }) + .expect_at_least(1) + .create_async() + .await; + + let started = Instant::now(); + let err = cmd_get_inner( + "bafkreihostileretry".to_string(), + server.url(), + None, + None, + Duration::from_secs(6), + MAX_SCAN_RESUMES, + ) + .await + .expect_err("a ladder that outruns the deadline must end in an error"); + let elapsed = started.elapsed(); + let calls = calls.load(Ordering::SeqCst); + + assert!( + calls >= 2, + "the clamp caps a single wait at {}s, well under the 6s deadline, so one \ + hostile Retry-After must not swallow the run: made {calls} calls", + MAX_RETRY_AFTER.as_secs() + ); + // Tight enough to bind. The waits are also clamped by the time LEFT on the + // deadline, and at 12s that term was free: dropping `.min(left)` let the run + // overshoot to 10.04s and still pass, so the doc claim that waits never run + // past the deadline rested on a term no test could see. With a 6s deadline + // and a 5s clamp the bounded run lands near 6s and the unbounded one near + // 10s, and 8s separates them. + assert!( + elapsed < Duration::from_secs(8), + "a wait is bounded by the time LEFT on the 6s deadline as well as by the \ + {}s clamp, so the run ends near the deadline rather than a full clamp \ + past it; took {elapsed:?}", + MAX_RETRY_AFTER.as_secs() + ); + assert!( + told(&err).to_lowercase().contains("deadline"), + "the give-up must name the deadline, got: {}", + told(&err) + ); + + m.assert_async().await; + } + + /// The transport-error terminal, which is the one arm of the token surfacing that + /// mockito cannot reach: its server outlives the call, and an unmatched route + /// answers 501, so a request always gets a response. + /// + /// A raw listener is what reproduces it. Rung 1 is a real `search_incomplete` 503 + /// carrying a valid continuation, answered with `Connection: close` so reqwest + /// opens a fresh connection for rung 2. Rung 2 is ACCEPTED and then dropped + /// without a byte written, which is what a reset mid-ladder looks like to the + /// client. The ladder ends holding a token that still points at a real position, + /// and losing it there means the only way forward is a bare re-run that restarts + /// at row 0 and re-spends the caller's per-IP budget. + /// + /// MUTATION (RED): drop the `surface_resume` call in the transport-error arm. + #[tokio::test] + async fn test_cmd_get_transport_failure_mid_ladder_surfaces_the_held_token() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + reset_diag(); + let t = make_token("t1"); + let body = incomplete_body(Some(&t), "scan truncated"); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let connections = Arc::new(AtomicUsize::new(0)); + let seen = connections.clone(); + + tokio::spawn(async move { + // Rung 1: a complete 503 with a continuation, then close the connection so + // rung 2 has to dial again. + let (mut sock, _) = listener.accept().await.unwrap(); + seen.fetch_add(1, Ordering::SeqCst); + let mut scratch = [0u8; 2048]; + let _ = sock.read(&mut scratch).await; + let resp = format!( + "HTTP/1.1 503 Service Unavailable\r\nContent-Type: application/json\r\n\ + Retry-After: 0\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + let _ = sock.write_all(resp.as_bytes()).await; + let _ = sock.flush().await; + drop(sock); + + // Rung 2: accept and hang up without a response. + let (sock, _) = listener.accept().await.unwrap(); + seen.fetch_add(1, Ordering::SeqCst); + drop(sock); + }); + + let err = cmd_get( + "bafkreireset".to_string(), + format!("http://{addr}"), + None, + None, + ) + .await + .expect_err("a connection dropped mid-ladder must be an error"); + let told = told(&err); + + assert_eq!( + connections.load(Ordering::SeqCst), + 2, + "the fixture must actually reach rung 2, or the transport arm was never \ + exercised" + ); + assert!( + told.contains(&t) && told.contains(&format!("--scan {t}")), + "a transport failure ends the ladder still holding a usable token, so it \ + must come back with the invocation that resumes from it, got: {told}" + ); + assert!( + told.contains("bafkreireset"), + "the failure must still name the CID it was fetching, got: {told}" + ); + } + + /// A terminal whose body FAILED to arrive must say so, not report the node as + /// silent. + /// + /// The listener answers 500, promises 512 bytes, writes none, and hangs up. The + /// read comes back empty, and the terminal used to render that as + /// `node returned 500: ` with nothing after the colon, which reads as a node that + /// sent no message at all. MUTATION (RED): render the tail with + /// `sanitize_node_msg` again and the message ends at the colon. + #[tokio::test] + async fn test_cmd_get_reports_a_body_that_could_not_be_read() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + reset_diag(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + let (mut sock, _) = listener.accept().await.unwrap(); + let mut scratch = [0u8; 2048]; + let _ = sock.read(&mut scratch).await; + let _ = sock + .write_all( + b"HTTP/1.1 500 Internal Server Error\r\nContent-Type: application/json\r\n\ + Content-Length: 512\r\nConnection: close\r\n\r\n", + ) + .await; + let _ = sock.flush().await; + }); + + let err = cmd_get( + "bafkreicutread".to_string(), + format!("http://{addr}"), + None, + None, + ) + .await + .expect_err("a 500 is an error whatever became of its body"); + let told = told(&err); + + assert!( + told.contains("500"), + "the terminal must still name the status, got: {told}" + ); + assert!( + told.to_lowercase().contains("could not be read"), + "a body that failed mid-read must be reported as unread rather than as an \ + empty message, got: {told}" + ); + assert!( + !told.contains("500: \n") && !told.ends_with("500: "), + "the terminal must not trail off after the colon, got: {told}" + ); + } + + /// #173 review (F9): a `search_incomplete` body the 8 KiB read cap CUT SHORT must + /// be terminal, not retried. + /// + /// A cut body cannot parse, so its `error` code reads as absent, and on a 503 that + /// used to fall through to the generic overload arm, which resumes on the token + /// ALREADY HELD. The fresh continuation the node offered is inside the part that + /// was never read, so the ladder replays one position for every remaining rung: a + /// 9000-character body drove eight requests carrying the old token. Unclassifiable + /// is terminal, like any unrecognized code. + #[tokio::test] + async fn test_cmd_get_truncated_incomplete_body_is_terminal_not_a_replay() { + reset_diag(); + let mut server = mockito::Server::new_async().await; + let t = make_token("t1"); + let calls = Arc::new(AtomicUsize::new(0)); + let c1 = calls.clone(); + let c2 = calls.clone(); + + let m1 = server + .mock("GET", "/ipfs/bafkreicutbody") + .with_status(503) + .with_header("content-type", "application/json") + .with_header("retry-after", "0") + .with_body_from_request({ + let t = t.clone(); + move |_req| { + c1.fetch_add(1, Ordering::SeqCst); + incomplete_body(Some(&t), "scan truncated").into_bytes() + } + }) + .expect(1) + .create_async() + .await; + // Well past the 8 KiB cap, with the fresh continuation behind the cut. + let m2 = server + .mock("GET", "/ipfs/bafkreicutbody") + .match_query(mockito::Matcher::Exact(format!("scan={t}"))) + .with_status(503) + .with_header("content-type", "application/json") + .with_header("retry-after", "0") + .with_body_from_request(move |_req| { + c2.fetch_add(1, Ordering::SeqCst); + incomplete_body(Some(&make_token("t2")), &"x".repeat(9000)).into_bytes() + }) + .expect(1) + .create_async() + .await; + + let err = cmd_get("bafkreicutbody".to_string(), server.url(), None, None) + .await + .expect_err("an unclassifiable 503 body must be an error"); + let told = told(&err); + + assert_eq!( + calls.load(Ordering::SeqCst), + 2, + "a body cut by the read cap must end the ladder, not replay the held token \ + for every remaining rung" + ); + assert!( + told.contains(&t) && told.contains(&format!("--scan {t}")), + "the still-held continuation and its resuming invocation must be surfaced, \ + got: {told}" + ); + + m1.assert_async().await; + m2.assert_async().await; + } + + /// The other half of the same defect: a `search_incomplete` 503 whose body read + /// FAILED part-way is just as unparseable as one the cap cut short, and it used to + /// fall through to the generic overload arm and be retried on the token ALREADY + /// HELD. Measured before the fix: rung 1 hands back t1, every later rung answers + /// headers plus a cut body, and the ladder made 9 calls with calls 2 through 9 all + /// carrying the identical `?scan=t1`, ending at the cap. That is the replay the + /// truncation exclusion exists to prevent, reached by the other door. + /// + /// mockito cannot express it: it always finishes the response it advertises. The + /// listener promises 512 bytes, writes a handful, and hangs up. + /// + /// MUTATION (RED): drop `&& !read_failed` from the retry arm and the count is 9. + #[tokio::test] + async fn test_cmd_get_unreadable_incomplete_body_is_terminal_not_a_replay() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + reset_diag(); + let t = make_token("t1"); + let complete = incomplete_body(Some(&t), "scan truncated"); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let calls = Arc::new(AtomicUsize::new(0)); + let seen = calls.clone(); + let scans = Arc::new(Mutex::new(Vec::::new())); + let recorded = scans.clone(); + + tokio::spawn(async move { + loop { + let Ok((mut sock, _)) = listener.accept().await else { + return; + }; + let n = seen.fetch_add(1, Ordering::SeqCst); + let mut scratch = [0u8; 4096]; + let read = sock.read(&mut scratch).await.unwrap_or(0); + let request = String::from_utf8_lossy(&scratch[..read]).into_owned(); + if let Some(line) = request.lines().next() { + if let Some(target) = line.split_whitespace().nth(1) { + recorded + .lock() + .unwrap() + .push(scan_of(target).unwrap_or_default()); + } + } + let resp = if n == 0 { + // Rung 1: a complete 503 offering a continuation. + format!( + "HTTP/1.1 503 Service Unavailable\r\nContent-Type: application/json\r\n\ + Retry-After: 0\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{complete}", + complete.len() + ) + } else { + // Every later rung: headers, then a body that stops part-way. + "HTTP/1.1 503 Service Unavailable\r\nContent-Type: application/json\r\n\ + Retry-After: 0\r\nContent-Length: 512\r\nConnection: close\r\n\r\n\ + {\"error\":\"search_inc" + .to_string() + }; + let _ = sock.write_all(resp.as_bytes()).await; + let _ = sock.flush().await; + drop(sock); + } + }); + + let err = cmd_get_inner( + "bafkreiunreadable".to_string(), + format!("http://{addr}"), + None, + None, + SCAN_DEADLINE, + MAX_SCAN_RESUMES, + ) + .await + .expect_err("an unclassifiable 503 body must be an error"); + let told = told(&err); + + assert_eq!( + calls.load(Ordering::SeqCst), + 2, + "a body whose read failed must end the ladder, not replay the held token \ + for every remaining rung; the scans seen were {:?}", + scans.lock().unwrap() + ); + assert_eq!( + scans.lock().unwrap().as_slice(), + [String::new(), t.clone()], + "rung 1 carries no token and rung 2 carries the one it was handed" + ); + assert!( + told.contains(&t) && told.contains(&format!("--scan {t}")), + "the still-held continuation and its resuming invocation must be \ + surfaced, got: {told}" + ); + } + + /// A 404 after a resume is an ANSWER, so it must not come with a resume hint that + /// contradicts it. Deleting the `status != NOT_FOUND` guard (replacing it with + /// `if true`) left the suite green, because no test had ever reached that arm + /// holding a token, which is the only state in which the guard does anything. + /// + /// Rung 1 hands back a valid continuation, rung 2 answers 404. + /// + /// MUTATION (RED): replace the guard with `if true`. + #[tokio::test] + async fn test_cmd_get_a_404_after_a_resume_offers_no_hint() { + reset_diag(); + let mut server = mockito::Server::new_async().await; + let t = make_token("t1"); + + let rung1 = server + .mock("GET", "/ipfs/bafkreignotfound") + .match_query(mockito::Matcher::Missing) + .with_status(503) + .with_header("content-type", "application/json") + .with_header("retry-after", "0") + .with_body(incomplete_body(Some(&t), "scan truncated")) + .expect(1) + .create_async() + .await; + let rung2 = server + .mock("GET", "/ipfs/bafkreignotfound") + .match_query(mockito::Matcher::Exact(format!("scan={t}"))) + .with_status(404) + .with_header("content-type", "application/json") + .with_body(r#"{"error":"not_found","message":"no such object"}"#) + .expect(1) + .create_async() + .await; + + let err = cmd_get("bafkreignotfound".to_string(), server.url(), None, None) + .await + .expect_err("a 404 is still an error exit"); + let told = told(&err); + + assert!( + told.contains("404"), + "the terminal must name the status, got: {told}" + ); + assert!( + !told.contains("--scan") && !told.contains(&t), + "a definitive 404 is an answer; a resume hint beside it would invite a \ + re-run that cannot do better, got: {told}" + ); + + rung1.assert_async().await; + rung2.assert_async().await; + } + + /// `search_incomplete` with NO continuation is the node's "the scan wrapped and + /// finished" signal, so that arm deliberately withholds the hint too. Adding a + /// `surface_resume` call to it left the suite green for the same reason: nothing + /// reached it holding a token. + /// + /// MUTATION (RED): add `surface_resume(&cid, token.as_deref());` to the + /// no-continuation branch. + #[tokio::test] + async fn test_cmd_get_a_wrapped_scan_after_a_resume_offers_no_hint() { + reset_diag(); + let mut server = mockito::Server::new_async().await; + let t = make_token("t1"); + + let rung1 = server + .mock("GET", "/ipfs/bafkreigwrapped") + .match_query(mockito::Matcher::Missing) + .with_status(503) + .with_header("content-type", "application/json") + .with_header("retry-after", "0") + .with_body(incomplete_body(Some(&t), "scan truncated")) + .expect(1) + .create_async() + .await; + let rung2 = server + .mock("GET", "/ipfs/bafkreigwrapped") + .match_query(mockito::Matcher::Exact(format!("scan={t}"))) + .with_status(503) + .with_header("content-type", "application/json") + .with_header("retry-after", "0") + .with_body(incomplete_body(None, "the scan wrapped")) + .expect(1) + .create_async() + .await; + + let err = cmd_get("bafkreigwrapped".to_string(), server.url(), None, None) + .await + .expect_err("an incomplete scan with nothing to resume from is an error"); + let told = told(&err); + + assert!( + told.contains("offered no continuation token"), + "the terminal must say why it stopped, got: {told}" + ); + assert!( + !told.contains("--scan") && !told.contains(&t), + "a wrapped scan has nowhere further to go, so a resume hint here would \ + invite a re-run that cannot find more, got: {told}" + ); + + rung1.assert_async().await; + rung2.assert_async().await; + } + + /// The success path streams. `resp.bytes()` buffered the whole object first, so a + /// hostile node answering 200 with a very large body delivered fast made the + /// client allocate all of it before a byte reached stdout, while the sibling error + /// read was capped at 8 KiB. + /// + /// What is asserted here is the CORRECTNESS of streaming, not the allocation: a + /// body far larger than one chunk must arrive at the sink whole, in order, byte + /// for byte. A chunk loop that dropped or reordered a chunk would be the obvious + /// way to get the memory right and the object wrong, and on a content-addressed + /// fetch that is the worse failure. + #[tokio::test] + async fn write_object_streams_a_large_body_through_intact() { + reset_diag(); + // 4 MiB of a non-repeating pattern, well past any single chunk. + let payload: Vec = (0..4 * 1024 * 1024u32).map(|i| (i % 251) as u8).collect(); + + let mut server = mockito::Server::new_async().await; + let m = server + .mock("GET", "/ipfs/bafkreibig") + .with_status(200) + .with_header("x-git-hash", "deadbeef") + .with_body(payload.clone()) + .create_async() + .await; + + let resp = reqwest::get(format!("{}/ipfs/bafkreibig", server.url())) + .await + .unwrap(); + let mut sink: Vec = Vec::new(); + write_object_to(resp, &mut sink).await.unwrap(); + + assert_eq!( + sink.len(), + payload.len(), + "a streamed body must arrive whole" + ); + assert!(sink == payload, "a streamed body must arrive unaltered"); + assert!( + diag_text().contains("deadbeef"), + "the header diagnostics still go to stderr, got: {}", + diag_text() + ); + m.assert_async().await; + } + + /// `node_tail`'s partial-body arm: a body that arrived part-way and then failed. + /// The other three `(read_failed, msg.is_empty())` combinations were covered; this + /// one, the shape a real broken connection most often produces, was not, because + /// the existing fixture writes zero body bytes. It is also the only arm where + /// node-supplied partial text reaches the terminal. + /// + /// The listener promises 512 bytes, writes a few, and hangs up. The terminal must + /// carry BOTH what did arrive and the fact that the rest did not. + #[tokio::test] + async fn test_cmd_get_reports_partial_text_and_the_unfinished_read() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + reset_diag(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + let (mut sock, _) = listener.accept().await.unwrap(); + let mut scratch = [0u8; 2048]; + let _ = sock.read(&mut scratch).await; + let _ = sock + .write_all( + b"HTTP/1.1 500 Internal Server Error\r\nContent-Type: application/json\r\n\ + Content-Length: 512\r\nConnection: close\r\n\r\n\ + {\"error\":\"boom\",\"message\":\"half a sen", + ) + .await; + let _ = sock.flush().await; + }); + + let err = cmd_get( + "bafkreipartial".to_string(), + format!("http://{addr}"), + None, + None, + ) + .await + .expect_err("a 500 is an error whatever became of its body"); + let told = told(&err); + + assert!( + told.contains("half a sen"), + "the text that DID arrive must reach the caller, got: {told}" + ); + assert!( + told.contains("could not be read in full"), + "and it must be marked as unfinished, or partial node text reads as the \ + node's whole answer, got: {told}" + ); + } } diff --git a/crates/gl/src/peer.rs b/crates/gl/src/peer.rs index 6b55b882..aa420f2e 100644 --- a/crates/gl/src/peer.rs +++ b/crates/gl/src/peer.rs @@ -199,7 +199,7 @@ async fn cmd_add(peer_url: String, node: String, dir: Option) -> Result // in the command: bound the read, and defang the message before it reaches // the terminal through the error return. An announce reply is a DID, a URL // and a count, so 8 KiB is well past what the shape needs. - let raw = read_body_capped(resp, 8 * 1024).await; + let raw = read_body_capped(resp, 8 * 1024).await.text; if let Some(failure) = remote_announce_failure(status, &raw) { anyhow::bail!("{failure}"); @@ -237,7 +237,7 @@ async fn cmd_add(peer_url: String, node: String, dir: Option) -> Result // node (or a MITM on plain http) must not force an unbounded // read. A body that does not parse stays `Null`, which still // routes a non-success status to the warning. - let raw = read_body_capped(resp, 8 * 1024).await; + let raw = read_body_capped(resp, 8 * 1024).await.text; let result: Value = serde_json::from_str(&raw).unwrap_or(Value::Null); let (is_warning, line) = local_add_report(status, &result); if is_warning { diff --git a/crates/gl/src/sync.rs b/crates/gl/src/sync.rs index c3ff7972..c4e830c3 100644 --- a/crates/gl/src/sync.rs +++ b/crates/gl/src/sync.rs @@ -46,7 +46,7 @@ pub async fn run(args: SyncArgs) -> Result<()> { if !status.is_success() { // Bound the read: a hostile or broken node must not force an // unbounded allocation just to surface a denial (INV-6, read half). - let raw = read_body_capped(resp, 8 * 1024).await; + let raw = read_body_capped(resp, 8 * 1024).await.text; let msg = serde_json::from_str::(&raw) .ok() .and_then(|v| { @@ -261,7 +261,7 @@ mod tests { .create_async() .await; let resp = reqwest::get(format!("{}/big", server.url())).await.unwrap(); - let out = read_body_capped(resp, 8192).await; + let out = read_body_capped(resp, 8192).await.text; assert!(out.len() <= 8192, "read not bounded: {} bytes", out.len()); assert!(!out.is_empty(), "expected some body"); } diff --git a/docs/RUN-A-NODE.md b/docs/RUN-A-NODE.md index 0a8a0f77..d0431941 100644 --- a/docs/RUN-A-NODE.md +++ b/docs/RUN-A-NODE.md @@ -188,6 +188,8 @@ GITLAWB_ENFORCE_OWNER_PUSH=true **Node refuses to start with "strict-mode operator check failed"** — either `gl node register` first, or unset `GITLAWB_OPERATOR_STRICT_MODE`. +**Node refuses to start with "GITLAWB_DB_MAX_CONNECTIONS (20) must be at least max_concurrent_git_pushes (32) + 8 headroom"**: raise `GITLAWB_DB_MAX_CONNECTIONS` to at least the push cap plus 8 (40 with the default cap of 32; 48 is the shipped default and the recommended value), or lower `GITLAWB_MAX_CONCURRENT_GIT_PUSHES`. This bites a node upgraded in place that still sets the old pool size of 20. The check is deliberate: each concurrent push pins one pooled connection for its whole receive-pack, so a pool that does not clear the push cap lets a burst of slow pushes starve every other database path. + **Rewards are 0 after a week** — run `gl node onchain-status`. If `currentlyActive: false`, check your heartbeat loop (node logs for `operator heartbeat sent`). **Want to rotate operator wallet** — requires unstake → re-register with new wallet. No in-place rotation in v1.