From 5cc7e7a564a4055342292f4dd35e90230aed52bf Mon Sep 17 00:00:00 2001 From: KRATOS <84986124+gnanam1990@users.noreply.github.com> Date: Fri, 14 Aug 2026 21:28:17 +0530 Subject: [PATCH 1/9] feat(node): persist upstream mirror transition state Origin-Session: local-d6a143 | Codex | 12 prompts --- crates/gitlawb-node/src/db/mod.rs | 617 ++++++++++++++++++++++++++++++ 1 file changed, 617 insertions(+) diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 50c3bdda..0eb48165 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -23,6 +23,130 @@ pub struct RepoRecord { pub machine_id: Option, } +/// Which side is authoritative for a continuously mirrored repository. +/// +/// A repository without an `upstream_url` has no mirror status. Transitioning +/// states always carry a durable job id and phase so a node restart can resume +/// or roll back instead of guessing which side owns writes. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +#[cfg_attr(not(test), allow(dead_code))] +pub enum MirrorStatus { + Inbound, + TransitioningInboundToOutbound, + Outbound, + TransitioningOutboundToInbound, +} + +#[cfg_attr(not(test), allow(dead_code))] +impl MirrorStatus { + fn from_db(value: &str) -> Result { + match value { + "inbound" => Ok(Self::Inbound), + "transitioning_inbound_to_outbound" => Ok(Self::TransitioningInboundToOutbound), + "outbound" => Ok(Self::Outbound), + "transitioning_outbound_to_inbound" => Ok(Self::TransitioningOutboundToInbound), + _ => anyhow::bail!("invalid mirror status stored in database"), + } + } + + fn is_transitioning(self) -> bool { + matches!( + self, + Self::TransitioningInboundToOutbound | Self::TransitioningOutboundToInbound + ) + } +} + +/// Durable progress marker for a mirror-authority transition job. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +#[cfg_attr(not(test), allow(dead_code))] +pub enum MirrorTransitionPhase { + Queued, + Starting, + InitializingMirrorFetch, + DrainingWrites, + FinalizingMirrorFetch, + FinalizingMirrorPush, + CommittingTargetStatus, + Completed, + RollingBack, +} + +#[cfg_attr(not(test), allow(dead_code))] +impl MirrorTransitionPhase { + fn from_db(value: &str) -> Result { + match value { + "queued" => Ok(Self::Queued), + "starting" => Ok(Self::Starting), + "initializing_mirror_fetch" => Ok(Self::InitializingMirrorFetch), + "draining_writes" => Ok(Self::DrainingWrites), + "finalizing_mirror_fetch" => Ok(Self::FinalizingMirrorFetch), + "finalizing_mirror_push" => Ok(Self::FinalizingMirrorPush), + "committing_target_status" => Ok(Self::CommittingTargetStatus), + "completed" => Ok(Self::Completed), + "rolling_back" => Ok(Self::RollingBack), + _ => anyhow::bail!("invalid mirror transition phase stored in database"), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(not(test), allow(dead_code))] +pub struct RepoMirrorState { + pub repo_id: String, + pub upstream_url: String, + pub status: MirrorStatus, + pub transition_id: Option, + pub transition_phase: Option, + pub updated_at: DateTime, +} + +#[cfg_attr(not(test), allow(dead_code))] +impl RepoMirrorState { + fn validate(&self) -> Result<()> { + validate_mirror_upstream_url(&self.upstream_url)?; + + match ( + self.status.is_transitioning(), + self.transition_id, + self.transition_phase, + ) { + (true, Some(_), Some(phase)) if phase != MirrorTransitionPhase::Completed => Ok(()), + (false, None, None) => Ok(()), + (false, Some(_), Some(MirrorTransitionPhase::Completed)) => Ok(()), + _ => anyhow::bail!("mirror status and transition metadata are inconsistent"), + } + } +} + +#[cfg_attr(not(test), allow(dead_code))] +fn validate_mirror_upstream_url(raw: &str) -> Result { + if raw.is_empty() || raw.len() > 2048 { + anyhow::bail!("mirror upstream URL must contain 1 to 2048 bytes"); + } + if raw.chars().any(|c| c.is_whitespace() || c.is_control()) { + anyhow::bail!("mirror upstream URL contains whitespace or control characters"); + } + + let url = reqwest::Url::parse(raw).context("parsing mirror upstream URL")?; + if url.scheme() != "https" { + anyhow::bail!("mirror upstream URL must use HTTPS"); + } + if url.host_str().is_none() || url.path().trim_matches('/').is_empty() { + anyhow::bail!("mirror upstream URL must include a host and repository path"); + } + if !url.username().is_empty() || url.password().is_some() { + anyhow::bail!("mirror upstream URL must not embed credentials"); + } + if url.query().is_some() || url.fragment().is_some() { + anyhow::bail!("mirror upstream URL must not contain a query or fragment"); + } + + Ok(url) +} + /// Per-rule replication mode for a visibility rule. /// `A` hides existence entirely (only valid at whole-repo scope `/`). /// `B` keeps object SHAs and the path visible but withholds content @@ -901,6 +1025,84 @@ const MIGRATIONS: &[Migration] = &[ "ALTER TABLE sync_queue ADD COLUMN IF NOT EXISTS attempted_at TEXT", ], }, + // Reservation: open branches currently claim migration versions through + // v29. Gaps are valid, while reusing a version can silently skip one + // branch's schema when both eventually land. v30 and v31 are intentionally + // separate so validation does not inherit v30's stronger table lock. + Migration { + version: 30, + name: "repo_upstream_mirror_state", + stmts: &[ + // Nullable, default-free columns keep this migration metadata-only + // for every existing non-mirror repository. + "ALTER TABLE repos ADD COLUMN IF NOT EXISTS upstream_url TEXT", + "ALTER TABLE repos ADD COLUMN IF NOT EXISTS mirror_status TEXT", + "ALTER TABLE repos ADD COLUMN IF NOT EXISTS mirror_transition_id UUID", + "ALTER TABLE repos ADD COLUMN IF NOT EXISTS mirror_transition_phase TEXT", + "ALTER TABLE repos ADD COLUMN IF NOT EXISTS mirror_updated_at TEXT", + // PostgreSQL has no ADD CONSTRAINT IF NOT EXISTS, so guard by name + // to keep manual/idempotent recovery safe. + r#"DO $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'repos'::regclass + AND conname = 'repos_mirror_state_valid' + ) THEN + ALTER TABLE repos ADD CONSTRAINT repos_mirror_state_valid CHECK ( + ( + upstream_url IS NULL + AND mirror_status IS NULL + AND mirror_transition_id IS NULL + AND mirror_transition_phase IS NULL + AND mirror_updated_at IS NULL + ) + OR + ( + upstream_url IS NOT NULL + AND upstream_url LIKE 'https://%' + AND upstream_url <> 'https://' + AND mirror_updated_at IS NOT NULL + AND ( + ( + mirror_status IN ('inbound', 'outbound') + AND ( + (mirror_transition_id IS NULL AND mirror_transition_phase IS NULL) + OR + (mirror_transition_id IS NOT NULL AND mirror_transition_phase = 'completed') + ) + ) + OR + ( + mirror_status IN ( + 'transitioning_inbound_to_outbound', + 'transitioning_outbound_to_inbound' + ) + AND mirror_transition_id IS NOT NULL + AND mirror_transition_phase IN ( + 'queued', + 'starting', + 'initializing_mirror_fetch', + 'draining_writes', + 'finalizing_mirror_fetch', + 'finalizing_mirror_push', + 'committing_target_status', + 'rolling_back' + ) + ) + ) + ) + ) NOT VALID; + END IF; + END + $$"#, + ], + }, + Migration { + version: 31, + name: "validate_repo_upstream_mirror_state", + stmts: &["ALTER TABLE repos VALIDATE CONSTRAINT repos_mirror_state_valid"], + }, ]; // ── Repos ───────────────────────────────────────────────────────────────────── @@ -1011,6 +1213,72 @@ impl Db { Ok(()) } + /// Attach an external HTTPS upstream to a canonical repository in inbound + /// mode. This is deliberately idempotent only for the same untouched + /// inbound configuration; it never overwrites an existing upstream or + /// resets a transition that may be in progress. + #[cfg_attr(not(test), allow(dead_code))] + pub async fn configure_inbound_mirror( + &self, + repo_id: &str, + upstream_url: &str, + ) -> Result { + Uuid::parse_str(repo_id) + .context("continuous mirrors require a canonical UUID repository id")?; + let upstream_url = validate_mirror_upstream_url(upstream_url)?.to_string(); + let updated_at = Utc::now().to_rfc3339(); + + let row = sqlx::query( + "UPDATE repos + SET upstream_url = $2, + mirror_status = 'inbound', + mirror_transition_id = NULL, + mirror_transition_phase = NULL, + mirror_updated_at = $3 + WHERE id = $1 + AND ( + upstream_url IS NULL + OR ( + upstream_url = $2 + AND mirror_status = 'inbound' + AND mirror_transition_id IS NULL + AND mirror_transition_phase IS NULL + ) + ) + RETURNING id, upstream_url, mirror_status, mirror_transition_id, + mirror_transition_phase, mirror_updated_at", + ) + .bind(repo_id) + .bind(&upstream_url) + .bind(&updated_at) + .fetch_optional(&self.pool) + .await?; + + let row = row.context( + "repository does not exist or already has a different/transitioning mirror configuration", + )?; + row_to_repo_mirror_state(row) + } + + /// Return the durable mirror state, or `None` for a normal non-mirrored + /// repository (and for an unknown id). Callers that need to distinguish + /// those cases must resolve the repository first through the auth-gated + /// repository path. + #[cfg_attr(not(test), allow(dead_code))] + pub async fn get_repo_mirror_state(&self, repo_id: &str) -> Result> { + let row = sqlx::query( + "SELECT id, upstream_url, mirror_status, mirror_transition_id, + mirror_transition_phase, mirror_updated_at + FROM repos + WHERE id = $1 AND upstream_url IS NOT NULL", + ) + .bind(repo_id) + .fetch_optional(&self.pool) + .await?; + + row.map(row_to_repo_mirror_state).transpose() + } + /// Register a mirrored repo from a peer in the local DB so git smart HTTP can serve it. /// Uses INSERT OR IGNORE (SQLite) / ON CONFLICT DO NOTHING (Postgres) so it's idempotent. pub async fn upsert_mirror_repo( @@ -3047,6 +3315,355 @@ impl Db { // ── Row helpers ─────────────────────────────────────────────────────────────── +#[cfg_attr(not(test), allow(dead_code))] +fn row_to_repo_mirror_state(r: sqlx::postgres::PgRow) -> Result { + let status = MirrorStatus::from_db(&r.get::("mirror_status"))?; + let transition_phase = r + .get::, _>("mirror_transition_phase") + .as_deref() + .map(MirrorTransitionPhase::from_db) + .transpose()?; + let updated_at = r + .get::("mirror_updated_at") + .parse::>() + .context("invalid mirror_updated_at stored in database")?; + + let state = RepoMirrorState { + repo_id: r.get("id"), + upstream_url: r.get("upstream_url"), + status, + transition_id: r.get("mirror_transition_id"), + transition_phase, + updated_at, + }; + state.validate()?; + Ok(state) +} + +#[cfg(test)] +mod mirror_state_tests { + use super::{ + validate_mirror_upstream_url, Db, MirrorStatus, MirrorTransitionPhase, RepoMirrorState, + RepoRecord, + }; + use chrono::{DateTime, Utc}; + use sqlx::PgPool; + use uuid::Uuid; + + fn ts(value: &str) -> DateTime { + value.parse().unwrap() + } + + fn repo(id: Uuid) -> RepoRecord { + RepoRecord { + id: id.to_string(), + name: "mirror-fixture".to_string(), + owner_did: "did:key:z6MkMirrorOwner".to_string(), + description: None, + is_public: true, + default_branch: "main".to_string(), + created_at: ts("2026-08-14T00:00:00Z"), + updated_at: ts("2026-08-14T00:00:00Z"), + disk_path: format!("/srv/repos/{id}.git"), + forked_from: None, + machine_id: None, + } + } + + async fn migrated_db(pool: PgPool) -> Db { + let db = Db::for_testing(pool); + db.run_migrations().await.unwrap(); + db + } + + #[test] + fn upstream_url_accepts_https_forges_without_credentials() { + for valid in [ + "https://github.com/Gitlawb/node.git", + "https://gitlab.example.com/platform/subgroup/node", + "https://ghe.internal.example/org/repo.git", + ] { + validate_mirror_upstream_url(valid).unwrap(); + } + } + + #[test] + fn upstream_url_rejects_unsafe_or_ambiguous_forms() { + for invalid in [ + "", + "http://github.com/Gitlawb/node.git", + "ssh://git@github.com/Gitlawb/node.git", + "file:///etc/passwd", + "https://user:secret@github.com/Gitlawb/node.git", + "https://github.com/Gitlawb/node.git?token=secret", + "https://github.com/Gitlawb/node.git#main", + "https://github.com/", + "https://github.com/Gitlawb/node.git\n", + ] { + assert!( + validate_mirror_upstream_url(invalid).is_err(), + "unsafe mirror URL was accepted: {invalid:?}" + ); + } + } + + #[test] + fn transition_metadata_must_match_the_status() { + let base = RepoMirrorState { + repo_id: Uuid::new_v4().to_string(), + upstream_url: "https://github.com/Gitlawb/node.git".to_string(), + status: MirrorStatus::Inbound, + transition_id: None, + transition_phase: None, + updated_at: Utc::now(), + }; + base.validate().unwrap(); + + let mut invalid = base.clone(); + invalid.status = MirrorStatus::TransitioningInboundToOutbound; + assert!(invalid.validate().is_err()); + + invalid.transition_id = Some(Uuid::new_v4()); + invalid.transition_phase = Some(MirrorTransitionPhase::DrainingWrites); + invalid.validate().unwrap(); + + invalid.transition_phase = Some(MirrorTransitionPhase::Completed); + assert!(invalid.validate().is_err()); + } + + #[sqlx::test] + async fn migrations_v30_v31_upgrade_an_existing_repo_without_rewriting_it(pool: PgPool) { + let db = migrated_db(pool).await; + + // Recreate the pre-v30 shape, then add data written by the old node. + sqlx::query("ALTER TABLE repos DROP COLUMN upstream_url CASCADE") + .execute(db.pool()) + .await + .unwrap(); + for column in [ + "mirror_status", + "mirror_transition_id", + "mirror_transition_phase", + "mirror_updated_at", + ] { + sqlx::query(&format!("ALTER TABLE repos DROP COLUMN {column}")) + .execute(db.pool()) + .await + .unwrap(); + } + sqlx::query("DELETE FROM schema_migrations WHERE version IN (30, 31)") + .execute(db.pool()) + .await + .unwrap(); + + let existing = repo(Uuid::new_v4()); + db.create_repo(&existing).await.unwrap(); + + db.run_migrations().await.unwrap(); + + let still_there: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM repos WHERE id = $1") + .bind(&existing.id) + .fetch_one(db.pool()) + .await + .unwrap(); + assert_eq!( + still_there, 1, + "the existing repository must survive v30/v31" + ); + + let mirror_values: (Option, Option, Option, Option) = + sqlx::query_as( + "SELECT upstream_url, mirror_status, mirror_transition_id, + mirror_transition_phase + FROM repos WHERE id = $1", + ) + .bind(&existing.id) + .fetch_one(db.pool()) + .await + .unwrap(); + assert_eq!(mirror_values, (None, None, None, None)); + + let recorded: Vec<(i64, String)> = sqlx::query_as( + "SELECT version, name FROM schema_migrations + WHERE version IN (30, 31) ORDER BY version", + ) + .fetch_all(db.pool()) + .await + .unwrap(); + assert_eq!( + recorded, + vec![ + (30, "repo_upstream_mirror_state".to_string()), + (31, "validate_repo_upstream_mirror_state".to_string()), + ] + ); + let constraint_validated: bool = sqlx::query_scalar( + "SELECT convalidated FROM pg_constraint + WHERE conrelid = 'repos'::regclass + AND conname = 'repos_mirror_state_valid'", + ) + .fetch_one(db.pool()) + .await + .unwrap(); + assert!(constraint_validated, "v31 must validate the v30 constraint"); + + // Production entry point remains idempotent after the upgrade. + db.run_migrations().await.unwrap(); + } + + #[sqlx::test] + async fn inbound_configuration_round_trips_without_overwriting_state(pool: PgPool) { + let db = migrated_db(pool).await; + let existing = repo(Uuid::new_v4()); + db.create_repo(&existing).await.unwrap(); + + assert!(db + .get_repo_mirror_state(&existing.id) + .await + .unwrap() + .is_none()); + + let first = db + .configure_inbound_mirror(&existing.id, "https://github.com/Gitlawb/node.git") + .await + .unwrap(); + assert_eq!(first.status, MirrorStatus::Inbound); + assert_eq!(first.transition_id, None); + assert_eq!(first.transition_phase, None); + + // An exact retry is safe, but a different source cannot replace it. + db.configure_inbound_mirror(&existing.id, "https://github.com/Gitlawb/node.git") + .await + .unwrap(); + assert!(db + .configure_inbound_mirror(&existing.id, "https://github.com/Gitlawb/other.git",) + .await + .is_err()); + + let stored = db + .get_repo_mirror_state(&existing.id) + .await + .unwrap() + .unwrap(); + assert_eq!(stored.upstream_url, first.upstream_url); + assert_eq!(stored.status, MirrorStatus::Inbound); + } + + #[sqlx::test] + async fn database_constraint_rejects_partial_or_insecure_mirror_state(pool: PgPool) { + let db = migrated_db(pool).await; + let existing = repo(Uuid::new_v4()); + db.create_repo(&existing).await.unwrap(); + + let partial = sqlx::query( + "UPDATE repos + SET upstream_url = 'https://github.com/Gitlawb/node.git', + mirror_status = 'transitioning_inbound_to_outbound', + mirror_updated_at = '2026-08-14T00:00:00Z' + WHERE id = $1", + ) + .bind(&existing.id) + .execute(db.pool()) + .await; + assert!(partial.is_err(), "transition metadata must be durable"); + + let insecure = sqlx::query( + "UPDATE repos + SET upstream_url = 'http://github.com/Gitlawb/node.git', + mirror_status = 'inbound', + mirror_updated_at = '2026-08-14T00:00:00Z' + WHERE id = $1", + ) + .bind(&existing.id) + .execute(db.pool()) + .await; + assert!( + insecure.is_err(), + "the database must reject non-HTTPS upstreams" + ); + } + + #[sqlx::test] + async fn database_constraint_accepts_the_complete_transition_vocabulary(pool: PgPool) { + let db = migrated_db(pool).await; + let existing = repo(Uuid::new_v4()); + db.create_repo(&existing).await.unwrap(); + + let phases = [ + "queued", + "starting", + "initializing_mirror_fetch", + "draining_writes", + "finalizing_mirror_fetch", + "finalizing_mirror_push", + "committing_target_status", + "rolling_back", + ]; + for status in [ + "transitioning_inbound_to_outbound", + "transitioning_outbound_to_inbound", + ] { + for phase in phases { + sqlx::query( + "UPDATE repos + SET upstream_url = 'https://github.com/Gitlawb/node.git', + mirror_status = $2, + mirror_transition_id = $3, + mirror_transition_phase = $4, + mirror_updated_at = '2026-08-14T00:00:00Z' + WHERE id = $1", + ) + .bind(&existing.id) + .bind(status) + .bind(Uuid::new_v4()) + .bind(phase) + .execute(db.pool()) + .await + .unwrap_or_else(|error| panic!("{status}/{phase} was rejected: {error}")); + } + } + + // A completed job remains auditable after authority reaches its stable + // target status; the initial never-transitioned state uses two NULLs. + sqlx::query( + "UPDATE repos + SET mirror_status = 'outbound', + mirror_transition_id = $2, + mirror_transition_phase = 'completed' + WHERE id = $1", + ) + .bind(&existing.id) + .bind(Uuid::new_v4()) + .execute(db.pool()) + .await + .unwrap(); + + let stored = db + .get_repo_mirror_state(&existing.id) + .await + .unwrap() + .unwrap(); + assert_eq!(stored.status, MirrorStatus::Outbound); + assert_eq!( + stored.transition_phase, + Some(MirrorTransitionPhase::Completed) + ); + } + + #[sqlx::test] + async fn peer_mirror_rows_cannot_become_continuous_upstreams(pool: PgPool) { + let db = migrated_db(pool).await; + db.upsert_mirror_repo("z6MkPeer", "repo", "/srv/peer/repo.git", None, false) + .await + .unwrap(); + + let result = db + .configure_inbound_mirror("z6MkPeer/repo", "https://github.com/Gitlawb/node.git") + .await; + assert!(result.is_err()); + } +} + fn row_to_repo(r: sqlx::postgres::PgRow) -> RepoRecord { let created_str: String = r.get("created_at"); let updated_str: String = r.get("updated_at"); From ea34d84c3a9bbfad643f1aa765fc28e93e4fedf1 Mon Sep 17 00:00:00 2001 From: KRATOS <84986124+gnanam1990@users.noreply.github.com> Date: Fri, 14 Aug 2026 21:44:27 +0530 Subject: [PATCH 2/9] fix(node): reject null mirror transition state Origin-Session: local-d6a143 | Codex | 14 prompts --- crates/gitlawb-node/src/db/mod.rs | 96 ++++++++++++++++++++++++++++++- 1 file changed, 94 insertions(+), 2 deletions(-) diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 0eb48165..115dddb4 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -1049,7 +1049,10 @@ const MIGRATIONS: &[Migration] = &[ WHERE conrelid = 'repos'::regclass AND conname = 'repos_mirror_state_valid' ) THEN - ALTER TABLE repos ADD CONSTRAINT repos_mirror_state_valid CHECK ( + -- PostgreSQL accepts a CHECK whose result is NULL. Wrap + -- the complete invariant in IS TRUE so a missing status + -- or phase cannot slip through three-valued logic. + ALTER TABLE repos ADD CONSTRAINT repos_mirror_state_valid CHECK (( ( upstream_url IS NULL AND mirror_status IS NULL @@ -1092,7 +1095,7 @@ const MIGRATIONS: &[Migration] = &[ ) ) ) - ) NOT VALID; + ) IS TRUE) NOT VALID; END IF; END $$"#, @@ -3549,6 +3552,47 @@ mod mirror_state_tests { assert_eq!(stored.status, MirrorStatus::Inbound); } + #[sqlx::test] + async fn inbound_configuration_never_resets_an_active_transition(pool: PgPool) { + let db = migrated_db(pool).await; + let existing = repo(Uuid::new_v4()); + db.create_repo(&existing).await.unwrap(); + db.configure_inbound_mirror(&existing.id, "https://github.com/Gitlawb/node.git") + .await + .unwrap(); + + let transition_id = Uuid::new_v4(); + sqlx::query( + "UPDATE repos + SET mirror_status = 'transitioning_inbound_to_outbound', + mirror_transition_id = $2, + mirror_transition_phase = 'draining_writes' + WHERE id = $1", + ) + .bind(&existing.id) + .bind(transition_id) + .execute(db.pool()) + .await + .unwrap(); + + let retry = db + .configure_inbound_mirror(&existing.id, "https://github.com/Gitlawb/node.git") + .await; + assert!(retry.is_err(), "configuration must not reset a transition"); + + let stored = db + .get_repo_mirror_state(&existing.id) + .await + .unwrap() + .unwrap(); + assert_eq!(stored.status, MirrorStatus::TransitioningInboundToOutbound); + assert_eq!(stored.transition_id, Some(transition_id)); + assert_eq!( + stored.transition_phase, + Some(MirrorTransitionPhase::DrainingWrites) + ); + } + #[sqlx::test] async fn database_constraint_rejects_partial_or_insecure_mirror_state(pool: PgPool) { let db = migrated_db(pool).await; @@ -3567,6 +3611,54 @@ mod mirror_state_tests { .await; assert!(partial.is_err(), "transition metadata must be durable"); + // PostgreSQL CHECK constraints accept NULL results. These cases pin + // the outer `IS TRUE` guard so SQL three-valued logic cannot admit a + // row that the Rust reader rejects or panics while decoding. + struct InvalidNullState<'a> { + case: &'a str, + status: Option<&'a str>, + transition_id: Option, + transition_phase: Option<&'a str>, + } + let invalid_null_states = [ + InvalidNullState { + case: "missing mirror status", + status: None, + transition_id: None, + transition_phase: None, + }, + InvalidNullState { + case: "stable status with job but no phase", + status: Some("inbound"), + transition_id: Some(Uuid::new_v4()), + transition_phase: None, + }, + InvalidNullState { + case: "transition with job but no phase", + status: Some("transitioning_inbound_to_outbound"), + transition_id: Some(Uuid::new_v4()), + transition_phase: None, + }, + ]; + for invalid in invalid_null_states { + let result = sqlx::query( + "UPDATE repos + SET upstream_url = 'https://github.com/Gitlawb/node.git', + mirror_status = $2, + mirror_transition_id = $3, + mirror_transition_phase = $4, + mirror_updated_at = '2026-08-14T00:00:00Z' + WHERE id = $1", + ) + .bind(&existing.id) + .bind(invalid.status) + .bind(invalid.transition_id) + .bind(invalid.transition_phase) + .execute(db.pool()) + .await; + assert!(result.is_err(), "constraint accepted {}", invalid.case); + } + let insecure = sqlx::query( "UPDATE repos SET upstream_url = 'http://github.com/Gitlawb/node.git', From 92997b369516871c2ca3d4f28a652d78abc556cf Mon Sep 17 00:00:00 2001 From: KRATOS <84986124+gnanam1990@users.noreply.github.com> Date: Fri, 14 Aug 2026 22:22:19 +0530 Subject: [PATCH 3/9] fix(node): address mirror state review findings Origin-Session: local-d6a143 | Codex | 16 prompts --- crates/gitlawb-node/src/db/mod.rs | 69 ++++++++++++++++++++++++++----- 1 file changed, 59 insertions(+), 10 deletions(-) diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 115dddb4..43c4b51c 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -1237,7 +1237,7 @@ impl Db { mirror_status = 'inbound', mirror_transition_id = NULL, mirror_transition_phase = NULL, - mirror_updated_at = $3 + mirror_updated_at = COALESCE(mirror_updated_at, $3) WHERE id = $1 AND ( upstream_url IS NULL @@ -3320,22 +3320,36 @@ impl Db { #[cfg_attr(not(test), allow(dead_code))] fn row_to_repo_mirror_state(r: sqlx::postgres::PgRow) -> Result { - let status = MirrorStatus::from_db(&r.get::("mirror_status"))?; + let repo_id = r + .try_get("id") + .context("reading mirror repository id from database")?; + let upstream_url = r + .try_get("upstream_url") + .context("reading mirror upstream URL from database")?; + let status = r + .try_get::("mirror_status") + .context("reading mirror status from database")?; + let status = MirrorStatus::from_db(&status)?; + let transition_id = r + .try_get("mirror_transition_id") + .context("reading mirror transition id from database")?; let transition_phase = r - .get::, _>("mirror_transition_phase") + .try_get::, _>("mirror_transition_phase") + .context("reading mirror transition phase from database")? .as_deref() .map(MirrorTransitionPhase::from_db) .transpose()?; let updated_at = r - .get::("mirror_updated_at") + .try_get::("mirror_updated_at") + .context("reading mirror update timestamp from database")? .parse::>() .context("invalid mirror_updated_at stored in database")?; let state = RepoMirrorState { - repo_id: r.get("id"), - upstream_url: r.get("upstream_url"), + repo_id, + upstream_url, status, - transition_id: r.get("mirror_transition_id"), + transition_id, transition_phase, updated_at, }; @@ -3346,8 +3360,8 @@ fn row_to_repo_mirror_state(r: sqlx::postgres::PgRow) -> Result #[cfg(test)] mod mirror_state_tests { use super::{ - validate_mirror_upstream_url, Db, MirrorStatus, MirrorTransitionPhase, RepoMirrorState, - RepoRecord, + row_to_repo_mirror_state, validate_mirror_upstream_url, Db, MirrorStatus, + MirrorTransitionPhase, RepoMirrorState, RepoRecord, }; use chrono::{DateTime, Utc}; use sqlx::PgPool; @@ -3535,9 +3549,20 @@ mod mirror_state_tests { assert_eq!(first.transition_phase, None); // An exact retry is safe, but a different source cannot replace it. - db.configure_inbound_mirror(&existing.id, "https://github.com/Gitlawb/node.git") + let original_updated_at = DateTime::parse_from_rfc3339("2026-01-02T03:04:05Z") + .unwrap() + .with_timezone(&Utc); + sqlx::query("UPDATE repos SET mirror_updated_at = $2 WHERE id = $1") + .bind(&existing.id) + .bind(original_updated_at.to_rfc3339()) + .execute(db.pool()) .await .unwrap(); + let retry = db + .configure_inbound_mirror(&existing.id, "https://github.com/Gitlawb/node.git") + .await + .unwrap(); + assert_eq!(retry.updated_at, original_updated_at); assert!(db .configure_inbound_mirror(&existing.id, "https://github.com/Gitlawb/other.git",) .await @@ -3550,6 +3575,30 @@ mod mirror_state_tests { .unwrap(); assert_eq!(stored.upstream_url, first.upstream_url); assert_eq!(stored.status, MirrorStatus::Inbound); + assert_eq!(stored.updated_at, original_updated_at); + } + + #[sqlx::test] + async fn malformed_mirror_row_returns_an_error_instead_of_panicking(pool: PgPool) { + let row = sqlx::query( + "SELECT 'repo-id'::text AS id, + 'https://github.com/Gitlawb/node.git'::text AS upstream_url, + 'inbound'::text AS mirror_status, + NULL::uuid AS mirror_transition_id, + NULL::text AS mirror_transition_phase, + NULL::text AS mirror_updated_at", + ) + .fetch_one(&pool) + .await + .unwrap(); + + let error = row_to_repo_mirror_state(row).unwrap_err(); + assert!( + error + .to_string() + .contains("reading mirror update timestamp from database"), + "unexpected error: {error:#}" + ); } #[sqlx::test] From 5b9e06d501e86b2145ec8230c4936fde43dc7995 Mon Sep 17 00:00:00 2001 From: KRATOS <84986124+gnanam1990@users.noreply.github.com> Date: Sat, 15 Aug 2026 00:25:14 +0530 Subject: [PATCH 4/9] fix(node): close mirror review blockers Origin-Session: local-d6a143 | Codex | 20 prompts --- crates/gitlawb-node/src/api/peers.rs | 6 +++-- crates/gitlawb-node/src/db/mod.rs | 40 +++++++++++++++++++++++++--- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/crates/gitlawb-node/src/api/peers.rs b/crates/gitlawb-node/src/api/peers.rs index 3934e7a3..6f4ed200 100644 --- a/crates/gitlawb-node/src/api/peers.rs +++ b/crates/gitlawb-node/src/api/peers.rs @@ -104,8 +104,8 @@ fn embedded_ipv4(v6: std::net::Ipv6Addr) -> Option { /// Whether a peer `http_url` is a public http(s) endpoint safe to register. /// Rejects non-http(s) schemes, loopback/unspecified/private/link-local IPs, -/// and `localhost` / `.local` / `.internal` hostnames. Used at announce time -/// and by the boot-time prune of already-poisoned rows. +/// and `localhost` / `.localhost` / `.local` / `.internal` hostnames. Used at +/// announce time and by the boot-time prune of already-poisoned rows. pub fn is_public_http_url(raw: &str) -> bool { let url = match reqwest::Url::parse(raw) { Ok(u) => u, @@ -125,6 +125,7 @@ pub fn is_public_http_url(raw: &str) -> bool { } if host.is_empty() || host == "localhost" + || host.ends_with(".localhost") || host.ends_with(".local") || host.ends_with(".internal") { @@ -529,6 +530,7 @@ mod tests { fn rejects_loopback_private_and_internal() { for bad in [ "http://localhost:7545", + "http://node.localhost:7545", "http://127.0.0.1:5432/", "http://localhost:22/", "http://0.0.0.0:7545", diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 43c4b51c..eebc3b64 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -134,6 +134,9 @@ fn validate_mirror_upstream_url(raw: &str) -> Result { if url.scheme() != "https" { anyhow::bail!("mirror upstream URL must use HTTPS"); } + if !crate::api::peers::is_public_http_url(raw) { + anyhow::bail!("mirror upstream URL must use a public host"); + } if url.host_str().is_none() || url.path().trim_matches('/').is_empty() { anyhow::bail!("mirror upstream URL must include a host and repository path"); } @@ -3416,6 +3419,17 @@ mod mirror_state_tests { "https://github.com/Gitlawb/node.git#main", "https://github.com/", "https://github.com/Gitlawb/node.git\n", + "https://localhost/Gitlawb/node.git", + "https://forge.localhost/Gitlawb/node.git", + "https://127.0.0.1/Gitlawb/node.git", + "https://[::1]/Gitlawb/node.git", + "https://169.254.169.254/Gitlawb/node.git", + "https://10.0.0.5/Gitlawb/node.git", + "https://172.16.0.5/Gitlawb/node.git", + "https://192.168.1.5/Gitlawb/node.git", + "https://forge.internal/Gitlawb/node.git", + "https://[::ffff:127.0.0.1]/Gitlawb/node.git", + "https://[64:ff9b::7f00:1]/Gitlawb/node.git", ] { assert!( validate_mirror_upstream_url(invalid).is_err(), @@ -3792,16 +3806,34 @@ mod mirror_state_tests { } #[sqlx::test] - async fn peer_mirror_rows_cannot_become_continuous_upstreams(pool: PgPool) { + async fn continuous_mirror_uuid_gate_leaves_peer_rows_untouched(pool: PgPool) { let db = migrated_db(pool).await; db.upsert_mirror_repo("z6MkPeer", "repo", "/srv/peer/repo.git", None, false) .await .unwrap(); - let result = db + let error = db .configure_inbound_mirror("z6MkPeer/repo", "https://github.com/Gitlawb/node.git") - .await; - assert!(result.is_err()); + .await + .unwrap_err(); + assert!( + format!("{error:#}") + .contains("continuous mirrors require a canonical UUID repository id"), + "unexpected error: {error:#}" + ); + + let unchanged: (Option, Option) = + sqlx::query_as("SELECT upstream_url, mirror_status FROM repos WHERE id = $1") + .bind("z6MkPeer/repo") + .fetch_one(db.pool()) + .await + .unwrap(); + assert_eq!(unchanged, (None, None)); + assert!(db + .get_repo_mirror_state("z6MkPeer/repo") + .await + .unwrap() + .is_none()); } } From 1b7639f0d7ad2e9276a854f54260f4b2a7927c85 Mon Sep 17 00:00:00 2001 From: KRATOS <84986124+gnanam1990@users.noreply.github.com> Date: Sat, 15 Aug 2026 00:33:01 +0530 Subject: [PATCH 5/9] fix(node): align peer URL validation message Origin-Session: local-d6a143 | Codex | 20 prompts --- crates/gitlawb-node/src/api/peers.rs | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/crates/gitlawb-node/src/api/peers.rs b/crates/gitlawb-node/src/api/peers.rs index 6f4ed200..bff7eb07 100644 --- a/crates/gitlawb-node/src/api/peers.rs +++ b/crates/gitlawb-node/src/api/peers.rs @@ -250,7 +250,7 @@ pub async fn announce( // the real peers under junk so node-origin repos stop replicating. if !is_public_http_url(&req.http_url) { return Err(AppError::BadRequest( - "http_url must be a public http(s) URL (no loopback, private, or .internal/.local hosts)".into(), + "http_url must be a public http(s) URL (no loopback, private, localhost, .localhost, .internal, or .local hosts)".into(), )); } @@ -1532,6 +1532,31 @@ mod tests { ) } + #[sqlx::test] + async fn announce_rejects_localhost_subdomains_with_an_accurate_error(pool: PgPool) { + let state = test_state(pool).await; + let did = Keypair::generate().did().to_string(); + let resp = announce_only(state.clone()) + .oneshot(announce_as( + &did, + &announce_body(&did, "https://node.localhost:7545"), + )) + .await + .unwrap(); + + let (status, error, message) = status_and_error(resp).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(error, "bad_request"); + assert_eq!( + message, + "http_url must be a public http(s) URL (no loopback, private, localhost, .localhost, .internal, or .local hosts)" + ); + assert!( + snapshot(&state.db, &did).await.is_none(), + "a rejected .localhost announce must leave no row behind" + ); + } + /// U4 scenario 1, and the first test the keyid branch has ever had: a caller /// who proved control of one DID must not announce another's. Kills /// neutralizing the handler's keyid comparison, and kills demoting this From a6d845900233f4d052a24dc3f83a62ce9303f14c Mon Sep 17 00:00:00 2001 From: KRATOS <84986124+gnanam1990@users.noreply.github.com> Date: Sat, 15 Aug 2026 09:47:18 +0530 Subject: [PATCH 6/9] fix(node): close remaining mirror review findings Origin-Session: local-d6a143 | Codex | 21 prompts --- crates/gitlawb-node/src/api/peers.rs | 8 ++- crates/gitlawb-node/src/api/webhooks.rs | 71 ++++++++++++++++++++-- crates/gitlawb-node/src/db/mod.rs | 80 +++++++++++++++++++------ 3 files changed, 135 insertions(+), 24 deletions(-) diff --git a/crates/gitlawb-node/src/api/peers.rs b/crates/gitlawb-node/src/api/peers.rs index bff7eb07..c11ba852 100644 --- a/crates/gitlawb-node/src/api/peers.rs +++ b/crates/gitlawb-node/src/api/peers.rs @@ -70,6 +70,8 @@ pub struct PeerResponse { pub reachable: bool, } +pub(crate) const PUBLIC_HTTP_URL_REQUIREMENT: &str = "must be a public http(s) URL (no loopback, private, localhost, .localhost, .internal, or .local hosts)"; + /// Extract an IPv4 address embedded in an IPv6 literal across the transition /// formats that carry one: IPv4-mapped (`::ffff:a.b.c.d`), IPv4-compatible /// (`::a.b.c.d`), 6to4 (`2002:WWXX:YYZZ::/16`), and the NAT64 well-known prefix @@ -249,9 +251,9 @@ pub async fn announce( // and turn our outbound sync-notify fan-out into an SSRF probe — and bury // the real peers under junk so node-origin repos stop replicating. if !is_public_http_url(&req.http_url) { - return Err(AppError::BadRequest( - "http_url must be a public http(s) URL (no loopback, private, localhost, .localhost, .internal, or .local hosts)".into(), - )); + return Err(AppError::BadRequest(format!( + "http_url {PUBLIC_HTTP_URL_REQUIREMENT}" + ))); } // Reject self-announcements: a peer row whose http_url is our own public diff --git a/crates/gitlawb-node/src/api/webhooks.rs b/crates/gitlawb-node/src/api/webhooks.rs index d6a6997c..7cecd202 100644 --- a/crates/gitlawb-node/src/api/webhooks.rs +++ b/crates/gitlawb-node/src/api/webhooks.rs @@ -46,9 +46,10 @@ pub async fn create_webhook( // endpoints (SSRF). Delivery runs on the shared no-redirect client // (main.rs), which closes the 3xx-to-internal bounce. if !crate::api::peers::is_public_http_url(&req.url) { - return Err(AppError::BadRequest( - "webhook URL must be a public http(s) URL (no loopback, private, or .internal/.local hosts)".into(), - )); + return Err(AppError::BadRequest(format!( + "webhook URL {}", + crate::api::peers::PUBLIC_HTTP_URL_REQUIREMENT + ))); } let events = req.events.unwrap_or_else(|| vec!["*".into()]); @@ -141,7 +142,19 @@ pub async fn delete_webhook( #[cfg(test)] mod tests { - use crate::api::peers::is_public_http_url; + use axum::extract::{Extension, Path, State}; + use axum::Json; + use chrono::Utc; + use gitlawb_core::identity::Keypair; + use sqlx::PgPool; + use uuid::Uuid; + + use super::{create_webhook, CreateWebhookRequest}; + use crate::api::peers::{is_public_http_url, PUBLIC_HTTP_URL_REQUIREMENT}; + use crate::auth::AuthenticatedDid; + use crate::db::RepoRecord; + use crate::error::AppError; + use crate::test_support::test_state; // create_webhook gates req.url through is_public_http_url. Pin the exact // SSRF targets from issue #81 so the webhook path can never regress to the @@ -152,6 +165,7 @@ mod tests { "http://127.0.0.1:5432/", "http://169.254.169.254/latest/meta-data/", "http://localhost/", + "http://node.localhost/", "http://10.0.0.5/", "http://[::1]/", // IPv6 transition encodings smuggling loopback v4 (6to4 / NAT64). @@ -169,4 +183,53 @@ mod tests { assert!(is_public_http_url("https://hooks.example.com/gitlawb")); assert!(is_public_http_url("http://203.0.113.10:7545/")); } + + #[sqlx::test] + async fn webhook_localhost_rejection_uses_shared_public_url_contract(pool: PgPool) { + let state = test_state(pool).await; + let owner = Keypair::generate().did().to_string(); + let repo_id = Uuid::new_v4().to_string(); + state + .db + .create_repo(&RepoRecord { + id: repo_id.clone(), + name: "webhook-contract".to_string(), + owner_did: owner.clone(), + description: None, + is_public: true, + default_branch: "main".to_string(), + created_at: Utc::now(), + updated_at: Utc::now(), + disk_path: "/tmp/webhook-contract.git".to_string(), + forked_from: None, + machine_id: None, + }) + .await + .unwrap(); + + let result = create_webhook( + State(state.clone()), + Extension(AuthenticatedDid(owner.clone())), + Path((owner, "webhook-contract".to_string())), + Json(CreateWebhookRequest { + url: "https://node.localhost/hook".to_string(), + secret: None, + events: None, + }), + ) + .await; + + match result { + Err(AppError::BadRequest(message)) => assert_eq!( + message, + format!("webhook URL {PUBLIC_HTTP_URL_REQUIREMENT}") + ), + Err(error) => panic!("unexpected webhook rejection: {error}"), + Ok(_) => panic!("a .localhost webhook must be rejected"), + } + assert!( + state.db.list_webhooks(&repo_id).await.unwrap().is_empty(), + "a rejected .localhost webhook must leave no row behind" + ); + } } diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index eebc3b64..f007e283 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -1043,19 +1043,14 @@ const MIGRATIONS: &[Migration] = &[ "ALTER TABLE repos ADD COLUMN IF NOT EXISTS mirror_transition_id UUID", "ALTER TABLE repos ADD COLUMN IF NOT EXISTS mirror_transition_phase TEXT", "ALTER TABLE repos ADD COLUMN IF NOT EXISTS mirror_updated_at TEXT", - // PostgreSQL has no ADD CONSTRAINT IF NOT EXISTS, so guard by name - // to keep manual/idempotent recovery safe. - r#"DO $$ - BEGIN - IF NOT EXISTS ( - SELECT 1 FROM pg_constraint - WHERE conrelid = 'repos'::regclass - AND conname = 'repos_mirror_state_valid' - ) THEN - -- PostgreSQL accepts a CHECK whose result is NULL. Wrap - -- the complete invariant in IS TRUE so a missing status - -- or phase cannot slip through three-valued logic. - ALTER TABLE repos ADD CONSTRAINT repos_mirror_state_valid CHECK (( + // A constraint name does not prove its definition. Replace any + // same-named manual or partial-recovery artifact before installing + // the exact invariant this migration promises. + "ALTER TABLE repos DROP CONSTRAINT IF EXISTS repos_mirror_state_valid", + // PostgreSQL accepts a CHECK whose result is NULL. Wrap the complete + // invariant in IS TRUE so a missing status or phase cannot slip + // through three-valued logic. + r#"ALTER TABLE repos ADD CONSTRAINT repos_mirror_state_valid CHECK (( ( upstream_url IS NULL AND mirror_status IS NULL @@ -1098,10 +1093,7 @@ const MIGRATIONS: &[Migration] = &[ ) ) ) - ) IS TRUE) NOT VALID; - END IF; - END - $$"#, + ) IS TRUE) NOT VALID"#, ], }, Migration { @@ -3542,6 +3534,60 @@ mod mirror_state_tests { db.run_migrations().await.unwrap(); } + #[sqlx::test] + async fn migration_replaces_a_same_named_weaker_constraint(pool: PgPool) { + let db = migrated_db(pool).await; + + sqlx::query("ALTER TABLE repos DROP CONSTRAINT repos_mirror_state_valid") + .execute(db.pool()) + .await + .unwrap(); + sqlx::query( + "ALTER TABLE repos ADD CONSTRAINT repos_mirror_state_valid CHECK (TRUE) NOT VALID", + ) + .execute(db.pool()) + .await + .unwrap(); + sqlx::query("DELETE FROM schema_migrations WHERE version IN (30, 31)") + .execute(db.pool()) + .await + .unwrap(); + + db.run_migrations().await.unwrap(); + + let definition: String = sqlx::query_scalar( + "SELECT pg_get_constraintdef(oid) + FROM pg_constraint + WHERE conrelid = 'repos'::regclass + AND conname = 'repos_mirror_state_valid'", + ) + .fetch_one(db.pool()) + .await + .unwrap(); + assert!( + definition.contains("transitioning_inbound_to_outbound") + && !definition.eq_ignore_ascii_case("CHECK (true)"), + "migration preserved an impostor constraint: {definition}" + ); + + let existing = repo(Uuid::new_v4()); + db.create_repo(&existing).await.unwrap(); + let malformed = sqlx::query( + "UPDATE repos + SET upstream_url = 'https://github.com/Gitlawb/node.git', + mirror_status = 'transitioning_inbound_to_outbound', + mirror_updated_at = '2026-08-14T00:00:00Z' + WHERE id = $1", + ) + .bind(&existing.id) + .execute(db.pool()) + .await; + assert!( + malformed.is_err(), + "replacement constraint accepted missing transition metadata" + ); + } + #[sqlx::test] async fn inbound_configuration_round_trips_without_overwriting_state(pool: PgPool) { let db = migrated_db(pool).await; From 5cad709659a3fca241e79a43d0bcdadec4c14da8 Mon Sep 17 00:00:00 2001 From: KRATOS <84986124+gnanam1990@users.noreply.github.com> Date: Fri, 14 Aug 2026 23:16:31 +0530 Subject: [PATCH 7/9] feat(node): add scheduled inbound mirror worker Origin-Session: local-d6a143 | Codex | 18 prompts --- .env.example | 17 + crates/gitlawb-node/src/api/peers.rs | 107 +++-- crates/gitlawb-node/src/config.rs | 91 ++++ crates/gitlawb-node/src/db/mod.rs | 117 ++++- crates/gitlawb-node/src/main.rs | 17 + crates/gitlawb-node/src/upstream_mirror.rs | 531 +++++++++++++++++++++ 6 files changed, 830 insertions(+), 50 deletions(-) create mode 100644 crates/gitlawb-node/src/upstream_mirror.rs diff --git a/.env.example b/.env.example index b70d1117..4529eb22 100644 --- a/.env.example +++ b/.env.example @@ -274,6 +274,23 @@ GITLAWB_TRUSTED_PROXY= # Enable automatic background sync from known peers GITLAWB_AUTO_SYNC=false +# ── External upstream mirroring ─────────────────────────────────────────── +# Default-off scheduled fetch worker for repositories configured as INBOUND. +# Enabling requires GITLAWB_ENFORCE_OWNER_PUSH=true; the node refuses to start +# otherwise. The worker updates branches and tags atomically while preserving +# Gitlawb-owned review/internal refs. +GITLAWB_UPSTREAM_MIRROR_ENABLED=false +# Pause after each complete scan. Must be 5..=86400 seconds. +GITLAWB_UPSTREAM_MIRROR_INTERVAL_SECS=300 +# Per-repository fetch deadline. Must be 1..=3153600000 seconds. +GITLAWB_UPSTREAM_MIRROR_FETCH_TIMEOUT_SECS=600 +# Keyset page size; every page is processed before the next scan delay. +GITLAWB_UPSTREAM_MIRROR_PAGE_SIZE=100 +# Exact operator-approved hostnames that may resolve to RFC1918/ULA addresses +# (for example an on-prem GitHub Enterprise host). Public hosts need no entry. +# Loopback/link-local targets and wildcards remain forbidden. +GITLAWB_UPSTREAM_MIRROR_ALLOWED_PRIVATE_HOSTS= + # ── iCaptcha proof-of-intelligence gate ─────────────────────────────────── # Optional gate on create_repo + register: require callers to present an # iCaptcha proof (X-ICaptcha-Proof header) earned at icaptcha.gitlawb.com. diff --git a/crates/gitlawb-node/src/api/peers.rs b/crates/gitlawb-node/src/api/peers.rs index c11ba852..5966ea52 100644 --- a/crates/gitlawb-node/src/api/peers.rs +++ b/crates/gitlawb-node/src/api/peers.rs @@ -104,6 +104,63 @@ fn embedded_ipv4(v6: std::net::Ipv6Addr) -> Option { None } +/// Whether an IP literal is outside the local/private ranges that an +/// attacker-controlled outbound URL must never reach. Kept separate from URL +/// parsing so callers that resolve a DNS name themselves can apply the same +/// classification to every answer and then pin the approved addresses. +pub(crate) fn is_public_ip(ip: std::net::IpAddr) -> bool { + // Reject loopback/unspecified on the literal as given — catches `::1` and + // `::` before the IPv4-folding below (`::1`.to_ipv4() would otherwise map + // to a non-loopback `0.0.0.1`). + if ip.is_loopback() || ip.is_unspecified() { + return false; + } + // Fold any IPv6 literal that embeds an IPv4 address (mapped, compatible, + // 6to4, NAT64) down to that IPv4 so the v4 range checks catch + // loopback/private addresses smuggled in via an IPv6 encoding, then + // re-check loopback/unspecified. + let ip = match ip { + std::net::IpAddr::V6(v6) => embedded_ipv4(v6).map(std::net::IpAddr::V4).unwrap_or(ip), + v4 => v4, + }; + if ip.is_loopback() || ip.is_unspecified() { + return false; + } + match ip { + std::net::IpAddr::V4(v4) => { + let o = v4.octets(); + // RFC1918 private, link-local, CGNAT (100.64.0.0/10), or the + // RFC1122 "this host" block 0.0.0.0/8 (never a valid destination; + // 0.0.0.0 itself is already caught by the is_unspecified check). + if v4.is_private() + || v4.is_link_local() + || (o[0] == 100 && (o[1] & 0xc0) == 64) + || o[0] == 0 + { + return false; + } + } + std::net::IpAddr::V6(v6) => { + let s = v6.segments(); + // fc00::/7 (unique-local) or fe80::/10 (link-local) + if (s[0] & 0xfe00) == 0xfc00 || (s[0] & 0xffc0) == 0xfe80 { + return false; + } + // Any NAT64 address (64:ff9b::/32) that is not the cleanly + // decodable well-known /96 — e.g. the RFC 8215 local-use + // 64:ff9b:1::/48 — carries a translated target whose embedded v4 + // sits at a prefix-length-dependent offset (RFC 6052 §2.2). + // Rather than risk a wrong decode across every prefix length we + // reject the whole NAT64 space here. The well-known /96 was already + // folded to its v4 above and never reaches this arm. + if s[0] == 0x0064 && s[1] == 0xff9b { + return false; + } + } + } + true +} + /// Whether a peer `http_url` is a public http(s) endpoint safe to register. /// Rejects non-http(s) schemes, loopback/unspecified/private/link-local IPs, /// and `localhost` / `.localhost` / `.local` / `.internal` hostnames. Used at @@ -137,55 +194,7 @@ pub fn is_public_http_url(raw: &str) -> bool { // before parsing as an IP. let ip_candidate = host.trim_start_matches('[').trim_end_matches(']'); if let Ok(ip) = ip_candidate.parse::() { - // Reject loopback/unspecified on the literal as given — catches `::1` - // and `::` before the IPv4-folding below (`::1`.to_ipv4() would - // otherwise map to a non-loopback `0.0.0.1`). - if ip.is_loopback() || ip.is_unspecified() { - return false; - } - // Fold any IPv6 literal that embeds an IPv4 address (mapped, compatible, - // 6to4, NAT64) down to that IPv4 so the v4 range checks catch - // loopback/private addresses smuggled in via an IPv6 encoding, then - // re-check loopback/unspecified. - let ip = match ip { - std::net::IpAddr::V6(v6) => embedded_ipv4(v6).map(std::net::IpAddr::V4).unwrap_or(ip), - v4 => v4, - }; - if ip.is_loopback() || ip.is_unspecified() { - return false; - } - match ip { - std::net::IpAddr::V4(v4) => { - let o = v4.octets(); - // RFC1918 private, link-local, CGNAT (100.64.0.0/10), or the - // RFC1122 "this host" block 0.0.0.0/8 (never a valid destination; - // 0.0.0.0 itself is already caught by the is_unspecified check). - if v4.is_private() - || v4.is_link_local() - || (o[0] == 100 && (o[1] & 0xc0) == 64) - || o[0] == 0 - { - return false; - } - } - std::net::IpAddr::V6(v6) => { - let s = v6.segments(); - // fc00::/7 (unique-local) or fe80::/10 (link-local) - if (s[0] & 0xfe00) == 0xfc00 || (s[0] & 0xffc0) == 0xfe80 { - return false; - } - // Any NAT64 address (64:ff9b::/32) that is not the cleanly - // decodable well-known /96 — e.g. the RFC 8215 local-use - // 64:ff9b:1::/48 — carries a translated target whose embedded v4 - // sits at a prefix-length-dependent offset (RFC 6052 §2.2). - // Rather than risk a wrong decode across every prefix length we - // reject the whole NAT64 space here. The well-known /96 was - // already folded to its v4 above and never reaches this arm. - if s[0] == 0x0064 && s[1] == 0xff9b { - return false; - } - } - } + return is_public_ip(ip); } true } diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index fefa063e..97ed89d7 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -113,6 +113,56 @@ pub struct Config { #[arg(long, env = "GITLAWB_AUTO_SYNC", default_value_t = false)] pub auto_sync: bool, + /// Periodically fetch configured INBOUND repositories from their external + /// HTTPS upstream. Disabled by default: enabling this also requires + /// `GITLAWB_ENFORCE_OWNER_PUSH=true`, so an upstream-authoritative mirror + /// cannot accept writes from arbitrary signed identities between fetches. + #[arg(long, env = "GITLAWB_UPSTREAM_MIRROR_ENABLED", default_value_t = false)] + pub upstream_mirror_enabled: bool, + + /// Pause between complete upstream-mirror scans. A scan is sequential and + /// paginated; the pause begins only after the prior scan has finished, so a + /// slow forge cannot create overlapping worker cycles. + #[arg( + long, + env = "GITLAWB_UPSTREAM_MIRROR_INTERVAL_SECS", + default_value_t = 300, + value_parser = clap::value_parser!(u64).range(5..=86_400) + )] + pub upstream_mirror_interval_secs: u64, + + /// Maximum wall-clock time for one external `git fetch`. The child runs in + /// its own process group and is fully reaped on timeout. + #[arg( + long, + env = "GITLAWB_UPSTREAM_MIRROR_FETCH_TIMEOUT_SECS", + default_value_t = 600, + value_parser = clap::value_parser!(u64).range(1..=GIT_SERVICE_TIMEOUT_SECS_MAX) + )] + pub upstream_mirror_fetch_timeout_secs: u64, + + /// Number of durable INBOUND rows read per database page. Every page is + /// processed before the scan sleeps; this bounds query memory without + /// permanently starving repositories beyond the first page. + #[arg( + long, + env = "GITLAWB_UPSTREAM_MIRROR_PAGE_SIZE", + default_value_t = 100, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_000) + )] + pub upstream_mirror_page_size: usize, + + /// Exact upstream hostnames permitted to resolve to RFC1918/ULA addresses, + /// for operator-approved GitHub Enterprise or other on-prem forges. + /// Public addresses need no entry. Loopback and link-local destinations are + /// always rejected, even when listed. Wildcards are not supported. + #[arg( + long, + env = "GITLAWB_UPSTREAM_MIRROR_ALLOWED_PRIVATE_HOSTS", + value_delimiter = ',' + )] + pub upstream_mirror_allowed_private_hosts: Vec, + /// Irys URL for Arweave permanent anchoring. /// Leave empty to disable. Use https://devnet.irys.xyz for free devnet. #[arg(long, env = "GITLAWB_IRYS_URL", default_value = "")] @@ -583,6 +633,13 @@ impl Config { floor )); } + if self.upstream_mirror_enabled && !self.enforce_owner_push { + return Err( + "GITLAWB_UPSTREAM_MIRROR_ENABLED requires GITLAWB_ENFORCE_OWNER_PUSH=true: \ + an upstream-authoritative mirror must not accept writes from arbitrary signed identities" + .to_string(), + ); + } Ok(()) } } @@ -591,6 +648,40 @@ impl Config { mod tests { use super::*; + #[test] + fn upstream_mirror_worker_is_default_off_and_bounded() { + let config = Config::parse_from(["gitlawb-node"]); + assert!(!config.upstream_mirror_enabled); + assert_eq!(config.upstream_mirror_interval_secs, 300); + assert_eq!(config.upstream_mirror_fetch_timeout_secs, 600); + assert_eq!(config.upstream_mirror_page_size, 100); + assert!(config.upstream_mirror_allowed_private_hosts.is_empty()); + config.validate().unwrap(); + + assert!( + Config::try_parse_from(["gitlawb-node", "--upstream-mirror-interval-secs", "4",]) + .is_err() + ); + assert!( + Config::try_parse_from(["gitlawb-node", "--upstream-mirror-page-size", "0",]).is_err() + ); + } + + #[test] + fn upstream_mirror_worker_requires_owner_push_enforcement() { + let unsafe_config = Config::parse_from(["gitlawb-node", "--upstream-mirror-enabled"]); + let error = unsafe_config.validate().unwrap_err(); + assert!(error.contains("GITLAWB_ENFORCE_OWNER_PUSH=true")); + + Config::parse_from([ + "gitlawb-node", + "--upstream-mirror-enabled", + "--enforce-owner-push", + ]) + .validate() + .unwrap(); + } + #[test] fn git_service_timeout_defaults_to_600_and_rejects_zero() { assert_eq!( diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index f007e283..e5dacdc9 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -103,6 +103,17 @@ pub struct RepoMirrorState { pub updated_at: DateTime, } +/// One canonical repository eligible for a scheduled external-forge fetch. +/// The worker keyset-pages these rows by `repo_id`; transition metadata is not +/// exposed because the selecting query admits only stable INBOUND state. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct InboundMirrorTarget { + pub repo_id: String, + pub owner_did: String, + pub name: String, + pub upstream_url: String, +} + #[cfg_attr(not(test), allow(dead_code))] impl RepoMirrorState { fn validate(&self) -> Result<()> { @@ -122,7 +133,7 @@ impl RepoMirrorState { } #[cfg_attr(not(test), allow(dead_code))] -fn validate_mirror_upstream_url(raw: &str) -> Result { +pub(crate) fn validate_mirror_upstream_url(raw: &str) -> Result { if raw.is_empty() || raw.len() > 2048 { anyhow::bail!("mirror upstream URL must contain 1 to 2048 bytes"); } @@ -1277,6 +1288,51 @@ impl Db { row.map(row_to_repo_mirror_state).transpose() } + /// Keyset-page stable INBOUND mirrors for the external-forge worker. + /// Transitioning/outbound rows are excluded at the database boundary so a + /// cycle cannot intentionally schedule them. The worker rechecks the state + /// after taking the repo write lock to close the selection-to-fetch window. + pub async fn list_inbound_mirror_targets( + &self, + after_repo_id: Option<&str>, + limit: i64, + ) -> Result> { + let rows = sqlx::query( + "SELECT id, owner_did, name, upstream_url + FROM repos + WHERE mirror_status = 'inbound' + AND upstream_url IS NOT NULL + AND mirror_transition_id IS NULL + AND mirror_transition_phase IS NULL + AND ($1::text IS NULL OR id > $1) + ORDER BY id ASC + LIMIT $2", + ) + .bind(after_repo_id) + .bind(limit.clamp(1, 1_000)) + .fetch_all(&self.pool) + .await?; + + rows.into_iter() + .map(|row| { + Ok(InboundMirrorTarget { + repo_id: row + .try_get("id") + .context("reading inbound mirror repository id")?, + owner_did: row + .try_get("owner_did") + .context("reading inbound mirror owner DID")?, + name: row + .try_get("name") + .context("reading inbound mirror repository name")?, + upstream_url: row + .try_get("upstream_url") + .context("reading inbound mirror upstream URL")?, + }) + }) + .collect() + } + /// Register a mirrored repo from a peer in the local DB so git smart HTTP can serve it. /// Uses INSERT OR IGNORE (SQLite) / ON CONFLICT DO NOTHING (Postgres) so it's idempotent. pub async fn upsert_mirror_repo( @@ -3638,6 +3694,65 @@ mod mirror_state_tests { assert_eq!(stored.updated_at, original_updated_at); } + #[sqlx::test] + async fn inbound_worker_pages_all_and_only_stable_inbound_mirrors(pool: PgPool) { + let db = migrated_db(pool).await; + let ids = [ + Uuid::parse_str("00000000-0000-0000-0000-000000000001").unwrap(), + Uuid::parse_str("00000000-0000-0000-0000-000000000002").unwrap(), + Uuid::parse_str("00000000-0000-0000-0000-000000000003").unwrap(), + ]; + for (index, id) in ids.into_iter().enumerate() { + let mut existing = repo(id); + existing.name = format!("mirror-{index}"); + db.create_repo(&existing).await.unwrap(); + db.configure_inbound_mirror( + &existing.id, + &format!("https://github.example/org/mirror-{index}.git"), + ) + .await + .unwrap(); + } + + // The middle row simulates a transition selected by the next B1 slice. + // It must not be scheduled while authority is changing. + sqlx::query( + "UPDATE repos + SET mirror_status = 'transitioning_inbound_to_outbound', + mirror_transition_id = $2, + mirror_transition_phase = 'queued' + WHERE id = $1", + ) + .bind(ids[1].to_string()) + .bind(Uuid::new_v4()) + .execute(db.pool()) + .await + .unwrap(); + + let first = db.list_inbound_mirror_targets(None, 1).await.unwrap(); + assert_eq!(first.len(), 1); + assert_eq!(first[0].repo_id, ids[0].to_string()); + assert_eq!(first[0].name, "mirror-0"); + assert_eq!(first[0].owner_did, "did:key:z6MkMirrorOwner"); + assert_eq!( + first[0].upstream_url, + "https://github.example/org/mirror-0.git" + ); + + let second = db + .list_inbound_mirror_targets(Some(&first[0].repo_id), 1) + .await + .unwrap(); + assert_eq!(second.len(), 1); + assert_eq!(second[0].repo_id, ids[2].to_string()); + + let exhausted = db + .list_inbound_mirror_targets(Some(&second[0].repo_id), 1) + .await + .unwrap(); + assert!(exhausted.is_empty()); + } + #[sqlx::test] async fn malformed_mirror_row_returns_an_error_instead_of_panicking(pool: PgPool) { let row = sqlx::query( diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 10aaca5b..cda0a3ca 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -21,6 +21,7 @@ mod state; mod sync; #[cfg(test)] mod test_support; +mod upstream_mirror; mod visibility; mod webhooks; @@ -572,6 +573,22 @@ async fn main() -> Result<()> { info!("auto-sync worker started"); } + // External-forge mirroring is a separate, default-off worker. Startup + // validation already requires owner-only pushes when it is enabled; the + // worker additionally verifies the Git runtime can pin DNS answers before + // any URL reaches `git fetch`. + if config.upstream_mirror_enabled { + upstream_mirror::start( + Arc::clone(&state.db), + Arc::clone(&state.config), + state.repo_store.clone(), + state.repo_write_leases.clone(), + Arc::clone(&state.git_write_semaphore), + state.subscribe_shutdown(), + ) + .context("starting upstream mirror worker")?; + } + // On-chain operator setup: verify stake + spawn heartbeat loop if !state.config.contract_node_staking.is_empty() && !state.config.operator_private_key.is_empty() diff --git a/crates/gitlawb-node/src/upstream_mirror.rs b/crates/gitlawb-node/src/upstream_mirror.rs new file mode 100644 index 00000000..0db50d5d --- /dev/null +++ b/crates/gitlawb-node/src/upstream_mirror.rs @@ -0,0 +1,531 @@ +//! Scheduled external-forge fetches for canonical INBOUND mirrors. +//! +//! This worker is deliberately separate from `sync`, which mirrors repositories +//! between Gitlawb peers. External upstream URLs are owner-controlled data that +//! eventually reach `git fetch`, so each cycle resolves DNS itself, validates +//! every address, and pins the approved answers into Git/libcurl with +//! `http.curloptResolve`. Redirects, proxies, credential helpers, submodule +//! recursion, and non-HTTPS protocols are disabled at the command boundary. + +use anyhow::{Context, Result}; +use std::collections::{BTreeSet, HashSet}; +use std::net::IpAddr; +use std::path::Path; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tracing::{info, warn}; + +use crate::config::Config; +use crate::db::{Db, InboundMirrorTarget, MirrorStatus}; +use crate::git::repo_store::RepoStore; +use crate::state::{repo_identity_key, RepoWriteLeases}; + +const DNS_TIMEOUT: Duration = Duration::from_secs(10); +const MIN_CURL_OPT_RESOLVE_GIT: (u64, u64) = (2, 37); + +#[derive(Debug, Clone)] +struct EgressPolicy { + allowed_private_hosts: HashSet, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct PinnedUpstream { + url: String, + curlopt_resolve: Option, +} + +impl EgressPolicy { + fn new(allowed_private_hosts: &[String]) -> Result { + let allowed_private_hosts = allowed_private_hosts + .iter() + .map(|host| normalize_allowlisted_host(host)) + .collect::>>()?; + Ok(Self { + allowed_private_hosts, + }) + } + + async fn resolve(&self, url: reqwest::Url) -> Result { + let raw_host = url.host_str().context("mirror upstream URL has no host")?; + let connect_host = raw_host + .trim_start_matches('[') + .trim_end_matches(']') + .to_ascii_lowercase(); + let policy_host = normalize_policy_host(&connect_host); + reject_always_local_hostname(&policy_host)?; + let allow_private = self.allowed_private_hosts.contains(&policy_host); + let port = url + .port_or_known_default() + .context("mirror upstream URL has no known port")?; + + if let Ok(ip) = connect_host.parse::() { + self.validate_addresses(&policy_host, &[ip], allow_private)?; + return Ok(PinnedUpstream { + url: url.to_string(), + // An IP literal performs no DNS lookup, so there is no rebinding + // window to pin. TLS still verifies the literal from the URL. + curlopt_resolve: None, + }); + } + + let resolved = tokio::time::timeout( + DNS_TIMEOUT, + tokio::net::lookup_host((connect_host.as_str(), port)), + ) + .await + .context("mirror upstream DNS lookup timed out")? + .context("resolving mirror upstream host")?; + let addresses: BTreeSet = resolved.map(|addr| addr.ip()).collect(); + if addresses.is_empty() { + anyhow::bail!("mirror upstream host resolved to no addresses"); + } + let addresses: Vec = addresses.into_iter().collect(); + self.validate_addresses(&policy_host, &addresses, allow_private)?; + + let pinned = addresses + .iter() + .map(|ip| match ip { + IpAddr::V4(v4) => v4.to_string(), + IpAddr::V6(v6) => format!("[{v6}]"), + }) + .collect::>() + .join(","); + Ok(PinnedUpstream { + url: url.to_string(), + curlopt_resolve: Some(format!("+{connect_host}:{port}:{pinned}")), + }) + } + + fn validate_addresses( + &self, + host: &str, + addresses: &[IpAddr], + allow_private: bool, + ) -> Result<()> { + for &ip in addresses { + if !address_is_permitted(ip, allow_private) { + anyhow::bail!("mirror upstream host {host:?} resolved to disallowed address {ip}"); + } + } + Ok(()) + } +} + +fn normalize_policy_host(host: &str) -> String { + host.trim_end_matches('.').to_ascii_lowercase() +} + +fn reject_always_local_hostname(host: &str) -> Result<()> { + if host.is_empty() + || host == "localhost" + || host.ends_with(".localhost") + || host.ends_with(".local") + { + anyhow::bail!("mirror upstream host is local-only and cannot be fetched"); + } + Ok(()) +} + +fn normalize_allowlisted_host(raw: &str) -> Result { + let trimmed = raw.trim(); + if trimmed.is_empty() + || trimmed.contains(|c: char| c.is_whitespace() || c.is_control()) + || trimmed.contains(['/', '?', '#', '@', '*']) + { + anyhow::bail!("invalid private mirror host allowlist entry {raw:?}"); + } + let unbracketed = trimmed + .strip_prefix('[') + .and_then(|value| value.strip_suffix(']')) + .unwrap_or(trimmed); + let normalized = normalize_policy_host(unbracketed); + reject_always_local_hostname(&normalized)?; + + if normalized.parse::().is_err() + && !normalized + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '-' | '_')) + { + anyhow::bail!("invalid private mirror host allowlist entry {raw:?}"); + } + Ok(normalized) +} + +fn address_is_permitted(ip: IpAddr, allow_private: bool) -> bool { + // Broadcast/multicast/reserved destinations are never forge endpoints. + match ip { + IpAddr::V4(v4) if v4.octets()[0] >= 224 => return false, + IpAddr::V6(v6) if v6.is_multicast() => return false, + _ => {} + } + if crate::api::peers::is_public_ip(ip) { + return true; + } + if !allow_private { + return false; + } + // An exact operator allowlist may admit only normal private address space + // used by an on-prem forge. Loopback, link-local, CGNAT, unspecified, and + // transition encodings remain denied because neither arm includes them. + match ip { + IpAddr::V4(v4) => v4.is_private(), + IpAddr::V6(v6) => (v6.segments()[0] & 0xfe00) == 0xfc00, + } +} + +fn parse_git_version(output: &str) -> Option<(u64, u64)> { + let version = output.split_whitespace().find(|part| { + part.as_bytes() + .first() + .is_some_and(|byte| byte.is_ascii_digit()) + })?; + let mut parts = version.split('.'); + Some((parts.next()?.parse().ok()?, parts.next()?.parse().ok()?)) +} + +fn ensure_safe_git_runtime(git_bin: &str) -> Result<()> { + let output = std::process::Command::new(git_bin) + .arg("--version") + .output() + .with_context(|| format!("running {git_bin} --version for upstream mirror safety check"))?; + if !output.status.success() { + anyhow::bail!("{git_bin} --version failed"); + } + let stdout = String::from_utf8_lossy(&output.stdout); + let version = parse_git_version(&stdout).context("parsing git version")?; + if version < MIN_CURL_OPT_RESOLVE_GIT { + anyhow::bail!( + "upstream mirroring requires Git 2.37+ for DNS pinning; found {}.{}", + version.0, + version.1 + ); + } + Ok(()) +} + +fn fetch_args(upstream: &PinnedUpstream) -> Vec { + let mut args = vec![ + "-c".to_string(), + "http.proxy=".to_string(), + "-c".to_string(), + "http.followRedirects=false".to_string(), + // Reset any inherited multi-valued resolver entries before adding the + // address set validated for this exact fetch. + "-c".to_string(), + "http.curloptResolve=".to_string(), + // Never forward operator or repository-scoped HTTP credentials to an + // owner-selected forge. Empty values reset Git's multi-valued headers + // and disable cookie state for this invocation. + "-c".to_string(), + "http.extraHeader=".to_string(), + "-c".to_string(), + "http.cookieFile=".to_string(), + "-c".to_string(), + "http.saveCookies=false".to_string(), + "-c".to_string(), + "http.sslVerify=true".to_string(), + ]; + if let Some(resolve) = &upstream.curlopt_resolve { + args.extend(["-c".to_string(), format!("http.curloptResolve={resolve}")]); + } + args.extend([ + "-c".to_string(), + "credential.helper=".to_string(), + "-c".to_string(), + "core.askPass=false".to_string(), + "-c".to_string(), + "fetch.recurseSubmodules=false".to_string(), + "-c".to_string(), + "protocol.allow=never".to_string(), + "-c".to_string(), + "protocol.https.allow=always".to_string(), + "fetch".to_string(), + "--atomic".to_string(), + "--force".to_string(), + "--prune".to_string(), + "--prune-tags".to_string(), + "--no-auto-maintenance".to_string(), + "--no-recurse-submodules".to_string(), + "--no-write-fetch-head".to_string(), + upstream.url.clone(), + "+refs/heads/*:refs/heads/*".to_string(), + "+refs/tags/*:refs/tags/*".to_string(), + ]); + args +} + +fn run_fetch( + git_bin: &str, + repo_path: &Path, + upstream: &PinnedUpstream, + timeout: Duration, +) -> Result<()> { + let args = fetch_args(upstream); + let borrowed: Vec<&str> = args.iter().map(String::as_str).collect(); + let deadline = Instant::now() + .checked_add(timeout) + .context("upstream mirror fetch timeout is not representable")?; + let (status, _stdout, stderr) = crate::git::visibility_pack::run_bounded_git_raw( + git_bin, &borrowed, repo_path, b"", deadline, + )?; + if !status.success() { + let stderr = String::from_utf8_lossy(&stderr); + let stderr = stderr.chars().take(4_096).collect::(); + anyhow::bail!("git upstream fetch failed: {stderr}"); + } + Ok(()) +} + +#[derive(Clone)] +struct Worker { + db: Arc, + config: Arc, + repo_store: RepoStore, + repo_write_leases: RepoWriteLeases, + git_write_semaphore: Arc, + policy: EgressPolicy, + git_bin: String, +} + +impl Worker { + async fn fetch_target(&self, target: InboundMirrorTarget) -> Result<()> { + let url = crate::db::validate_mirror_upstream_url(&target.upstream_url)?; + let upstream = self.policy.resolve(url).await?; + + let repo_identity = repo_identity_key(&target.owner_did, &target.name); + let steal_after = Duration::from_secs( + self.config + .upstream_mirror_fetch_timeout_secs + .saturating_mul(2) + .saturating_add(60), + ); + let _lease = self + .repo_write_leases + .acquire(&repo_identity, steal_after) + .await + .context("upstream mirror write-lease waiter cap reached")?; + + let _write_permit = Arc::clone(&self.git_write_semaphore) + .acquire_owned() + .await + .context("upstream mirror write semaphore closed")?; + let acquire_timeout = Duration::from_secs(self.config.git_acquire_timeout_secs); + let guard = tokio::time::timeout( + acquire_timeout, + self.repo_store + .acquire_write(&target.owner_did, &target.name), + ) + .await + .context("upstream mirror write-lock acquisition timed out")??; + let repo_path = guard.path().to_path_buf(); + + // A row can leave INBOUND state after selection. Recheck only after + // both the process-local lease and the cluster-wide advisory write lock + // are held. Every future transition path must take these same locks + // before changing authority, so no other node can commit a transition + // between this check and the fetch. + let current = match self.db.get_repo_mirror_state(&target.repo_id).await { + Ok(Some(current)) => current, + Ok(None) => { + guard.release(false).await; + anyhow::bail!("inbound mirror state disappeared before fetch"); + } + Err(error) => { + guard.release(false).await; + return Err(error); + } + }; + if current.status != MirrorStatus::Inbound + || current.transition_id.is_some() + || current.transition_phase.is_some() + || current.upstream_url != target.upstream_url + { + guard.release(false).await; + info!(repo_id = %target.repo_id, "upstream mirror state changed before fetch; skipping stale target"); + return Ok(()); + } + + let git_bin = self.git_bin.clone(); + let fetch_timeout = Duration::from_secs(self.config.upstream_mirror_fetch_timeout_secs); + let fetch = tokio::task::spawn_blocking(move || { + run_fetch(&git_bin, &repo_path, &upstream, fetch_timeout) + }) + .await + .context("upstream mirror fetch task panicked")?; + let success = fetch.is_ok(); + guard.release(success).await; + fetch + } + + async fn scan_once(&self) { + let mut cursor: Option = None; + loop { + let targets = match self + .db + .list_inbound_mirror_targets( + cursor.as_deref(), + self.config.upstream_mirror_page_size as i64, + ) + .await + { + Ok(targets) => targets, + Err(error) => { + warn!(err = %error, "failed to list inbound mirror targets"); + return; + } + }; + if targets.is_empty() { + return; + } + let page_len = targets.len(); + cursor = targets.last().map(|target| target.repo_id.clone()); + for target in targets { + let repo_id = target.repo_id.clone(); + let upstream_url = target.upstream_url.clone(); + match self.fetch_target(target).await { + Ok(()) => { + info!(repo_id, upstream = %upstream_url, "upstream mirror fetch completed") + } + Err(error) => { + warn!(repo_id, upstream = %upstream_url, err = %error, "upstream mirror fetch failed") + } + } + } + if page_len < self.config.upstream_mirror_page_size { + return; + } + } + } +} + +/// Validate the opt-in runtime and spawn the scheduled worker. The worker runs +/// one scan immediately, sleeps only after a complete scan, and exits between +/// scans when graceful shutdown is signalled. An in-flight Git child is bounded +/// and reaped before the worker observes shutdown. +pub fn start( + db: Arc, + config: Arc, + repo_store: RepoStore, + repo_write_leases: RepoWriteLeases, + git_write_semaphore: Arc, + mut shutdown_rx: tokio::sync::watch::Receiver, +) -> Result<()> { + ensure_safe_git_runtime("git")?; + let policy = EgressPolicy::new(&config.upstream_mirror_allowed_private_hosts)?; + let worker = Worker { + db, + config: Arc::clone(&config), + repo_store, + repo_write_leases, + git_write_semaphore, + policy, + git_bin: "git".to_string(), + }; + tokio::spawn(async move { + info!( + interval_secs = config.upstream_mirror_interval_secs, + "upstream mirror worker started" + ); + loop { + worker.scan_once().await; + tokio::select! { + _ = tokio::time::sleep(Duration::from_secs(config.upstream_mirror_interval_secs)) => {} + changed = shutdown_rx.changed() => { + if changed.is_err() || *shutdown_rx.borrow() { + info!("upstream mirror worker stopped"); + return; + } + } + } + } + }); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn git_version_gate_requires_curlopt_resolve_support() { + assert_eq!(parse_git_version("git version 2.37.0"), Some((2, 37))); + assert_eq!( + parse_git_version("git version 2.39.5 (Apple Git-154)"), + Some((2, 39)) + ); + assert_eq!(parse_git_version("not git"), None); + assert!((2, 36) < MIN_CURL_OPT_RESOLVE_GIT); + } + + #[test] + fn private_targets_require_an_exact_operator_allowlist() { + let policy = EgressPolicy::new(&["ghe.internal".to_string()]).unwrap(); + let public_v4: IpAddr = "8.8.8.8".parse().unwrap(); + let private_v4: IpAddr = "10.2.3.4".parse().unwrap(); + let private_v6: IpAddr = "fd00::20".parse().unwrap(); + assert!(policy + .validate_addresses("ghe.internal", &[private_v4, private_v6], true) + .is_ok()); + assert!(policy + .validate_addresses("other.internal", &[private_v4], false) + .is_err()); + assert!(policy + .validate_addresses("public.example", &[public_v4, private_v4], false) + .is_err()); + + for never in ["127.0.0.1", "169.254.169.254", "::1", "fe80::1"] { + let ip = never.parse().unwrap(); + assert!(!address_is_permitted(ip, true), "{never} must stay denied"); + } + } + + #[test] + fn private_host_allowlist_rejects_wildcards_and_localhost() { + for invalid in ["", "*.internal", "localhost", "forge.local", "host/path"] { + assert!(normalize_allowlisted_host(invalid).is_err(), "{invalid:?}"); + } + assert_eq!( + normalize_allowlisted_host("GHE.INTERNAL.").unwrap(), + "ghe.internal" + ); + } + + #[test] + fn fetch_command_pins_dns_and_only_updates_branches_and_tags() { + let upstream = PinnedUpstream { + url: "https://github.example/org/repo.git".to_string(), + curlopt_resolve: Some("+github.example:443:203.0.113.10,[2001:db8::10]".to_string()), + }; + let args = fetch_args(&upstream); + assert!(args.contains(&"http.followRedirects=false".to_string())); + assert!(args.contains(&"http.proxy=".to_string())); + assert!(args.contains(&"http.extraHeader=".to_string())); + assert!(args.contains(&"http.cookieFile=".to_string())); + assert!(args.contains(&"http.saveCookies=false".to_string())); + assert!(args.contains(&"http.sslVerify=true".to_string())); + assert!(args.contains( + &"http.curloptResolve=+github.example:443:203.0.113.10,[2001:db8::10]".to_string() + )); + assert!(args.contains(&"protocol.allow=never".to_string())); + assert!(args.contains(&"protocol.https.allow=always".to_string())); + assert!(args.contains(&"--atomic".to_string())); + assert!(args.contains(&"+refs/heads/*:refs/heads/*".to_string())); + assert!(args.contains(&"+refs/tags/*:refs/tags/*".to_string())); + assert!(!args.iter().any(|arg| arg == "+refs/*:refs/*")); + } + + #[tokio::test] + async fn literal_addresses_are_checked_without_dns() { + let policy = EgressPolicy::new(&["10.2.3.4".to_string()]).unwrap(); + let allowed = policy + .resolve(reqwest::Url::parse("https://10.2.3.4/org/repo.git").unwrap()) + .await + .unwrap(); + assert_eq!(allowed.curlopt_resolve, None); + + assert!(policy + .resolve(reqwest::Url::parse("https://127.0.0.1/org/repo.git").unwrap()) + .await + .is_err()); + } +} From 111d1ccd6b00ad17dc13c3bfcd662a5ccc7cf964 Mon Sep 17 00:00:00 2001 From: KRATOS <84986124+gnanam1990@users.noreply.github.com> Date: Sat, 15 Aug 2026 15:41:50 +0530 Subject: [PATCH 8/9] fix(node): harden scheduled mirror fetches Origin-Session: local-d6a143 | Codex | 24 prompts --- .env.example | 4 - crates/gitlawb-node/src/config.rs | 12 - .../gitlawb-node/src/git/visibility_pack.rs | 82 +++- crates/gitlawb-node/src/upstream_mirror.rs | 430 ++++++++++++++---- 4 files changed, 406 insertions(+), 122 deletions(-) diff --git a/.env.example b/.env.example index 4529eb22..d2c2591e 100644 --- a/.env.example +++ b/.env.example @@ -286,10 +286,6 @@ GITLAWB_UPSTREAM_MIRROR_INTERVAL_SECS=300 GITLAWB_UPSTREAM_MIRROR_FETCH_TIMEOUT_SECS=600 # Keyset page size; every page is processed before the next scan delay. GITLAWB_UPSTREAM_MIRROR_PAGE_SIZE=100 -# Exact operator-approved hostnames that may resolve to RFC1918/ULA addresses -# (for example an on-prem GitHub Enterprise host). Public hosts need no entry. -# Loopback/link-local targets and wildcards remain forbidden. -GITLAWB_UPSTREAM_MIRROR_ALLOWED_PRIVATE_HOSTS= # ── iCaptcha proof-of-intelligence gate ─────────────────────────────────── # Optional gate on create_repo + register: require callers to present an diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 97ed89d7..a64aba1b 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -152,17 +152,6 @@ pub struct Config { )] pub upstream_mirror_page_size: usize, - /// Exact upstream hostnames permitted to resolve to RFC1918/ULA addresses, - /// for operator-approved GitHub Enterprise or other on-prem forges. - /// Public addresses need no entry. Loopback and link-local destinations are - /// always rejected, even when listed. Wildcards are not supported. - #[arg( - long, - env = "GITLAWB_UPSTREAM_MIRROR_ALLOWED_PRIVATE_HOSTS", - value_delimiter = ',' - )] - pub upstream_mirror_allowed_private_hosts: Vec, - /// Irys URL for Arweave permanent anchoring. /// Leave empty to disable. Use https://devnet.irys.xyz for free devnet. #[arg(long, env = "GITLAWB_IRYS_URL", default_value = "")] @@ -655,7 +644,6 @@ mod tests { assert_eq!(config.upstream_mirror_interval_secs, 300); assert_eq!(config.upstream_mirror_fetch_timeout_secs, 600); assert_eq!(config.upstream_mirror_page_size, 100); - assert!(config.upstream_mirror_allowed_private_hosts.is_empty()); config.validate().unwrap(); assert!( diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index f26caf18..fb2fa18e 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -29,6 +29,44 @@ const WALK_TIMEOUT: Duration = Duration::from_secs(600); #[cfg(unix)] const WATCHDOG_TERM_GRACE: Duration = Duration::from_secs(1); +/// Build a Git child command, optionally with an isolated environment for an +/// owner-selected HTTPS destination. The isolated form intentionally drops +/// ambient proxy, credential-helper, askpass, and Git config environment that +/// could otherwise forward node-operator secrets to the selected upstream. +fn git_child_command( + git_bin: &str, + args: &[&str], + repo_path: &Path, + isolated_https: bool, +) -> std::process::Command { + let mut command = std::process::Command::new(git_bin); + command + .args(args) + .current_dir(repo_path) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + if isolated_https { + let path = std::env::var_os("PATH"); + command.env_clear(); + if let Some(path) = path { + command.env("PATH", path); + } + command + .env("GIT_CONFIG_NOSYSTEM", "1") + .env( + "GIT_CONFIG_GLOBAL", + if cfg!(windows) { "NUL" } else { "/dev/null" }, + ) + .env("GIT_TERMINAL_PROMPT", "0") + .env("GIT_ASKPASS", "false") + .env("SSH_ASKPASS", "false") + .env("LANG", "C") + .env("LC_ALL", "C"); + } + command +} + /// Run one git child under a shared `deadline` with process-group teardown, /// BLOCKING, and return its stdout. The child runs in its own process group; a /// watchdog thread SIGTERMs (lets git clean up its `*.lock` files), then SIGKILLs, @@ -67,24 +105,21 @@ fn child_terminated_without_reaping(pid: i32) -> bool { } #[cfg(unix)] -pub(crate) fn run_bounded_git_raw( +fn run_bounded_git_raw_impl( git_bin: &str, args: &[&str], repo_path: &Path, stdin_bytes: &[u8], deadline: Instant, + isolated_https: bool, ) -> Result<(std::process::ExitStatus, Vec, Vec)> { use std::io::{Read, Write}; use std::os::unix::process::CommandExt; use std::sync::mpsc::RecvTimeoutError; let label = args.first().copied().unwrap_or("git"); - let mut child = std::process::Command::new(git_bin) - .args(args) - .current_dir(repo_path) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) + let mut command = git_child_command(git_bin, args, repo_path, isolated_https); + let mut child = command .process_group(0) .spawn() .with_context(|| format!("failed to spawn git {label}"))?; @@ -194,6 +229,29 @@ pub(crate) fn run_bounded_git_raw( Ok((status, out, err)) } +pub(crate) fn run_bounded_git_raw( + git_bin: &str, + args: &[&str], + repo_path: &Path, + stdin_bytes: &[u8], + deadline: Instant, +) -> Result<(std::process::ExitStatus, Vec, Vec)> { + run_bounded_git_raw_impl(git_bin, args, repo_path, stdin_bytes, deadline, false) +} + +/// Run a bounded Git child without inheriting ambient Git/proxy/credential +/// configuration. Reserved for owner-selected external HTTPS destinations; +/// ordinary local Git operations retain their existing environment. +pub(crate) fn run_bounded_git_raw_isolated_https( + git_bin: &str, + args: &[&str], + repo_path: &Path, + stdin_bytes: &[u8], + deadline: Instant, +) -> Result<(std::process::ExitStatus, Vec, Vec)> { + run_bounded_git_raw_impl(git_bin, args, repo_path, stdin_bytes, deadline, true) +} + /// Bounded git returning only stdout, `bail!`ing on any nonzero exit. The thin /// wrapper the walk callers use. Probes that must distinguish exit classes — /// `git cat-file` absence vs an object-store access failure — call @@ -227,23 +285,19 @@ pub(crate) fn run_bounded_git( /// the Unix version's signature and result semantics so every caller compiles on all /// targets (#174). #[cfg(not(unix))] -pub(crate) fn run_bounded_git_raw( +fn run_bounded_git_raw_impl( git_bin: &str, args: &[&str], repo_path: &Path, stdin_bytes: &[u8], deadline: Instant, + isolated_https: bool, ) -> Result<(std::process::ExitStatus, Vec, Vec)> { use std::io::{Read, Write}; use std::sync::mpsc::RecvTimeoutError; let label = args.first().copied().unwrap_or("git"); - let mut child = std::process::Command::new(git_bin) - .args(args) - .current_dir(repo_path) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) + let mut child = git_child_command(git_bin, args, repo_path, isolated_https) .spawn() .with_context(|| format!("failed to spawn git {label}"))?; diff --git a/crates/gitlawb-node/src/upstream_mirror.rs b/crates/gitlawb-node/src/upstream_mirror.rs index 0db50d5d..4d57c9d2 100644 --- a/crates/gitlawb-node/src/upstream_mirror.rs +++ b/crates/gitlawb-node/src/upstream_mirror.rs @@ -8,7 +8,7 @@ //! recursion, and non-HTTPS protocols are disabled at the command boundary. use anyhow::{Context, Result}; -use std::collections::{BTreeSet, HashSet}; +use std::collections::BTreeSet; use std::net::IpAddr; use std::path::Path; use std::sync::Arc; @@ -21,12 +21,11 @@ use crate::git::repo_store::RepoStore; use crate::state::{repo_identity_key, RepoWriteLeases}; const DNS_TIMEOUT: Duration = Duration::from_secs(10); +const MAX_PINNED_ADDRESSES: usize = 32; const MIN_CURL_OPT_RESOLVE_GIT: (u64, u64) = (2, 37); -#[derive(Debug, Clone)] -struct EgressPolicy { - allowed_private_hosts: HashSet, -} +#[derive(Debug, Clone, Copy)] +struct EgressPolicy; #[derive(Debug, Clone, PartialEq, Eq)] struct PinnedUpstream { @@ -35,16 +34,6 @@ struct PinnedUpstream { } impl EgressPolicy { - fn new(allowed_private_hosts: &[String]) -> Result { - let allowed_private_hosts = allowed_private_hosts - .iter() - .map(|host| normalize_allowlisted_host(host)) - .collect::>>()?; - Ok(Self { - allowed_private_hosts, - }) - } - async fn resolve(&self, url: reqwest::Url) -> Result { let raw_host = url.host_str().context("mirror upstream URL has no host")?; let connect_host = raw_host @@ -53,13 +42,12 @@ impl EgressPolicy { .to_ascii_lowercase(); let policy_host = normalize_policy_host(&connect_host); reject_always_local_hostname(&policy_host)?; - let allow_private = self.allowed_private_hosts.contains(&policy_host); let port = url .port_or_known_default() .context("mirror upstream URL has no known port")?; if let Ok(ip) = connect_host.parse::() { - self.validate_addresses(&policy_host, &[ip], allow_private)?; + self.validate_addresses(&policy_host, &[ip])?; return Ok(PinnedUpstream { url: url.to_string(), // An IP literal performs no DNS lookup, so there is no rebinding @@ -80,7 +68,7 @@ impl EgressPolicy { anyhow::bail!("mirror upstream host resolved to no addresses"); } let addresses: Vec = addresses.into_iter().collect(); - self.validate_addresses(&policy_host, &addresses, allow_private)?; + self.validate_addresses(&policy_host, &addresses)?; let pinned = addresses .iter() @@ -96,14 +84,14 @@ impl EgressPolicy { }) } - fn validate_addresses( - &self, - host: &str, - addresses: &[IpAddr], - allow_private: bool, - ) -> Result<()> { + fn validate_addresses(&self, host: &str, addresses: &[IpAddr]) -> Result<()> { + if addresses.is_empty() || addresses.len() > MAX_PINNED_ADDRESSES { + anyhow::bail!( + "mirror upstream host must resolve to 1..={MAX_PINNED_ADDRESSES} addresses" + ); + } for &ip in addresses { - if !address_is_permitted(ip, allow_private) { + if !address_is_permitted(ip) { anyhow::bail!("mirror upstream host {host:?} resolved to disallowed address {ip}"); } } @@ -120,57 +108,21 @@ fn reject_always_local_hostname(host: &str) -> Result<()> { || host == "localhost" || host.ends_with(".localhost") || host.ends_with(".local") + || host.ends_with(".internal") { anyhow::bail!("mirror upstream host is local-only and cannot be fetched"); } Ok(()) } -fn normalize_allowlisted_host(raw: &str) -> Result { - let trimmed = raw.trim(); - if trimmed.is_empty() - || trimmed.contains(|c: char| c.is_whitespace() || c.is_control()) - || trimmed.contains(['/', '?', '#', '@', '*']) - { - anyhow::bail!("invalid private mirror host allowlist entry {raw:?}"); - } - let unbracketed = trimmed - .strip_prefix('[') - .and_then(|value| value.strip_suffix(']')) - .unwrap_or(trimmed); - let normalized = normalize_policy_host(unbracketed); - reject_always_local_hostname(&normalized)?; - - if normalized.parse::().is_err() - && !normalized - .chars() - .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '-' | '_')) - { - anyhow::bail!("invalid private mirror host allowlist entry {raw:?}"); - } - Ok(normalized) -} - -fn address_is_permitted(ip: IpAddr, allow_private: bool) -> bool { +fn address_is_permitted(ip: IpAddr) -> bool { // Broadcast/multicast/reserved destinations are never forge endpoints. match ip { IpAddr::V4(v4) if v4.octets()[0] >= 224 => return false, IpAddr::V6(v6) if v6.is_multicast() => return false, _ => {} } - if crate::api::peers::is_public_ip(ip) { - return true; - } - if !allow_private { - return false; - } - // An exact operator allowlist may admit only normal private address space - // used by an on-prem forge. Loopback, link-local, CGNAT, unspecified, and - // transition encodings remain denied because neither arm includes them. - match ip { - IpAddr::V4(v4) => v4.is_private(), - IpAddr::V6(v6) => (v6.segments()[0] & 0xfe00) == 0xfc00, - } + crate::api::peers::is_public_ip(ip) } fn parse_git_version(output: &str) -> Option<(u64, u64)> { @@ -204,6 +156,9 @@ fn ensure_safe_git_runtime(git_bin: &str) -> Result<()> { } fn fetch_args(upstream: &PinnedUpstream) -> Vec { + let http_scope = |name: &str, value: &str| format!("http.{}.{name}={value}", upstream.url); + let credential_scope = + |name: &str, value: &str| format!("credential.{}.{name}={value}", upstream.url); let mut args = vec![ "-c".to_string(), "http.proxy=".to_string(), @@ -224,18 +179,55 @@ fn fetch_args(upstream: &PinnedUpstream) -> Vec { "http.saveCookies=false".to_string(), "-c".to_string(), "http.sslVerify=true".to_string(), + // URL-specific configuration outranks generic `http.*` values even + // when the generic value came from `-c`. Repeat every security control + // at the exact validated URL so a repository-local scoped setting + // cannot re-enable redirects/proxies/headers/cookies or disable TLS. + "-c".to_string(), + http_scope("proxy", ""), + "-c".to_string(), + http_scope("followRedirects", "false"), + "-c".to_string(), + http_scope("curloptResolve", ""), + "-c".to_string(), + http_scope("extraHeader", ""), + "-c".to_string(), + http_scope("cookieFile", ""), + "-c".to_string(), + http_scope("saveCookies", "false"), + "-c".to_string(), + http_scope("sslVerify", "true"), + "-c".to_string(), + http_scope("sslCert", ""), + "-c".to_string(), + http_scope("sslKey", ""), ]; if let Some(resolve) = &upstream.curlopt_resolve { - args.extend(["-c".to_string(), format!("http.curloptResolve={resolve}")]); + args.extend([ + "-c".to_string(), + format!("http.curloptResolve={resolve}"), + "-c".to_string(), + http_scope("curloptResolve", resolve), + ]); } args.extend([ "-c".to_string(), "credential.helper=".to_string(), "-c".to_string(), + credential_scope("helper", ""), + "-c".to_string(), + "credential.interactive=false".to_string(), + "-c".to_string(), "core.askPass=false".to_string(), "-c".to_string(), "fetch.recurseSubmodules=false".to_string(), "-c".to_string(), + // Do not let a smart server or repository-local config introduce a + // second, unvalidated pack/bundle URL outside the pinned upstream. + "fetch.uriprotocols=".to_string(), + "-c".to_string(), + "fetch.bundleURI=".to_string(), + "-c".to_string(), "protocol.allow=never".to_string(), "-c".to_string(), "protocol.https.allow=always".to_string(), @@ -254,6 +246,32 @@ fn fetch_args(upstream: &PinnedUpstream) -> Vec { args } +fn reject_local_url_rewrite( + git_bin: &str, + repo_path: &Path, + upstream_url: &str, + deadline: Instant, +) -> Result<()> { + let args = ["ls-remote", "--get-url", upstream_url]; + let (status, stdout, stderr) = crate::git::visibility_pack::run_bounded_git_raw_isolated_https( + git_bin, &args, repo_path, b"", deadline, + )?; + if !status.success() { + let stderr = String::from_utf8_lossy(&stderr); + anyhow::bail!( + "checking mirror upstream URL rewrite failed: {}", + stderr.chars().take(4_096).collect::() + ); + } + let expanded = std::str::from_utf8(&stdout) + .context("Git returned a non-UTF-8 expanded mirror upstream URL")? + .trim_end_matches(['\r', '\n']); + if expanded != upstream_url { + anyhow::bail!("repository-local Git config attempted to rewrite the mirror upstream URL"); + } + Ok(()) +} + fn run_fetch( git_bin: &str, repo_path: &Path, @@ -265,9 +283,11 @@ fn run_fetch( let deadline = Instant::now() .checked_add(timeout) .context("upstream mirror fetch timeout is not representable")?; - let (status, _stdout, stderr) = crate::git::visibility_pack::run_bounded_git_raw( - git_bin, &borrowed, repo_path, b"", deadline, - )?; + reject_local_url_rewrite(git_bin, repo_path, &upstream.url, deadline)?; + let (status, _stdout, stderr) = + crate::git::visibility_pack::run_bounded_git_raw_isolated_https( + git_bin, &borrowed, repo_path, b"", deadline, + )?; if !status.success() { let stderr = String::from_utf8_lossy(&stderr); let stderr = stderr.chars().take(4_096).collect::(); @@ -287,8 +307,14 @@ struct Worker { git_bin: String, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum FetchOutcome { + Updated, + SkippedStale, +} + impl Worker { - async fn fetch_target(&self, target: InboundMirrorTarget) -> Result<()> { + async fn fetch_target(&self, target: InboundMirrorTarget) -> Result { let url = crate::db::validate_mirror_upstream_url(&target.upstream_url)?; let upstream = self.policy.resolve(url).await?; @@ -342,7 +368,7 @@ impl Worker { { guard.release(false).await; info!(repo_id = %target.repo_id, "upstream mirror state changed before fetch; skipping stale target"); - return Ok(()); + return Ok(FetchOutcome::SkippedStale); } let git_bin = self.git_bin.clone(); @@ -354,7 +380,7 @@ impl Worker { .context("upstream mirror fetch task panicked")?; let success = fetch.is_ok(); guard.release(success).await; - fetch + fetch.map(|()| FetchOutcome::Updated) } async fn scan_once(&self) { @@ -383,9 +409,10 @@ impl Worker { let repo_id = target.repo_id.clone(); let upstream_url = target.upstream_url.clone(); match self.fetch_target(target).await { - Ok(()) => { + Ok(FetchOutcome::Updated) => { info!(repo_id, upstream = %upstream_url, "upstream mirror fetch completed") } + Ok(FetchOutcome::SkippedStale) => {} Err(error) => { warn!(repo_id, upstream = %upstream_url, err = %error, "upstream mirror fetch failed") } @@ -411,7 +438,7 @@ pub fn start( mut shutdown_rx: tokio::sync::watch::Receiver, ) -> Result<()> { ensure_safe_git_runtime("git")?; - let policy = EgressPolicy::new(&config.upstream_mirror_allowed_private_hosts)?; + let policy = EgressPolicy; let worker = Worker { db, config: Arc::clone(&config), @@ -445,6 +472,104 @@ pub fn start( #[cfg(test)] mod tests { use super::*; + use clap::Parser; + use std::path::PathBuf; + + async fn test_worker( + pool: sqlx::PgPool, + git_bin: PathBuf, + page_size: usize, + ) -> (Worker, Arc, tempfile::TempDir) { + let db = Arc::new(Db::for_testing(pool.clone())); + db.run_migrations().await.unwrap(); + let repos = tempfile::tempdir().unwrap(); + let config = Arc::new(Config::parse_from([ + "gitlawb-node".to_string(), + "--upstream-mirror-enabled".to_string(), + "--enforce-owner-push".to_string(), + "--upstream-mirror-fetch-timeout-secs".to_string(), + "5".to_string(), + "--upstream-mirror-page-size".to_string(), + page_size.to_string(), + ])); + config.validate().unwrap(); + let worker = Worker { + db: Arc::clone(&db), + config, + repo_store: RepoStore::for_testing(repos.path().to_path_buf(), pool), + repo_write_leases: RepoWriteLeases::new(8), + git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(4)), + policy: EgressPolicy, + git_bin: git_bin.to_string_lossy().into_owned(), + }; + (worker, db, repos) + } + + async fn add_inbound_mirror( + worker: &Worker, + db: &Db, + id: &str, + name: &str, + upstream_url: &str, + ) -> InboundMirrorTarget { + let now = chrono::Utc::now(); + let record = crate::db::RepoRecord { + id: id.to_string(), + name: name.to_string(), + owner_did: "did:key:z6MkMirrorWorker".to_string(), + description: None, + is_public: true, + default_branch: "main".to_string(), + created_at: now, + updated_at: now, + disk_path: format!("/test/{id}.git"), + forked_from: None, + machine_id: None, + }; + db.create_repo(&record).await.unwrap(); + worker + .repo_store + .init(&record.owner_did, &record.name) + .await + .unwrap(); + let state = db + .configure_inbound_mirror(&record.id, upstream_url) + .await + .unwrap(); + InboundMirrorTarget { + repo_id: record.id, + owner_did: record.owner_did, + name: record.name, + upstream_url: state.upstream_url, + } + } + + #[cfg(unix)] + fn fake_git(repos: &tempfile::TempDir) -> (PathBuf, PathBuf) { + use std::os::unix::fs::PermissionsExt; + + let git = repos.path().join("fake-git"); + let calls = repos.path().join("fetch-calls"); + let script = format!( + r#"#!/bin/sh +if [ "$1" = "ls-remote" ]; then + printf '%s\n' "$3" + exit 0 +fi +case "$*" in + *fail.git*) printf '%s\n' fail >> '{}'; exit 7 ;; + *) printf '%s\n' ok >> '{}'; exit 0 ;; +esac +"#, + calls.display(), + calls.display() + ); + std::fs::write(&git, script).unwrap(); + let mut permissions = std::fs::metadata(&git).unwrap().permissions(); + permissions.set_mode(0o700); + std::fs::set_permissions(&git, permissions).unwrap(); + (git, calls) + } #[test] fn git_version_gate_requires_curlopt_resolve_support() { @@ -458,36 +583,48 @@ mod tests { } #[test] - fn private_targets_require_an_exact_operator_allowlist() { - let policy = EgressPolicy::new(&["ghe.internal".to_string()]).unwrap(); + fn every_resolved_address_must_be_public() { + let policy = EgressPolicy; let public_v4: IpAddr = "8.8.8.8".parse().unwrap(); let private_v4: IpAddr = "10.2.3.4".parse().unwrap(); let private_v6: IpAddr = "fd00::20".parse().unwrap(); assert!(policy - .validate_addresses("ghe.internal", &[private_v4, private_v6], true) + .validate_addresses("public.example", &[public_v4]) .is_ok()); assert!(policy - .validate_addresses("other.internal", &[private_v4], false) + .validate_addresses("private.example", &[private_v4, private_v6]) + .is_err()); + assert!(policy + .validate_addresses("mixed.example", &[public_v4, private_v4]) .is_err()); + let too_many = (1..=MAX_PINNED_ADDRESSES + 1) + .map(|last| IpAddr::V4(std::net::Ipv4Addr::new(8, 8, 0, last as u8))) + .collect::>(); assert!(policy - .validate_addresses("public.example", &[public_v4, private_v4], false) + .validate_addresses("oversized.example", &too_many) .is_err()); for never in ["127.0.0.1", "169.254.169.254", "::1", "fe80::1"] { let ip = never.parse().unwrap(); - assert!(!address_is_permitted(ip, true), "{never} must stay denied"); + assert!(!address_is_permitted(ip), "{never} must stay denied"); } } #[test] - fn private_host_allowlist_rejects_wildcards_and_localhost() { - for invalid in ["", "*.internal", "localhost", "forge.local", "host/path"] { - assert!(normalize_allowlisted_host(invalid).is_err(), "{invalid:?}"); + fn local_only_hostnames_are_always_rejected() { + for invalid in [ + "", + "localhost", + "forge.localhost", + "forge.local", + "ghe.internal", + ] { + assert!( + reject_always_local_hostname(invalid).is_err(), + "{invalid:?}" + ); } - assert_eq!( - normalize_allowlisted_host("GHE.INTERNAL.").unwrap(), - "ghe.internal" - ); + assert!(reject_always_local_hostname("github.example").is_ok()); } #[test] @@ -506,26 +643,135 @@ mod tests { assert!(args.contains( &"http.curloptResolve=+github.example:443:203.0.113.10,[2001:db8::10]".to_string() )); + assert!(args.contains( + &"http.https://github.example/org/repo.git.followRedirects=false".to_string() + )); + assert!(args.contains(&"http.https://github.example/org/repo.git.proxy=".to_string())); + assert!(args.contains(&"http.https://github.example/org/repo.git.extraHeader=".to_string())); + assert!(args.contains( + &"http.https://github.example/org/repo.git.curloptResolve=+github.example:443:203.0.113.10,[2001:db8::10]".to_string() + )); + assert!( + args.contains(&"credential.https://github.example/org/repo.git.helper=".to_string()) + ); assert!(args.contains(&"protocol.allow=never".to_string())); assert!(args.contains(&"protocol.https.allow=always".to_string())); + assert!(args.contains(&"fetch.uriprotocols=".to_string())); + assert!(args.contains(&"fetch.bundleURI=".to_string())); assert!(args.contains(&"--atomic".to_string())); assert!(args.contains(&"+refs/heads/*:refs/heads/*".to_string())); assert!(args.contains(&"+refs/tags/*:refs/tags/*".to_string())); assert!(!args.iter().any(|arg| arg == "+refs/*:refs/*")); } + #[test] + fn repository_local_instead_of_rewrite_is_rejected() { + let repo = tempfile::tempdir().unwrap(); + let init = std::process::Command::new("git") + .args(["init", "--bare"]) + .current_dir(repo.path()) + .output() + .unwrap(); + assert!(init.status.success()); + let config = std::process::Command::new("git") + .args([ + "config", + "url.https://evil.example/.insteadOf", + "https://github.example/", + ]) + .current_dir(repo.path()) + .output() + .unwrap(); + assert!(config.status.success()); + + let error = reject_local_url_rewrite( + "git", + repo.path(), + "https://github.example/org/repo.git", + Instant::now() + Duration::from_secs(5), + ) + .unwrap_err(); + assert!(error.to_string().contains("attempted to rewrite")); + } + #[tokio::test] async fn literal_addresses_are_checked_without_dns() { - let policy = EgressPolicy::new(&["10.2.3.4".to_string()]).unwrap(); + let policy = EgressPolicy; let allowed = policy - .resolve(reqwest::Url::parse("https://10.2.3.4/org/repo.git").unwrap()) + .resolve(reqwest::Url::parse("https://203.0.113.10/org/repo.git").unwrap()) .await .unwrap(); assert_eq!(allowed.curlopt_resolve, None); - assert!(policy - .resolve(reqwest::Url::parse("https://127.0.0.1/org/repo.git").unwrap()) - .await - .is_err()); + for denied in [ + "127.0.0.1", + "10.2.3.4", + "169.254.169.254", + "[::1]", + "[fd00::20]", + ] { + let url = reqwest::Url::parse(&format!("https://{denied}/org/repo.git")).unwrap(); + assert!( + policy.resolve(url).await.is_err(), + "{denied} must stay denied" + ); + } + } + + #[sqlx::test] + async fn stale_target_is_rechecked_under_the_write_locks(pool: sqlx::PgPool) { + let missing_git = PathBuf::from("/definitely/missing/git"); + let (worker, db, _repos) = test_worker(pool, missing_git, 1).await; + let target = add_inbound_mirror( + &worker, + &db, + "00000000-0000-0000-0000-000000000001", + "stale", + "https://8.8.8.8/org/stale.git", + ) + .await; + sqlx::query( + "UPDATE repos + SET mirror_status = 'outbound' + WHERE id = $1", + ) + .bind(&target.repo_id) + .execute(db.pool()) + .await + .unwrap(); + + let outcome = worker.fetch_target(target).await.unwrap(); + assert_eq!(outcome, FetchOutcome::SkippedStale); + assert!(worker.repo_write_leases.is_empty()); + } + + #[cfg(unix)] + #[sqlx::test] + async fn scan_pages_past_one_failure_and_fetches_the_next_repo(pool: sqlx::PgPool) { + let bootstrap = tempfile::tempdir().unwrap(); + let (git, calls) = fake_git(&bootstrap); + let (worker, db, _repos) = test_worker(pool, git, 1).await; + add_inbound_mirror( + &worker, + &db, + "00000000-0000-0000-0000-000000000001", + "fails", + "https://8.8.8.8/org/fail.git", + ) + .await; + add_inbound_mirror( + &worker, + &db, + "00000000-0000-0000-0000-000000000002", + "succeeds", + "https://8.8.8.8/org/ok.git", + ) + .await; + + worker.scan_once().await; + + let calls = std::fs::read_to_string(calls).unwrap(); + assert_eq!(calls.lines().collect::>(), ["fail", "ok"]); + assert!(worker.repo_write_leases.is_empty()); } } From d34aac8c2adb80f1cb57bf2a8415fc83d3fa1282 Mon Sep 17 00:00:00 2001 From: KRATOS <84986124+gnanam1990@users.noreply.github.com> Date: Sat, 15 Aug 2026 16:14:59 +0530 Subject: [PATCH 9/9] fix(node): close inbound mirror review blockers Origin-Session: local-d6a143 | Codex | 24 prompts --- crates/gitlawb-node/src/api/peers.rs | 2 +- .../gitlawb-node/src/git/visibility_pack.rs | 41 +- crates/gitlawb-node/src/upstream_mirror.rs | 388 +++++++++++++----- 3 files changed, 321 insertions(+), 110 deletions(-) diff --git a/crates/gitlawb-node/src/api/peers.rs b/crates/gitlawb-node/src/api/peers.rs index 5966ea52..4c7bdd42 100644 --- a/crates/gitlawb-node/src/api/peers.rs +++ b/crates/gitlawb-node/src/api/peers.rs @@ -84,7 +84,7 @@ pub(crate) const PUBLIC_HTTP_URL_REQUIREMENT: &str = "must be a public http(s) U /// sits at a prefix-length-dependent offset (RFC 6052 §2.2) — so they return /// `None`. Any caller that needs them blocked must reject the wider /// `64:ff9b::/32` itself; `is_public_http_url` does this in its native-v6 arm. -fn embedded_ipv4(v6: std::net::Ipv6Addr) -> Option { +pub(crate) fn embedded_ipv4(v6: std::net::Ipv6Addr) -> Option { if let Some(v4) = v6.to_ipv4_mapped().or_else(|| v6.to_ipv4()) { return Some(v4); } diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index fb2fa18e..8b84acad 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -38,6 +38,7 @@ fn git_child_command( args: &[&str], repo_path: &Path, isolated_https: bool, + isolated_config: &[(&str, &str)], ) -> std::process::Command { let mut command = std::process::Command::new(git_bin); command @@ -63,6 +64,14 @@ fn git_child_command( .env("SSH_ASKPASS", "false") .env("LANG", "C") .env("LC_ALL", "C"); + if !isolated_config.is_empty() { + command.env("GIT_CONFIG_COUNT", isolated_config.len().to_string()); + for (index, (key, value)) in isolated_config.iter().enumerate() { + command + .env(format!("GIT_CONFIG_KEY_{index}"), key) + .env(format!("GIT_CONFIG_VALUE_{index}"), value); + } + } } command } @@ -112,13 +121,14 @@ fn run_bounded_git_raw_impl( stdin_bytes: &[u8], deadline: Instant, isolated_https: bool, + isolated_config: &[(&str, &str)], ) -> Result<(std::process::ExitStatus, Vec, Vec)> { use std::io::{Read, Write}; use std::os::unix::process::CommandExt; use std::sync::mpsc::RecvTimeoutError; let label = args.first().copied().unwrap_or("git"); - let mut command = git_child_command(git_bin, args, repo_path, isolated_https); + let mut command = git_child_command(git_bin, args, repo_path, isolated_https, isolated_config); let mut child = command .process_group(0) .spawn() @@ -236,7 +246,7 @@ pub(crate) fn run_bounded_git_raw( stdin_bytes: &[u8], deadline: Instant, ) -> Result<(std::process::ExitStatus, Vec, Vec)> { - run_bounded_git_raw_impl(git_bin, args, repo_path, stdin_bytes, deadline, false) + run_bounded_git_raw_impl(git_bin, args, repo_path, stdin_bytes, deadline, false, &[]) } /// Run a bounded Git child without inheriting ambient Git/proxy/credential @@ -249,7 +259,29 @@ pub(crate) fn run_bounded_git_raw_isolated_https( stdin_bytes: &[u8], deadline: Instant, ) -> Result<(std::process::ExitStatus, Vec, Vec)> { - run_bounded_git_raw_impl(git_bin, args, repo_path, stdin_bytes, deadline, true) + run_bounded_git_raw_impl(git_bin, args, repo_path, stdin_bytes, deadline, true, &[]) +} + +/// Isolated HTTPS runner with command-scope Git configuration supplied as +/// separate key/value environment entries. Unlike `git -c key=value`, this +/// preserves valid URL subsection keys whose path contains `=`. +pub(crate) fn run_bounded_git_raw_isolated_https_with_config( + git_bin: &str, + args: &[&str], + repo_path: &Path, + stdin_bytes: &[u8], + deadline: Instant, + config: &[(&str, &str)], +) -> Result<(std::process::ExitStatus, Vec, Vec)> { + run_bounded_git_raw_impl( + git_bin, + args, + repo_path, + stdin_bytes, + deadline, + true, + config, + ) } /// Bounded git returning only stdout, `bail!`ing on any nonzero exit. The thin @@ -292,12 +324,13 @@ fn run_bounded_git_raw_impl( stdin_bytes: &[u8], deadline: Instant, isolated_https: bool, + isolated_config: &[(&str, &str)], ) -> Result<(std::process::ExitStatus, Vec, Vec)> { use std::io::{Read, Write}; use std::sync::mpsc::RecvTimeoutError; let label = args.first().copied().unwrap_or("git"); - let mut child = git_child_command(git_bin, args, repo_path, isolated_https) + let mut child = git_child_command(git_bin, args, repo_path, isolated_https, isolated_config) .spawn() .with_context(|| format!("failed to spawn git {label}"))?; diff --git a/crates/gitlawb-node/src/upstream_mirror.rs b/crates/gitlawb-node/src/upstream_mirror.rs index 4d57c9d2..0441ebd7 100644 --- a/crates/gitlawb-node/src/upstream_mirror.rs +++ b/crates/gitlawb-node/src/upstream_mirror.rs @@ -116,13 +116,47 @@ fn reject_always_local_hostname(host: &str) -> Result<()> { } fn address_is_permitted(ip: IpAddr) -> bool { - // Broadcast/multicast/reserved destinations are never forge endpoints. - match ip { - IpAddr::V4(v4) if v4.octets()[0] >= 224 => return false, - IpAddr::V6(v6) if v6.is_multicast() => return false, - _ => {} + if !crate::api::peers::is_public_ip(ip) { + return false; + } + let classified = match ip { + IpAddr::V6(v6) => crate::api::peers::embedded_ipv4(v6) + .map(IpAddr::V4) + .unwrap_or(ip), + _ => ip, + }; + match classified { + IpAddr::V4(v4) => { + let [a, b, c, _] = v4.octets(); + // IANA special-purpose ranges that are not globally reachable but + // may be routed inside labs, clouds, or operator networks. They are + // unsuitable for an owner-controlled outbound fetch even though + // `Ipv4Addr::is_private` does not classify them as RFC1918. + !(a >= 224 + || (a == 192 && b == 0 && c == 0) + || (a == 192 && b == 0 && c == 2) + || (a == 192 && b == 88 && c == 99) + || (a == 198 && matches!(b, 18 | 19)) + || (a == 198 && b == 51 && c == 100) + || (a == 203 && b == 0 && c == 113)) + } + IpAddr::V6(v6) => { + let s = v6.segments(); + // IANA special-purpose ranges that are non-global, locally + // meaningful, or unsafe to treat as ordinary forge endpoints. + // Reject the complete IETF protocol-assignment /23 rather than + // trying to maintain its narrow globally reachable exceptions; + // a public forge has no reason to depend on those anycast ranges. + !(v6.is_multicast() + || (s[0] & 0xffc0) == 0xfec0 + || (s[0] == 0x0100 && s[1] == 0 && s[2] == 0 && s[3] == 0) + || (s[0] == 0x0100 && s[1] == 0 && s[2] == 0 && s[3] == 1) + || (s[0] == 0x2001 && (s[1] & 0xfe00) == 0) + || (s[0] == 0x2001 && s[1] == 0x0db8) + || (s[0] == 0x3fff && (s[1] & 0xf000) == 0) + || s[0] == 0x5f00) + } } - crate::api::peers::is_public_ip(ip) } fn parse_git_version(output: &str) -> Option<(u64, u64)> { @@ -155,82 +189,62 @@ fn ensure_safe_git_runtime(git_bin: &str) -> Result<()> { Ok(()) } -fn fetch_args(upstream: &PinnedUpstream) -> Vec { - let http_scope = |name: &str, value: &str| format!("http.{}.{name}={value}", upstream.url); - let credential_scope = - |name: &str, value: &str| format!("credential.{}.{name}={value}", upstream.url); - let mut args = vec![ - "-c".to_string(), - "http.proxy=".to_string(), - "-c".to_string(), - "http.followRedirects=false".to_string(), +#[derive(Debug, Clone, PartialEq, Eq)] +struct FetchCommand { + config: Vec<(String, String)>, + args: Vec, +} + +fn fetch_command(upstream: &PinnedUpstream) -> FetchCommand { + let http_scope = |name: &str| format!("http.{}.{name}", upstream.url); + let credential_scope = |name: &str| format!("credential.{}.{name}", upstream.url); + let mut config = vec![ + ("http.proxy".to_string(), "".to_string()), + ("http.followRedirects".to_string(), "false".to_string()), // Reset any inherited multi-valued resolver entries before adding the // address set validated for this exact fetch. - "-c".to_string(), - "http.curloptResolve=".to_string(), + ("http.curloptResolve".to_string(), "".to_string()), // Never forward operator or repository-scoped HTTP credentials to an // owner-selected forge. Empty values reset Git's multi-valued headers // and disable cookie state for this invocation. - "-c".to_string(), - "http.extraHeader=".to_string(), - "-c".to_string(), - "http.cookieFile=".to_string(), - "-c".to_string(), - "http.saveCookies=false".to_string(), - "-c".to_string(), - "http.sslVerify=true".to_string(), + ("http.extraHeader".to_string(), "".to_string()), + ("http.cookieFile".to_string(), "".to_string()), + ("http.saveCookies".to_string(), "false".to_string()), + ("http.sslVerify".to_string(), "true".to_string()), // URL-specific configuration outranks generic `http.*` values even - // when the generic value came from `-c`. Repeat every security control - // at the exact validated URL so a repository-local scoped setting - // cannot re-enable redirects/proxies/headers/cookies or disable TLS. - "-c".to_string(), - http_scope("proxy", ""), - "-c".to_string(), - http_scope("followRedirects", "false"), - "-c".to_string(), - http_scope("curloptResolve", ""), - "-c".to_string(), - http_scope("extraHeader", ""), - "-c".to_string(), - http_scope("cookieFile", ""), - "-c".to_string(), - http_scope("saveCookies", "false"), - "-c".to_string(), - http_scope("sslVerify", "true"), - "-c".to_string(), - http_scope("sslCert", ""), - "-c".to_string(), - http_scope("sslKey", ""), + // when the generic value is command-scoped. Repeat every security + // control at the exact validated URL so a repository-local scoped + // setting cannot re-enable redirects/proxies/headers/cookies or disable + // TLS. Key/value environment entries avoid `git -c` splitting a valid + // URL path that contains `=`. + (http_scope("proxy"), "".to_string()), + (http_scope("followRedirects"), "false".to_string()), + (http_scope("curloptResolve"), "".to_string()), + (http_scope("extraHeader"), "".to_string()), + (http_scope("cookieFile"), "".to_string()), + (http_scope("saveCookies"), "false".to_string()), + (http_scope("sslVerify"), "true".to_string()), ]; if let Some(resolve) = &upstream.curlopt_resolve { - args.extend([ - "-c".to_string(), - format!("http.curloptResolve={resolve}"), - "-c".to_string(), - http_scope("curloptResolve", resolve), + config.extend([ + ("http.curloptResolve".to_string(), resolve.clone()), + (http_scope("curloptResolve"), resolve.clone()), ]); } - args.extend([ - "-c".to_string(), - "credential.helper=".to_string(), - "-c".to_string(), - credential_scope("helper", ""), - "-c".to_string(), - "credential.interactive=false".to_string(), - "-c".to_string(), - "core.askPass=false".to_string(), - "-c".to_string(), - "fetch.recurseSubmodules=false".to_string(), - "-c".to_string(), + config.extend([ + ("credential.helper".to_string(), "".to_string()), + (credential_scope("helper"), "".to_string()), + ("credential.interactive".to_string(), "false".to_string()), + ("core.askPass".to_string(), "false".to_string()), + ("fetch.recurseSubmodules".to_string(), "false".to_string()), // Do not let a smart server or repository-local config introduce a // second, unvalidated pack/bundle URL outside the pinned upstream. - "fetch.uriprotocols=".to_string(), - "-c".to_string(), - "fetch.bundleURI=".to_string(), - "-c".to_string(), - "protocol.allow=never".to_string(), - "-c".to_string(), - "protocol.https.allow=always".to_string(), + ("fetch.uriprotocols".to_string(), "".to_string()), + ("fetch.bundleURI".to_string(), "".to_string()), + ("protocol.allow".to_string(), "never".to_string()), + ("protocol.https.allow".to_string(), "always".to_string()), + ]); + let args = vec![ "fetch".to_string(), "--atomic".to_string(), "--force".to_string(), @@ -242,8 +256,8 @@ fn fetch_args(upstream: &PinnedUpstream) -> Vec { upstream.url.clone(), "+refs/heads/*:refs/heads/*".to_string(), "+refs/tags/*:refs/tags/*".to_string(), - ]); - args + ]; + FetchCommand { config, args } } fn reject_local_url_rewrite( @@ -272,21 +286,53 @@ fn reject_local_url_rewrite( Ok(()) } +fn reject_sensitive_local_config(git_bin: &str, repo_path: &Path, deadline: Instant) -> Result<()> { + let args = [ + "config", + "--local", + "--includes", + "--get-regexp", + r"^(http|credential|url)\.", + ]; + let (status, stdout, stderr) = crate::git::visibility_pack::run_bounded_git_raw_isolated_https( + git_bin, &args, repo_path, b"", deadline, + )?; + match status.code() { + Some(1) if stdout.is_empty() => Ok(()), + Some(0) => anyhow::bail!( + "repository-local network or credential Git config is not allowed for upstream mirrors" + ), + _ => { + let stderr = String::from_utf8_lossy(&stderr); + anyhow::bail!( + "checking repository-local Git config failed: {}", + stderr.chars().take(4_096).collect::() + ) + } + } +} + fn run_fetch( git_bin: &str, repo_path: &Path, upstream: &PinnedUpstream, timeout: Duration, ) -> Result<()> { - let args = fetch_args(upstream); - let borrowed: Vec<&str> = args.iter().map(String::as_str).collect(); + let command = fetch_command(upstream); + let args: Vec<&str> = command.args.iter().map(String::as_str).collect(); + let config: Vec<(&str, &str)> = command + .config + .iter() + .map(|(key, value)| (key.as_str(), value.as_str())) + .collect(); let deadline = Instant::now() .checked_add(timeout) .context("upstream mirror fetch timeout is not representable")?; reject_local_url_rewrite(git_bin, repo_path, &upstream.url, deadline)?; + reject_sensitive_local_config(git_bin, repo_path, deadline)?; let (status, _stdout, stderr) = - crate::git::visibility_pack::run_bounded_git_raw_isolated_https( - git_bin, &borrowed, repo_path, b"", deadline, + crate::git::visibility_pack::run_bounded_git_raw_isolated_https_with_config( + git_bin, &args, repo_path, b"", deadline, &config, )?; if !status.success() { let stderr = String::from_utf8_lossy(&stderr); @@ -556,6 +602,10 @@ if [ "$1" = "ls-remote" ]; then printf '%s\n' "$3" exit 0 fi +if [ "$1" = "config" ]; then + # Match real Git when no sensitive repository-local keys are configured. + exit 1 +fi case "$*" in *fail.git*) printf '%s\n' fail >> '{}'; exit 7 ;; *) printf '%s\n' ok >> '{}'; exit 0 ;; @@ -604,7 +654,27 @@ esac .validate_addresses("oversized.example", &too_many) .is_err()); - for never in ["127.0.0.1", "169.254.169.254", "::1", "fe80::1"] { + for never in [ + "127.0.0.1", + "169.254.169.254", + "192.0.2.1", + "198.18.0.1", + "198.51.100.1", + "203.0.113.10", + "::1", + "::ffff:198.18.0.1", + "2002:c612:1::", + "64:ff9b::c612:1", + "100::1", + "100:0:0:1::1", + "2001::1", + "2001:2::1", + "2001:db8::1", + "fec0::1", + "fe80::1", + "3fff::1", + "5f00::1", + ] { let ip = never.parse().unwrap(); assert!(!address_is_permitted(ip), "{never} must stay denied"); } @@ -631,37 +701,60 @@ esac fn fetch_command_pins_dns_and_only_updates_branches_and_tags() { let upstream = PinnedUpstream { url: "https://github.example/org/repo.git".to_string(), - curlopt_resolve: Some("+github.example:443:203.0.113.10,[2001:db8::10]".to_string()), + curlopt_resolve: Some("+github.example:443:8.8.8.8,[2606:4700:4700::1111]".to_string()), + }; + let command = fetch_command(&upstream); + let has_config = |key: &str, value: &str| { + command + .config + .contains(&(key.to_string(), value.to_string())) }; - let args = fetch_args(&upstream); - assert!(args.contains(&"http.followRedirects=false".to_string())); - assert!(args.contains(&"http.proxy=".to_string())); - assert!(args.contains(&"http.extraHeader=".to_string())); - assert!(args.contains(&"http.cookieFile=".to_string())); - assert!(args.contains(&"http.saveCookies=false".to_string())); - assert!(args.contains(&"http.sslVerify=true".to_string())); - assert!(args.contains( - &"http.curloptResolve=+github.example:443:203.0.113.10,[2001:db8::10]".to_string() + assert!(has_config("http.followRedirects", "false")); + assert!(has_config("http.proxy", "")); + assert!(has_config("http.extraHeader", "")); + assert!(has_config("http.cookieFile", "")); + assert!(has_config("http.saveCookies", "false")); + assert!(has_config("http.sslVerify", "true")); + assert!(has_config( + "http.curloptResolve", + "+github.example:443:8.8.8.8,[2606:4700:4700::1111]" )); - assert!(args.contains( - &"http.https://github.example/org/repo.git.followRedirects=false".to_string() + assert!(has_config( + "http.https://github.example/org/repo.git.followRedirects", + "false" )); - assert!(args.contains(&"http.https://github.example/org/repo.git.proxy=".to_string())); - assert!(args.contains(&"http.https://github.example/org/repo.git.extraHeader=".to_string())); - assert!(args.contains( - &"http.https://github.example/org/repo.git.curloptResolve=+github.example:443:203.0.113.10,[2001:db8::10]".to_string() + assert!(has_config( + "http.https://github.example/org/repo.git.proxy", + "" )); - assert!( - args.contains(&"credential.https://github.example/org/repo.git.helper=".to_string()) - ); - assert!(args.contains(&"protocol.allow=never".to_string())); - assert!(args.contains(&"protocol.https.allow=always".to_string())); - assert!(args.contains(&"fetch.uriprotocols=".to_string())); - assert!(args.contains(&"fetch.bundleURI=".to_string())); - assert!(args.contains(&"--atomic".to_string())); - assert!(args.contains(&"+refs/heads/*:refs/heads/*".to_string())); - assert!(args.contains(&"+refs/tags/*:refs/tags/*".to_string())); - assert!(!args.iter().any(|arg| arg == "+refs/*:refs/*")); + assert!(has_config( + "http.https://github.example/org/repo.git.extraHeader", + "" + )); + assert!(has_config( + "http.https://github.example/org/repo.git.curloptResolve", + "+github.example:443:8.8.8.8,[2606:4700:4700::1111]" + )); + assert!(has_config( + "credential.https://github.example/org/repo.git.helper", + "" + )); + assert!(has_config("protocol.allow", "never")); + assert!(has_config("protocol.https.allow", "always")); + assert!(has_config("fetch.uriprotocols", "")); + assert!(has_config("fetch.bundleURI", "")); + assert!(!command + .config + .iter() + .any(|(key, _)| key.ends_with("sslCert") || key.ends_with("sslKey"))); + assert!(command.args.contains(&"--atomic".to_string())); + assert!(command + .args + .contains(&"+refs/heads/*:refs/heads/*".to_string())); + assert!(command + .args + .contains(&"+refs/tags/*:refs/tags/*".to_string())); + assert!(!command.args.iter().any(|arg| arg == "+refs/*:refs/*")); } #[test] @@ -694,11 +787,86 @@ esac assert!(error.to_string().contains("attempted to rewrite")); } + #[test] + fn repository_local_network_and_credential_config_is_rejected() { + let repo = tempfile::tempdir().unwrap(); + let init = std::process::Command::new("git") + .args(["init", "--bare"]) + .current_dir(repo.path()) + .output() + .unwrap(); + assert!(init.status.success()); + + reject_sensitive_local_config("git", repo.path(), Instant::now() + Duration::from_secs(5)) + .unwrap(); + + let config = std::process::Command::new("git") + .args(["config", "http.sslCert", "/tmp/untrusted-client.pem"]) + .current_dir(repo.path()) + .output() + .unwrap(); + assert!(config.status.success()); + + let error = reject_sensitive_local_config( + "git", + repo.path(), + Instant::now() + Duration::from_secs(5), + ) + .unwrap_err(); + assert!(error.to_string().contains("network or credential")); + } + + #[cfg(unix)] + #[test] + fn hardened_real_git_reaches_https_transport_for_equals_path() { + let repo = tempfile::tempdir().unwrap(); + let init = std::process::Command::new("git") + .args(["init", "--bare"]) + .current_dir(repo.path()) + .output() + .unwrap(); + assert!(init.status.success()); + + // This is intentionally a loopback test seam around `run_fetch`, below + // the egress-policy layer. Reaching the listener proves the generated + // Git config neither dies on an empty client certificate nor splits the + // valid `=` path at the command-config boundary. The listener closes + // immediately, so the expected final result is a TLS transport error. + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + listener.set_nonblocking(true).unwrap(); + let accepted = std::thread::spawn(move || { + let deadline = Instant::now() + Duration::from_secs(3); + while Instant::now() < deadline { + match listener.accept() { + Ok((_stream, _address)) => return true, + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + std::thread::sleep(Duration::from_millis(10)); + } + Err(error) => panic!("accepting real-Git transport probe: {error}"), + } + } + false + }); + let upstream = PinnedUpstream { + url: format!("https://127.0.0.1:{port}/org/repo=mirror.git"), + curlopt_resolve: None, + }; + let error = run_fetch("git", repo.path(), &upstream, Duration::from_secs(5)).unwrap_err(); + assert!( + accepted.join().unwrap(), + "real Git must reach the HTTPS transport; got {error:#}" + ); + let error = error.to_string(); + assert!(!error.contains("credential missing host field")); + assert!(!error.contains("invalid key")); + } + #[tokio::test] async fn literal_addresses_are_checked_without_dns() { let policy = EgressPolicy; let allowed = policy - .resolve(reqwest::Url::parse("https://203.0.113.10/org/repo.git").unwrap()) + .resolve(reqwest::Url::parse("https://8.8.8.8/org/repo.git").unwrap()) .await .unwrap(); assert_eq!(allowed.curlopt_resolve, None); @@ -707,8 +875,18 @@ esac "127.0.0.1", "10.2.3.4", "169.254.169.254", + "192.0.2.1", + "198.18.0.1", + "203.0.113.10", "[::1]", + "[100::1]", + "[100:0:0:1::1]", + "[2001::1]", + "[2001:db8::1]", + "[fec0::1]", "[fd00::20]", + "[3fff::1]", + "[5f00::1]", ] { let url = reqwest::Url::parse(&format!("https://{denied}/org/repo.git")).unwrap(); assert!(