Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -472,6 +472,12 @@ Godot project graphs), graph traversal, FTS5 search, whole-graph export with
deterministic PageRank centrality, MCP/CLI surfaces, and golden byte-stable
output.

Extraction is fail-closed per file: a source tree deeper than the deterministic
256-level logical named-node limit records a stable file error, emits no partial
graph for that file, and does not stop other files from being indexed. Incremental
`sync` removes any older graph for the failed file and converges with
`index --force`.

**Doesn't:** no AI / vector / embedding / LLM anywhere inside the binary (hard
constraint, `scripts/guardrail.sh`-enforced); no semantic search; no languages
beyond the fixed `LANGUAGES` set.
Expand Down
91 changes: 91 additions & 0 deletions crates/codegraph-cli/tests/parallel_index.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,10 @@
//! `max_file_size` yields the exact size-skip `FileRecord` through the
//! parallel parse path (T2): size set to the real byte length, node_count 0,
//! and the exact `"File exceeds max size (N > M): path"` error string.
//! * `deep_file_sync_discards_stale_graph_and_matches_force_index` — replacing
//! a previously valid file with a 30,000-level AST removes its old graph,
//! records one stable file error, leaves sibling files intact, and converges
//! to the same canonical DB as a fresh `index --force`.
//! * `full_index_leaves_synchronous_at_normal` — after a full `index --force`
//! (which runs under `synchronous=OFF` per T5), a FRESH `Store::open` reports
//! `PRAGMA synchronous != 0` (NORMAL), proving the bulk-index pragma never
Expand Down Expand Up @@ -262,6 +266,93 @@ fn oversized_file_size_skips_with_exact_file_record() {
);
}

/// DEPTH-ABORT PARITY: a file that crosses the deterministic AST traversal
/// limit contributes no partial graph. Incremental sync must first remove the
/// file's previous graph, persist the stable error, and converge to the same DB
/// as a clean full rebuild of the final filesystem state.
#[test]
fn deep_file_sync_discards_stale_graph_and_matches_force_index() {
const DEPTH: usize = 30_000;
const REL_PATH: &str = "deep.c";
const EXPECTED_ERROR: &str = "AST nesting exceeds safe traversal limit (256): deep.c";

let deep_source = format!(
"void before(void) {{}}\nvoid deep(void) {{{}{}}}\nvoid after(void) {{}}\n",
"{".repeat(DEPTH),
"}".repeat(DEPTH)
);
let normal_source = "int healthy_target(void) { return 7; }\n";

let synced = TestDir::new("deep-sync");
let synced_project = synced.path().join("project");
fs::create_dir_all(&synced_project).unwrap();
fs::write(
synced_project.join(REL_PATH),
"int stale_target(void) { return 1; }\n",
)
.unwrap();
fs::write(synced_project.join("ok.c"), normal_source).unwrap();
init_and_force_index(&synced_project);

fs::write(synced_project.join(REL_PATH), &deep_source).unwrap();
let (out, err, ok) = cli(&["sync", synced_project.to_str().unwrap()]);
assert!(ok, "sync failed: stdout={out} stderr={err}");

let synced_store = Store::open(&db_path(&synced_project)).unwrap();
let deep_node_count: i64 = synced_store
.connection()
.query_row(
"SELECT COUNT(*) FROM nodes WHERE file_path = ?1",
[REL_PATH],
|row| row.get(0),
)
.unwrap();
assert_eq!(
deep_node_count, 0,
"the failed file's previous and partial nodes must be removed"
);

let healthy_count: i64 = synced_store
.connection()
.query_row(
"SELECT COUNT(*) FROM nodes WHERE file_path = 'ok.c' AND name = 'healthy_target'",
[],
|row| row.get(0),
)
.unwrap();
assert_eq!(
healthy_count, 1,
"one invalid file must not remove a healthy sibling's graph"
);

let (node_count, errors_json): (i64, Option<String>) = synced_store
.connection()
.query_row(
"SELECT node_count, errors FROM files WHERE path = ?1",
[REL_PATH],
|row| Ok((row.get(0)?, row.get(1)?)),
)
.expect("the failed file must retain a files row");
assert_eq!(node_count, 0);
let errors: Vec<String> =
serde_json::from_str(&errors_json.expect("the failed file must persist its depth error"))
.unwrap();
assert_eq!(errors, vec![EXPECTED_ERROR]);
drop(synced_store);

let rebuilt = TestDir::new("deep-force");
let rebuilt_project = rebuilt.path().join("project");
fs::create_dir_all(&rebuilt_project).unwrap();
fs::write(rebuilt_project.join(REL_PATH), &deep_source).unwrap();
fs::write(rebuilt_project.join("ok.c"), normal_source).unwrap();
init_and_force_index(&rebuilt_project);

let after_sync = canonicalize_db(&db_path(&synced_project)).unwrap();
let after_force = canonicalize_db(&db_path(&rebuilt_project)).unwrap();
diff_canonical(&after_sync, &after_force, None)
.expect("sync and index --force must converge after a depth-aborted file");
}

/// WRITE PATH 1 (`index --force`, the rayon producer closure): the persisted
/// `files.generated` must carry the real per-file verdict, not a hardcoded false.
/// This is the CLI counterpart to the two `codegraph-watch` tests that cover the
Expand Down
109 changes: 109 additions & 0 deletions crates/codegraph-extract/src/walker.rs
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,57 @@ fn is_plain_receiver_segment(segment: &str) -> bool {
.all(|c| c.is_alphanumeric() || c == '_' || c == '$')
}

/// Deterministic logical recursion bound for the native AST walker.
///
/// Tree-sitter parses deeply nested input iteratively, but this walker descends
/// recursively. Bounding named-node depth keeps one adversarial file from
/// overflowing the process stack. The value is deliberately well above normal
/// source nesting while remaining safe on a 1 MiB worker stack.
const MAX_WALK_DEPTH: usize = 256;

fn is_transparent_walk_wrapper(language: Language, node: SyntaxNode<'_>) -> bool {
language == Language::Rust
&& node.kind() == "expression_statement"
&& node.named_child_count() == 1
}

fn unwrap_transparent_walk_wrappers(
language: Language,
mut node: SyntaxNode<'_>,
) -> SyntaxNode<'_> {
while is_transparent_walk_wrapper(language, node) {
let Some(child) = node.named_child(0) else {
break;
};
node = child;
}
node
}

fn ast_exceeds_safe_depth(root: SyntaxNode<'_>, language: Language) -> bool {
let mut pending = vec![(root, 1usize)];

while let Some((node, depth)) = pending.pop() {
if depth > MAX_WALK_DEPTH {
return true;
}

for index in 0..node.named_child_count() {
if let Some(child) = node.named_child(index as u32) {
// tree-sitter-rust inserts an inert `expression_statement`
// between every nested block. It carries no extraction
// semantics, so count the source-level block rather than
// double-charging this grammar wrapper.
let child_depth =
depth + usize::from(!is_transparent_walk_wrapper(language, child));
pending.push((child, child_depth));
}
}
}

false
}

pub struct TreeSitterWalker<'a, 'tree> {
file_path: &'a str,
source: &'a str,
Expand All @@ -170,6 +221,11 @@ pub struct TreeSitterWalker<'a, 'tree> {
/// component body) have already been consumed, so the top-level `program`
/// walk skips them instead of re-visiting. 0 outside CFML tag files.
cfml_consumed_until: usize,
/// Current named-node recursion depth. Root is depth 1.
walk_depth: usize,
/// Latched once a child would exceed [`MAX_WALK_DEPTH`]. All remaining
/// descent stops and `extract` discards the partial graph.
walk_aborted: bool,
}

impl<'a, 'tree> TreeSitterWalker<'a, 'tree> {
Expand All @@ -194,10 +250,16 @@ impl<'a, 'tree> TreeSitterWalker<'a, 'tree> {
erlang_last_fn_name: None,
erlang_last_fn_id: None,
cfml_consumed_until: 0,
walk_depth: 0,
walk_aborted: false,
}
}

pub fn extract(mut self, duration_ms: i64) -> ExtractionResult {
if ast_exceeds_safe_depth(self.root, self.spec.language()) {
return self.depth_error_result(duration_ms);
}

let file_node = self.create_file_node();
let file_id = file_node.id.clone();
self.nodes.push(file_node);
Expand All @@ -212,6 +274,10 @@ impl<'a, 'tree> TreeSitterWalker<'a, 'tree> {
}
self.node_stack.pop();

if self.walk_aborted {
return self.depth_error_result(duration_ms);
}

self.flush_fn_ref_candidates();

ExtractionResult {
Expand All @@ -223,7 +289,35 @@ impl<'a, 'tree> TreeSitterWalker<'a, 'tree> {
}
}

fn depth_error_result(&self, duration_ms: i64) -> ExtractionResult {
ExtractionResult {
nodes: Vec::new(),
edges: Vec::new(),
unresolved_references: Vec::new(),
errors: vec![format!(
"AST nesting exceeds safe traversal limit ({MAX_WALK_DEPTH}): {}",
self.file_path
)],
duration_ms,
}
}

fn visit_node(&mut self, node: SyntaxNode<'tree>) {
let node = unwrap_transparent_walk_wrappers(self.spec.language(), node);
if self.walk_aborted {
return;
}
if self.walk_depth >= MAX_WALK_DEPTH {
self.walk_aborted = true;
return;
}

self.walk_depth += 1;
self.visit_node_inner(node);
self.walk_depth -= 1;
}

fn visit_node_inner(&mut self, node: SyntaxNode<'tree>) {
let node_type = node.kind();
let mut skip_children = false;

Expand Down Expand Up @@ -3322,6 +3416,21 @@ impl<'a, 'tree> TreeSitterWalker<'a, 'tree> {
}

fn visit_body_node(&mut self, node: SyntaxNode<'tree>) {
let node = unwrap_transparent_walk_wrappers(self.spec.language(), node);
if self.walk_aborted {
return;
}
if self.walk_depth >= MAX_WALK_DEPTH {
self.walk_aborted = true;
return;
}

self.walk_depth += 1;
self.visit_body_node_inner(node);
self.walk_depth -= 1;
}

fn visit_body_node_inner(&mut self, node: SyntaxNode<'tree>) {
let node_type = node.kind();
self.maybe_capture_fn_refs(node, node_type);
// Inside a function body, GDScript `preload(...)`/`load(...)` calls must
Expand Down
Loading
Loading