diff --git a/README.md b/README.md index 623bda19..45e9a37c 100644 --- a/README.md +++ b/README.md @@ -472,6 +472,12 @@ Godot project graphs), graph traversal, FTS5 search, whole-graph export with deterministic PageRank centrality, MCP/CLI surfaces, and golden byte-stable output. +Extraction is fail-closed per file: a source tree deeper than the deterministic +256-level logical named-node limit records a stable file error, emits no partial +graph for that file, and does not stop other files from being indexed. Incremental +`sync` removes any older graph for the failed file and converges with +`index --force`. + **Doesn't:** no AI / vector / embedding / LLM anywhere inside the binary (hard constraint, `scripts/guardrail.sh`-enforced); no semantic search; no languages beyond the fixed `LANGUAGES` set. diff --git a/crates/codegraph-cli/tests/parallel_index.rs b/crates/codegraph-cli/tests/parallel_index.rs index 6b63f0a7..c824766c 100644 --- a/crates/codegraph-cli/tests/parallel_index.rs +++ b/crates/codegraph-cli/tests/parallel_index.rs @@ -27,6 +27,10 @@ //! `max_file_size` yields the exact size-skip `FileRecord` through the //! parallel parse path (T2): size set to the real byte length, node_count 0, //! and the exact `"File exceeds max size (N > M): path"` error string. +//! * `deep_file_sync_discards_stale_graph_and_matches_force_index` — replacing +//! a previously valid file with a 30,000-level AST removes its old graph, +//! records one stable file error, leaves sibling files intact, and converges +//! to the same canonical DB as a fresh `index --force`. //! * `full_index_leaves_synchronous_at_normal` — after a full `index --force` //! (which runs under `synchronous=OFF` per T5), a FRESH `Store::open` reports //! `PRAGMA synchronous != 0` (NORMAL), proving the bulk-index pragma never @@ -262,6 +266,93 @@ fn oversized_file_size_skips_with_exact_file_record() { ); } +/// DEPTH-ABORT PARITY: a file that crosses the deterministic AST traversal +/// limit contributes no partial graph. Incremental sync must first remove the +/// file's previous graph, persist the stable error, and converge to the same DB +/// as a clean full rebuild of the final filesystem state. +#[test] +fn deep_file_sync_discards_stale_graph_and_matches_force_index() { + const DEPTH: usize = 30_000; + const REL_PATH: &str = "deep.c"; + const EXPECTED_ERROR: &str = "AST nesting exceeds safe traversal limit (256): deep.c"; + + let deep_source = format!( + "void before(void) {{}}\nvoid deep(void) {{{}{}}}\nvoid after(void) {{}}\n", + "{".repeat(DEPTH), + "}".repeat(DEPTH) + ); + let normal_source = "int healthy_target(void) { return 7; }\n"; + + let synced = TestDir::new("deep-sync"); + let synced_project = synced.path().join("project"); + fs::create_dir_all(&synced_project).unwrap(); + fs::write( + synced_project.join(REL_PATH), + "int stale_target(void) { return 1; }\n", + ) + .unwrap(); + fs::write(synced_project.join("ok.c"), normal_source).unwrap(); + init_and_force_index(&synced_project); + + fs::write(synced_project.join(REL_PATH), &deep_source).unwrap(); + let (out, err, ok) = cli(&["sync", synced_project.to_str().unwrap()]); + assert!(ok, "sync failed: stdout={out} stderr={err}"); + + let synced_store = Store::open(&db_path(&synced_project)).unwrap(); + let deep_node_count: i64 = synced_store + .connection() + .query_row( + "SELECT COUNT(*) FROM nodes WHERE file_path = ?1", + [REL_PATH], + |row| row.get(0), + ) + .unwrap(); + assert_eq!( + deep_node_count, 0, + "the failed file's previous and partial nodes must be removed" + ); + + let healthy_count: i64 = synced_store + .connection() + .query_row( + "SELECT COUNT(*) FROM nodes WHERE file_path = 'ok.c' AND name = 'healthy_target'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!( + healthy_count, 1, + "one invalid file must not remove a healthy sibling's graph" + ); + + let (node_count, errors_json): (i64, Option) = synced_store + .connection() + .query_row( + "SELECT node_count, errors FROM files WHERE path = ?1", + [REL_PATH], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .expect("the failed file must retain a files row"); + assert_eq!(node_count, 0); + let errors: Vec = + serde_json::from_str(&errors_json.expect("the failed file must persist its depth error")) + .unwrap(); + assert_eq!(errors, vec![EXPECTED_ERROR]); + drop(synced_store); + + let rebuilt = TestDir::new("deep-force"); + let rebuilt_project = rebuilt.path().join("project"); + fs::create_dir_all(&rebuilt_project).unwrap(); + fs::write(rebuilt_project.join(REL_PATH), &deep_source).unwrap(); + fs::write(rebuilt_project.join("ok.c"), normal_source).unwrap(); + init_and_force_index(&rebuilt_project); + + let after_sync = canonicalize_db(&db_path(&synced_project)).unwrap(); + let after_force = canonicalize_db(&db_path(&rebuilt_project)).unwrap(); + diff_canonical(&after_sync, &after_force, None) + .expect("sync and index --force must converge after a depth-aborted file"); +} + /// WRITE PATH 1 (`index --force`, the rayon producer closure): the persisted /// `files.generated` must carry the real per-file verdict, not a hardcoded false. /// This is the CLI counterpart to the two `codegraph-watch` tests that cover the diff --git a/crates/codegraph-extract/src/walker.rs b/crates/codegraph-extract/src/walker.rs index f3fe8738..b2d20b0b 100644 --- a/crates/codegraph-extract/src/walker.rs +++ b/crates/codegraph-extract/src/walker.rs @@ -145,6 +145,57 @@ fn is_plain_receiver_segment(segment: &str) -> bool { .all(|c| c.is_alphanumeric() || c == '_' || c == '$') } +/// Deterministic logical recursion bound for the native AST walker. +/// +/// Tree-sitter parses deeply nested input iteratively, but this walker descends +/// recursively. Bounding named-node depth keeps one adversarial file from +/// overflowing the process stack. The value is deliberately well above normal +/// source nesting while remaining safe on a 1 MiB worker stack. +const MAX_WALK_DEPTH: usize = 256; + +fn is_transparent_walk_wrapper(language: Language, node: SyntaxNode<'_>) -> bool { + language == Language::Rust + && node.kind() == "expression_statement" + && node.named_child_count() == 1 +} + +fn unwrap_transparent_walk_wrappers( + language: Language, + mut node: SyntaxNode<'_>, +) -> SyntaxNode<'_> { + while is_transparent_walk_wrapper(language, node) { + let Some(child) = node.named_child(0) else { + break; + }; + node = child; + } + node +} + +fn ast_exceeds_safe_depth(root: SyntaxNode<'_>, language: Language) -> bool { + let mut pending = vec![(root, 1usize)]; + + while let Some((node, depth)) = pending.pop() { + if depth > MAX_WALK_DEPTH { + return true; + } + + for index in 0..node.named_child_count() { + if let Some(child) = node.named_child(index as u32) { + // tree-sitter-rust inserts an inert `expression_statement` + // between every nested block. It carries no extraction + // semantics, so count the source-level block rather than + // double-charging this grammar wrapper. + let child_depth = + depth + usize::from(!is_transparent_walk_wrapper(language, child)); + pending.push((child, child_depth)); + } + } + } + + false +} + pub struct TreeSitterWalker<'a, 'tree> { file_path: &'a str, source: &'a str, @@ -170,6 +221,11 @@ pub struct TreeSitterWalker<'a, 'tree> { /// component body) have already been consumed, so the top-level `program` /// walk skips them instead of re-visiting. 0 outside CFML tag files. cfml_consumed_until: usize, + /// Current named-node recursion depth. Root is depth 1. + walk_depth: usize, + /// Latched once a child would exceed [`MAX_WALK_DEPTH`]. All remaining + /// descent stops and `extract` discards the partial graph. + walk_aborted: bool, } impl<'a, 'tree> TreeSitterWalker<'a, 'tree> { @@ -194,10 +250,16 @@ impl<'a, 'tree> TreeSitterWalker<'a, 'tree> { erlang_last_fn_name: None, erlang_last_fn_id: None, cfml_consumed_until: 0, + walk_depth: 0, + walk_aborted: false, } } pub fn extract(mut self, duration_ms: i64) -> ExtractionResult { + if ast_exceeds_safe_depth(self.root, self.spec.language()) { + return self.depth_error_result(duration_ms); + } + let file_node = self.create_file_node(); let file_id = file_node.id.clone(); self.nodes.push(file_node); @@ -212,6 +274,10 @@ impl<'a, 'tree> TreeSitterWalker<'a, 'tree> { } self.node_stack.pop(); + if self.walk_aborted { + return self.depth_error_result(duration_ms); + } + self.flush_fn_ref_candidates(); ExtractionResult { @@ -223,7 +289,35 @@ impl<'a, 'tree> TreeSitterWalker<'a, 'tree> { } } + fn depth_error_result(&self, duration_ms: i64) -> ExtractionResult { + ExtractionResult { + nodes: Vec::new(), + edges: Vec::new(), + unresolved_references: Vec::new(), + errors: vec![format!( + "AST nesting exceeds safe traversal limit ({MAX_WALK_DEPTH}): {}", + self.file_path + )], + duration_ms, + } + } + fn visit_node(&mut self, node: SyntaxNode<'tree>) { + let node = unwrap_transparent_walk_wrappers(self.spec.language(), node); + if self.walk_aborted { + return; + } + if self.walk_depth >= MAX_WALK_DEPTH { + self.walk_aborted = true; + return; + } + + self.walk_depth += 1; + self.visit_node_inner(node); + self.walk_depth -= 1; + } + + fn visit_node_inner(&mut self, node: SyntaxNode<'tree>) { let node_type = node.kind(); let mut skip_children = false; @@ -3322,6 +3416,21 @@ impl<'a, 'tree> TreeSitterWalker<'a, 'tree> { } fn visit_body_node(&mut self, node: SyntaxNode<'tree>) { + let node = unwrap_transparent_walk_wrappers(self.spec.language(), node); + if self.walk_aborted { + return; + } + if self.walk_depth >= MAX_WALK_DEPTH { + self.walk_aborted = true; + return; + } + + self.walk_depth += 1; + self.visit_body_node_inner(node); + self.walk_depth -= 1; + } + + fn visit_body_node_inner(&mut self, node: SyntaxNode<'tree>) { let node_type = node.kind(); self.maybe_capture_fn_refs(node, node_type); // Inside a function body, GDScript `preload(...)`/`load(...)` calls must diff --git a/crates/codegraph-extract/tests/deep_nesting.rs b/crates/codegraph-extract/tests/deep_nesting.rs new file mode 100644 index 00000000..7aaad04b --- /dev/null +++ b/crates/codegraph-extract/tests/deep_nesting.rs @@ -0,0 +1,148 @@ +use codegraph_core::types::Language; +use codegraph_extract::extract_source; + +const DEEP: usize = 30_000; +const SHALLOW: usize = 200; +const EXPECTED_ERROR: &str = "AST nesting exceeds safe traversal limit (256)"; + +fn nested_source(language: Language, depth: usize) -> (&'static str, String) { + match language { + Language::C => ( + "deep.c", + format!( + "void before(void) {{}}\nvoid deep(void) {{{}{}}}\nvoid after(void) {{}}\n", + "{".repeat(depth), + "}".repeat(depth) + ), + ), + Language::Cpp => ( + "deep.cpp", + format!( + "void before() {{}}\nvoid deep() {{{}{}}}\nvoid after() {{}}\n", + "{".repeat(depth), + "}".repeat(depth) + ), + ), + Language::Rust => ( + "deep.rs", + format!( + "fn before() {{}}\nfn deep() {{{}{}}}\nfn after() {{}}\n", + "{".repeat(depth), + "}".repeat(depth) + ), + ), + Language::TypeScript => ( + "deep.ts", + format!( + "function before() {{}}\nfunction deep() {{ return {}value{}; }}\nfunction after() {{}}\n", + "(".repeat(depth), + ")".repeat(depth) + ), + ), + Language::Python => ( + "deep.py", + format!( + "def before():\n pass\nvalue = {}0{}\ndef after():\n pass\n", + "[".repeat(depth), + "]".repeat(depth) + ), + ), + _ => unreachable!("test helper only supports the five guarded languages"), + } +} + +fn extract_on_small_stack( + language: Language, + depth: usize, +) -> codegraph_core::types::ExtractionResult { + let (file_path, source) = nested_source(language, depth); + std::thread::Builder::new() + .name(format!("deep-nesting-{language}")) + .stack_size(1024 * 1024) + .spawn(move || extract_source(file_path, &source, Some(language))) + .expect("spawn 1 MiB extraction thread") + .join() + .expect("deep extraction must return instead of overflowing") +} + +#[test] +fn deep_native_walkers_fail_per_file_without_partial_graph() { + for language in [ + Language::C, + Language::Cpp, + Language::Rust, + Language::TypeScript, + Language::Python, + ] { + let result = extract_on_small_stack(language, DEEP); + assert!( + result + .errors + .iter() + .any(|error| error.contains(EXPECTED_ERROR)), + "{language}: errors={:?}", + result.errors + ); + assert!( + result.nodes.is_empty(), + "{language}: partial nodes survived" + ); + assert!( + result.edges.is_empty(), + "{language}: partial edges survived" + ); + assert!( + result.unresolved_references.is_empty(), + "{language}: partial references survived" + ); + } +} + +#[test] +fn ordinary_two_hundred_level_sources_remain_clean() { + for language in [ + Language::C, + Language::Cpp, + Language::Rust, + Language::TypeScript, + Language::Python, + ] { + let result = extract_on_small_stack(language, SHALLOW); + assert!(result.errors.is_empty(), "{language}: {:?}", result.errors); + assert!( + result.nodes.iter().any(|node| node.name == "before"), + "{language}: normal leading symbol disappeared" + ); + assert!( + result.nodes.iter().any(|node| node.name == "after"), + "{language}: normal trailing symbol disappeared" + ); + } +} + +#[test] +fn logical_depth_limit_is_inclusive_and_deterministic() { + // For this C shape the root + function + outer body contribute three + // named-node levels, so 253 nested compounds land exactly at depth 256. + let at_limit = extract_on_small_stack(Language::C, 253); + assert!(at_limit.errors.is_empty(), "{:?}", at_limit.errors); + + let beyond_limit = extract_on_small_stack(Language::C, 254); + assert_eq!(beyond_limit.errors.len(), 1, "{:?}", beyond_limit.errors); + assert!(beyond_limit.errors[0].contains(EXPECTED_ERROR)); + assert!(beyond_limit.nodes.is_empty()); +} + +#[test] +fn depth_abort_is_scoped_to_one_extraction() { + let deep = extract_on_small_stack(Language::C, DEEP); + assert_eq!(deep.errors.len(), 1, "{:?}", deep.errors); + + let normal = extract_source( + "normal.c", + "int add(int a, int b) { return a + b; }\n", + Some(Language::C), + ); + assert!(normal.errors.is_empty(), "{:?}", normal.errors); + assert!(normal.nodes.iter().any(|node| node.name == "add")); +} diff --git a/docs/architecture.md b/docs/architecture.md index fb7c82e2..638a32f8 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -128,6 +128,12 @@ graph LR - `create_node`：生成节点 ID（非文件节点走 `generate_node_id(file, kind, name, start_line)`，1-based 行号），从语义栈 （排除文件节点）构造 `qualified_name`，并即时发出 `contains` 边。 +- **深度安全边界**：正式语义遍历前先用显式 `Vec` 迭代检查逻辑 named-node + 深度，固定上限 256；主 walker 与函数体 walker 共享运行时深度闩锁。超限时整份 + 文件结果 fail-closed（节点、边、未解析引用全部丢弃，只保留稳定文件错误），其他 + 文件继续处理。Rust grammar 在嵌套 block 之间插入的单子 + `expression_statement` 是无语义包装层，预检与 walker 都将其折叠，因此 200 层 + 正常源码嵌套仍在安全预算内。`sync` 会先替换该文件旧结果，所以不会遗留陈旧图谱。 **关键边界：** 提取层把**结构性边**（`contains`）与**未解析引用** （calls/imports/instantiates/decorates）分离——`contains` 立即发出，其余作为 diff --git a/docs/readme/README.zh-CN.md b/docs/readme/README.zh-CN.md index bd97a202..fabeb8b1 100644 --- a/docs/readme/README.zh-CN.md +++ b/docs/readme/README.zh-CN.md @@ -450,6 +450,11 @@ Java、C/C++、C#、Vue、Svelte、GDScript 等——详见 [`../languages.md`](../languages.md)），跨文件解析（含 Godot 项目图），图遍历， FTS5 检索，全图导出（含确定性 PageRank 中心性），MCP/CLI 表面，golden 字节稳定输出。 +提取按文件 fail-closed：若源码树超过确定性的 256 层逻辑 named-node 深度限制， +CodeGraph 会记录稳定的文件错误，不保留该文件的部分节点、边或未解析引用，并继续 +索引项目中的其他文件。后续增量 `sync` 会删除该文件原有图谱，并与 +`index --force` 收敛到相同结果。 + **不做什么：** 二进制内部无任何 AI/向量/嵌入/LLM（硬约束，`scripts/guardrail.sh` 强制执行）；无语义检索；不新增固定 `LANGUAGES` 集以外的语言。 diff --git a/docs/upstream-sync/UPSTREAM.md b/docs/upstream-sync/UPSTREAM.md index 7c8409d4..601576b7 100644 --- a/docs/upstream-sync/UPSTREAM.md +++ b/docs/upstream-sync/UPSTREAM.md @@ -77,6 +77,34 @@ ## Sync log +### 2026-08-31 — Wave 0 deep-AST guard IMPLEMENTED in PR #249; release pending + +Upstream `838006c947ebe69407e8f64ba756f97132f411f4` fixed the process-crash +class caused by deeply recursive native walkers. Rust PR +[#249](https://github.com/sunerpy/codegraph-rust/pull/249), commit `53360d9`, +implements the portable behavior with a deterministic design rather than +copying upstream's platform stack-pointer / wasm fallback: + +- an iterative preflight rejects logical named-node depth above 256 before any + recursive semantic traversal; +- the main walker and function-body walker share a fail-closed runtime depth + latch, while Rust's inert single-child `expression_statement` grammar wrapper + is folded so 200 levels of ordinary source nesting remain valid; +- an over-limit file contributes no partial nodes, edges, or unresolved + references, records one stable file error, and does not stop sibling files; +- incremental `sync` removes the file's previous graph and is canonically + identical to a fresh `index --force` for the same final filesystem state. + +Coverage includes 30,000-level C, C++, Rust, TypeScript, and Python inputs on a +1 MiB test thread, the exact inclusive 256/257 boundary, per-file isolation, and +the incremental/full-rebuild equivalence oracle. Local `make ci` and the +pre-push gate both passed. This change deliberately keeps extraction version 11 +and changes no golden artifact or node-id formula. + +This entry records implementation only. The item remains release-pending until +the release-please PR, six-platform artifacts, checksums, and published Linux +black-box smoke test complete; tracked colby parity therefore remains `1.5.0`. + ### 2026-08-31 — implementation baseline refreshed at codegraph-rs `v0.48.1` Before implementation began, both repositories and the three load-bearing open diff --git a/skills/codegraph/SKILL.md b/skills/codegraph/SKILL.md index abf7a561..f05d1eb4 100644 --- a/skills/codegraph/SKILL.md +++ b/skills/codegraph/SKILL.md @@ -71,6 +71,13 @@ failed command explicitly prints `codegraph init ...`, run that exact command to replace the stale index; do not keep retrying `sync`. Do not generalize this to mixed or other corruption when the CLI does not prescribe `init`. +Extraction is fail-closed per file at a deterministic logical named-node depth +of 256. If an adversarial or generated file exceeds that limit, CodeGraph records +one file error, contributes no partial nodes/edges/references for that file, and +continues indexing the rest of the project. A later `sync` also removes any graph +previously produced by that file. Do not retry with `index --force`: a full +rebuild intentionally produces the same file-level result. + ### Start the MCP server ```bash