From 1e114cdfa7263837b9e90930cae568b5787c80f1 Mon Sep 17 00:00:00 2001 From: sunerpy Date: Mon, 31 Aug 2026 23:28:42 +0800 Subject: [PATCH] feat(search): port upstream retrieval and ranking controls --- Cargo.lock | 3 + Cargo.toml | 1 + README.md | 22 +- crates/codegraph-cli/src/main.rs | 12 +- crates/codegraph-core/Cargo.toml | 1 + crates/codegraph-core/src/config.rs | 25 + crates/codegraph-core/src/deprioritize.rs | 257 ++++++++++ crates/codegraph-core/src/lib.rs | 1 + crates/codegraph-graph/Cargo.toml | 2 + crates/codegraph-graph/src/lib.rs | 2 + crates/codegraph-graph/src/query/mod.rs | 69 ++- crates/codegraph-graph/src/query/scoring.rs | 23 +- .../src/segment_match.rs | 3 +- .../src/segments.rs | 39 +- crates/codegraph-graph/tests/search.rs | 140 ++++++ crates/codegraph-mcp/src/engine.rs | 442 ++++++++++++++++-- crates/codegraph-mcp/src/lib.rs | 1 + crates/codegraph-mcp/src/query_paths.rs | 391 ++++++++++++++++ crates/codegraph-store/src/queries.rs | 93 +++- docs/cli.md | 53 ++- docs/equivalence.md | 16 + docs/mcp.md | 21 + docs/readme/README.zh-CN.md | 20 +- skills/codegraph/SKILL.md | 35 +- 24 files changed, 1589 insertions(+), 83 deletions(-) create mode 100644 crates/codegraph-core/src/deprioritize.rs rename crates/{codegraph-cli => codegraph-graph}/src/segment_match.rs (98%) rename crates/{codegraph-cli => codegraph-graph}/src/segments.rs (90%) create mode 100644 crates/codegraph-mcp/src/query_paths.rs diff --git a/Cargo.lock b/Cargo.lock index e7c7d658..7b4bab49 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -456,6 +456,7 @@ name = "codegraph-core" version = "0.48.3" dependencies = [ "anyhow", + "globset", "serde", "serde_json", "sha2", @@ -544,9 +545,11 @@ dependencies = [ "codegraph-extract", "codegraph-resolve", "codegraph-store", + "regex", "rusqlite", "serde", "serde_json", + "unicode-normalization", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 09b7e5b4..e02977a3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,6 +36,7 @@ thiserror = "1" sha2 = "0.10" sha1_smol = "1" rayon = "1.8" +globset = "0.4" # Logging (rust-base-convention) tracing = "0.1" diff --git a/README.md b/README.md index 347a53cc..bda31f67 100644 --- a/README.md +++ b/README.md @@ -130,7 +130,9 @@ codegraph install --yes # detects installed agents and wires them u **Rules of thumb:** reach for `codegraph_explore` _before_ reading files; trust its results (full AST parse — don't re-verify with grep); use `codegraph_impact` for refactor blast-radius rather than walking callers by hand. The index lags -file writes by ~1s; tool responses flag any stale files. +file writes by ~1s; tool responses flag any stale files. If the question names +a source path, include it in the explore query: explicit paths are normalized +and pinned ahead of fuzzy matches, while unresolved paths are reported. @@ -431,6 +433,19 @@ and newly included/custom-extension files are added without restarting the MCP server. Invalid TOML keeps the last valid watcher scope and reports the error; malformed JSON keeps the existing tolerant empty-override behavior. +Keep third-party or generated source searchable while ranking it below +first-party code with ranking-only rules: + +```toml +[indexing] +deprioritize = ["vendor/**", "generated/**"] +``` + +`.codegraph/codegraph.json` also accepts a top-level `deprioritize` array for +compatibility. JSON rules run first, TOML rules run last, ordered `!pattern` +exceptions make matching last-rule-wins, and no file/node/edge is removed. +Search and explore reload the addressed project's policy on every request. + Full env-var table, HTTP server details, filesystem fallback behavior, and the Claude prompt-hook: [`docs/mcp.md`](docs/mcp.md) and [`docs/cli.md`](docs/cli.md#daemon-watch--environment-variables). @@ -439,8 +454,9 @@ Claude prompt-hook: [`docs/mcp.md`](docs/mcp.md) and ## CLI Subcommands -Core commands: `init`, `index`, `sync`, `search`, `files`, `status`, `serve`, -`callers`, `callees`, `impact`, `affected`, `check`, `export`, `unlock`. +Core commands: `init`, `index`, `sync`, `search`, `explore`, `node`, `files`, +`status`, `serve`, `callers`, `callees`, `impact`, `affected`, `check`, `audit`, +`export`, `unlock`, `http`, `mcp`. Agent / install commands: `install`, `uninstall`, `skill`, `self-update`, `completions` (`--install` sets up tab completion for bash/zsh/fish/powershell/elvish). diff --git a/crates/codegraph-cli/src/main.rs b/crates/codegraph-cli/src/main.rs index dee1371a..e4e6701c 100644 --- a/crates/codegraph-cli/src/main.rs +++ b/crates/codegraph-cli/src/main.rs @@ -17,6 +17,7 @@ use anyhow::{Context, Result, anyhow, bail}; use clap::{CommandFactory, Parser, Subcommand, ValueEnum}; use clap_complete::{Shell, generate}; use codegraph_core::config::Config; +use codegraph_core::deprioritize::DeprioritizeMatcher; use codegraph_core::generated_header::detect_generated_file; use codegraph_core::logger::{LoggerConfig, init_logger}; use codegraph_core::node_id::hash_content; @@ -24,6 +25,7 @@ use codegraph_core::types::{ExtractionResult, FileRecord, Language, Node, NodeKi use codegraph_extract::{ExtractOptions, detect_language_with, extract_source_with}; use codegraph_graph::graph::{GodotReach, GraphTraverser}; use codegraph_graph::query::{SearchOptions, search_nodes}; +use codegraph_graph::{segment_match, segments}; use codegraph_mcp::{McpServer, RunUntilAdoption}; use codegraph_resolve::ReferenceResolver; use codegraph_store::queries::SearchResult; @@ -36,8 +38,6 @@ use time::OffsetDateTime; use time::format_description::well_known::Rfc3339; mod installer; -mod segment_match; -mod segments; mod structural_gate; /// Test-only: the ONE process-wide environment lock for this binary. @@ -1963,6 +1963,12 @@ fn cmd_search( ) -> Result<()> { let project = resolve_required_project(path)?; let store = open_store(&project)?; + let paths = codegraph_core::IndexPaths::resolve( + &project, + std::env::var("CODEGRAPH_DIR").ok().as_deref(), + )?; + let config = Config::load_for_paths(None, &paths)?; + let deprioritize = Arc::new(DeprioritizeMatcher::load_for_paths(&paths, &config)); let kinds = kind .as_deref() .map(parse_node_kind) @@ -1977,6 +1983,8 @@ fn cmd_search( languages: Vec::new(), limit: Some(limit), offset: Some(0), + seed_names: Vec::new(), + deprioritize: Some(deprioritize), }, &project_name_tokens(&project), )?; diff --git a/crates/codegraph-core/Cargo.toml b/crates/codegraph-core/Cargo.toml index bba78fc5..3ec6ecc4 100644 --- a/crates/codegraph-core/Cargo.toml +++ b/crates/codegraph-core/Cargo.toml @@ -23,3 +23,4 @@ tracing-subscriber = { workspace = true } tracing-appender = { workspace = true } time = { workspace = true } sha2 = { workspace = true } +globset = { workspace = true } diff --git a/crates/codegraph-core/src/config.rs b/crates/codegraph-core/src/config.rs index e81743a1..f90eb566 100644 --- a/crates/codegraph-core/src/config.rs +++ b/crates/codegraph-core/src/config.rs @@ -83,6 +83,13 @@ pub struct IndexingConfig { /// behavior byte-identical). #1063. #[serde(default)] pub include: Vec, + /// Root-relative gitignore-style patterns for source that remains indexed + /// and directly retrievable, but should lose ranking ties to first-party + /// code. Rules are evaluated in order with last-match-wins semantics; + /// `!pattern` clears an earlier match. The project's `codegraph.json` + /// rules run first and these TOML rules run last, so TOML is authoritative. + #[serde(default)] + pub deprioritize: Vec, } fn default_max_file_size() -> u64 { @@ -206,6 +213,7 @@ impl Default for IndexingConfig { ignore_paths: default_ignore_paths(), exclude: Vec::new(), include: Vec::new(), + deprioritize: Vec::new(), } } } @@ -404,6 +412,7 @@ name = "my-project" assert!(cfg.watch.enabled); // default assert_eq!(cfg.watch.debounce_ms, 2000); // default assert!(cfg.indexing.exclude.is_empty()); // off by default + assert!(cfg.indexing.deprioritize.is_empty()); // ranking-only, off by default } #[test] @@ -429,6 +438,22 @@ max_file_size = 2097152 assert!(cfg.indexing.exclude.is_empty()); } + #[test] + fn test_deprioritize_parses_and_preserves_rule_order() { + let config = r#" +[app] +name = "p" + +[indexing] +deprioritize = ["vendor/**", "!vendor/first-party/**", "generated/**"] +"#; + let cfg: Config = toml::from_str(config).expect("should parse"); + assert_eq!( + cfg.indexing.deprioritize, + vec!["vendor/**", "!vendor/first-party/**", "generated/**"] + ); + } + #[test] fn test_include_parses_and_defaults_empty() { let with_include = r#" diff --git a/crates/codegraph-core/src/deprioritize.rs b/crates/codegraph-core/src/deprioritize.rs new file mode 100644 index 00000000..ac132564 --- /dev/null +++ b/crates/codegraph-core/src/deprioritize.rs @@ -0,0 +1,257 @@ +//! Ranking-only project path de-prioritization. +//! +//! `codegraph.json` and `[indexing].deprioritize` describe source that must stay +//! indexed and directly retrievable while losing search/explore ranking ties to +//! first-party code. Matching a rule never changes files, nodes, edges, node +//! ids, or extraction goldens. + +use std::path::Path; + +use globset::{GlobBuilder, GlobMatcher}; + +use crate::{IndexPaths, config::Config}; + +/// Ordered, immutable last-match-wins matcher for project-relative paths. +#[derive(Debug, Clone, Default)] +pub struct DeprioritizeMatcher { + rules: Vec, +} + +#[derive(Debug, Clone)] +struct Rule { + matchers: Vec, + deprioritize: bool, +} + +impl DeprioritizeMatcher { + /// Load one project's ranking rules. + /// + /// The tolerant JSON compatibility source runs first. TOML then appends its + /// rules and therefore has the final say under last-match-wins semantics. + #[must_use] + pub fn load_for_paths(paths: &IndexPaths, config: &Config) -> Self { + let mut patterns = load_json_patterns(&paths.extension_config()); + patterns.extend(config.indexing.deprioritize.iter().cloned()); + Self::from_patterns(patterns) + } + + /// Compile an ordered rule stream. Blank or invalid rules are ignored. + #[must_use] + pub fn from_patterns(patterns: impl IntoIterator) -> Self { + let rules = patterns + .into_iter() + .filter_map(|pattern| compile_rule(&pattern)) + .collect(); + Self { rules } + } + + /// Whether `path` is de-prioritized after applying every matching rule. + #[must_use] + pub fn is_match(&self, path: &str) -> bool { + let normalized = normalize_path(path); + let mut deprioritized = false; + for rule in &self.rules { + if rule + .matchers + .iter() + .any(|matcher| matcher.is_match(&normalized)) + { + deprioritized = rule.deprioritize; + } + } + deprioritized + } + + #[must_use] + pub fn is_empty(&self) -> bool { + self.rules.is_empty() + } +} + +fn load_json_patterns(path: &Path) -> Vec { + let Ok(contents) = std::fs::read_to_string(path) else { + return Vec::new(); + }; + let parsed: serde_json::Value = match serde_json::from_str(&contents) { + Ok(parsed) => parsed, + Err(error) => { + tracing::warn!( + target: "codegraph_core::deprioritize", + path = %path.display(), + %error, + "ignoring malformed codegraph.json ranking config" + ); + return Vec::new(); + } + }; + let Some(raw) = parsed.get("deprioritize") else { + return Vec::new(); + }; + let Some(entries) = raw.as_array() else { + tracing::warn!( + target: "codegraph_core::deprioritize", + path = %path.display(), + "ignoring codegraph.json deprioritize: expected an array" + ); + return Vec::new(); + }; + entries + .iter() + .filter_map(|entry| { + let Some(pattern) = entry.as_str() else { + tracing::warn!( + target: "codegraph_core::deprioritize", + path = %path.display(), + "ignoring non-string codegraph.json deprioritize rule" + ); + return None; + }; + let trimmed = pattern.trim(); + (!trimmed.is_empty()).then(|| trimmed.to_string()) + }) + .collect() +} + +fn compile_rule(raw: &str) -> Option { + let trimmed = raw.trim(); + if trimmed.is_empty() { + return None; + } + let (deprioritize, body) = match trimmed.strip_prefix('!') { + Some(body) => (false, body.trim()), + None => (true, trimmed), + }; + if body.is_empty() { + return None; + } + + let normalized = normalize_pattern(body); + if normalized.is_empty() { + return None; + } + let mut globs = vec![normalized.clone()]; + + // Gitignore-style bare patterns match a path component at any depth. + if !normalized.contains('/') { + globs.push(format!("**/{normalized}")); + globs.push(format!("**/{normalized}/**")); + } else if !has_glob_meta(&normalized) { + // A literal directory pattern also covers its descendants. + globs.push(format!("{normalized}/**")); + } + + let mut matchers = Vec::new(); + for glob in globs { + match GlobBuilder::new(&glob) + .literal_separator(true) + .backslash_escape(false) + .build() + { + Ok(glob) => matchers.push(glob.compile_matcher()), + Err(error) => { + tracing::warn!( + target: "codegraph_core::deprioritize", + pattern = %raw, + %error, + "ignoring invalid deprioritize glob" + ); + } + } + } + (!matchers.is_empty()).then_some(Rule { + matchers, + deprioritize, + }) +} + +fn normalize_pattern(pattern: &str) -> String { + let trailing_slash = pattern.trim().ends_with(['/', '\\']); + let mut normalized = normalize_path(pattern); + if trailing_slash { + normalized.push_str("/**"); + } + normalized +} + +fn normalize_path(path: &str) -> String { + let mut normalized = path.trim().replace('\\', "/"); + while let Some(rest) = normalized.strip_prefix("./") { + normalized = rest.to_string(); + } + normalized = normalized.trim_start_matches('/').to_string(); + while normalized.contains("//") { + normalized = normalized.replace("//", "/"); + } + normalized.trim_end_matches('/').to_string() +} + +fn has_glob_meta(pattern: &str) -> bool { + pattern + .bytes() + .any(|byte| matches!(byte, b'*' | b'?' | b'[' | b'{')) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn matcher(patterns: &[&str]) -> DeprioritizeMatcher { + DeprioritizeMatcher::from_patterns(patterns.iter().map(|p| (*p).to_string())) + } + + #[test] + fn ordered_rules_are_last_match_wins() { + let matcher = matcher(&[ + "vendor/**", + "!vendor/first-party/**", + "vendor/first-party/generated/**", + ]); + assert!(matcher.is_match("vendor/sdk/client.rs")); + assert!(!matcher.is_match("vendor/first-party/api.rs")); + assert!(matcher.is_match("vendor/first-party/generated/api.rs")); + } + + #[test] + fn bare_and_windows_patterns_match_components() { + let matcher = matcher(&["fixtures", "generated/**"]); + assert!(matcher.is_match(r"crates\demo\fixtures\sample.rs")); + assert!(matcher.is_match("generated/schema.rs")); + assert!(!matcher.is_match("src/generated_helper.rs")); + } + + #[test] + fn blanks_and_invalid_globs_are_ignored() { + let matcher = matcher(&["", " ", "["]); + assert!(matcher.is_empty()); + } + + #[test] + fn json_rules_run_before_authoritative_toml_rules() { + let root = std::env::temp_dir().join(format!( + "codegraph-deprioritize-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::create_dir_all(&root).unwrap(); + let paths = IndexPaths::resolve(&root, None).unwrap(); + std::fs::create_dir_all(paths.current_root()).unwrap(); + std::fs::write( + paths.extension_config(), + r#"{"deprioritize":["vendor/**","!vendor/keep/**",42,""]}"#, + ) + .unwrap(); + let mut config = Config::default(); + config.indexing.deprioritize = vec!["vendor/keep/**".to_string()]; + + let matcher = DeprioritizeMatcher::load_for_paths(&paths, &config); + assert!(matcher.is_match("vendor/sdk/a.rs")); + assert!( + matcher.is_match("vendor/keep/a.rs"), + "later TOML rule must override JSON negation" + ); + std::fs::remove_dir_all(root).unwrap(); + } +} diff --git a/crates/codegraph-core/src/lib.rs b/crates/codegraph-core/src/lib.rs index e90c6fda..cef3be8b 100644 --- a/crates/codegraph-core/src/lib.rs +++ b/crates/codegraph-core/src/lib.rs @@ -1,6 +1,7 @@ //! codegraph-core crate pub mod config; +pub mod deprioritize; pub mod errors; pub mod file_class; pub mod generated_header; diff --git a/crates/codegraph-graph/Cargo.toml b/crates/codegraph-graph/Cargo.toml index 767e2b80..15830028 100644 --- a/crates/codegraph-graph/Cargo.toml +++ b/crates/codegraph-graph/Cargo.toml @@ -19,6 +19,8 @@ anyhow = { workspace = true } codegraph-core = { path = "../codegraph-core" } codegraph-store = { path = "../codegraph-store" } rusqlite = { workspace = true } +regex = "1.10" +unicode-normalization = "0.1" [dev-dependencies] serde_json = { workspace = true } diff --git a/crates/codegraph-graph/src/lib.rs b/crates/codegraph-graph/src/lib.rs index cb214d06..c7627602 100644 --- a/crates/codegraph-graph/src/lib.rs +++ b/crates/codegraph-graph/src/lib.rs @@ -3,3 +3,5 @@ pub mod centrality; pub mod export; pub mod graph; pub mod query; +pub mod segment_match; +pub mod segments; diff --git a/crates/codegraph-graph/src/query/mod.rs b/crates/codegraph-graph/src/query/mod.rs index 8b71b5c6..612fa6b4 100644 --- a/crates/codegraph-graph/src/query/mod.rs +++ b/crates/codegraph-graph/src/query/mod.rs @@ -2,7 +2,9 @@ pub mod parser; pub mod scoring; use std::collections::HashSet; +use std::sync::Arc; +use codegraph_core::deprioritize::DeprioritizeMatcher; use codegraph_core::types::{Language, Node, NodeKind}; use codegraph_store::Store; use codegraph_store::queries::SearchResult; @@ -26,6 +28,13 @@ pub struct SearchOptions { pub languages: Vec, pub limit: Option, pub offset: Option, + /// Exact names supplied by a higher-level retrieval surface, such as + /// explore's identifier-segment vocabulary. Ordinary search leaves this + /// empty, avoiding an all-name vocabulary scan on its hot path. + pub seed_names: Vec, + /// Ranking-only project path matcher. `None` preserves the pre-config + /// ordering byte-for-byte. + pub deprioritize: Option>, } fn merge_unique(base: &[T], extra: &[T]) -> Vec { @@ -103,21 +112,36 @@ pub fn search_nodes( } } + seed_named_nodes(store, &options.seed_names, &kinds, &languages, &mut results)?; seed_multi_segment_definers(store, query, &kinds, &mut results)?; if !results.is_empty() && (!text.is_empty() || !query.is_empty()) { let scoring_query = if !text.is_empty() { &text } else { query }; for result in &mut results { + let deprioritized = options + .deprioritize + .as_ref() + .is_some_and(|matcher| matcher.is_match(&result.node.file_path)); + let name_bonus = scoring::name_match_bonus(&result.node.name, scoring_query); result.score += scoring::kind_bonus(result.node.kind) + scoring::score_path_relevance( &result.node.file_path, scoring_query, project_name_tokens, + deprioritized, ) - + scoring::name_match_bonus(&result.node.name, scoring_query); + + if deprioritized { + (name_bonus * 0.75).round() + } else { + name_bonus + }; } - sort_by_exact_name_then_score_desc(&mut results, scoring_query); + sort_by_exact_name_then_score_desc( + &mut results, + scoring_query, + options.deprioritize.as_deref(), + ); if results.len() > limit as usize { results.truncate(limit as usize); } @@ -149,17 +173,50 @@ fn sort_by_score_desc(results: &mut [SearchResult]) { }); } -fn sort_by_exact_name_then_score_desc(results: &mut [SearchResult], query: &str) { +fn sort_by_exact_name_then_score_desc( + results: &mut [SearchResult], + query: &str, + deprioritize: Option<&DeprioritizeMatcher>, +) { // Both sorts are stable: establish score order first, then group exact whole-name // matches ahead of non-exact rows without mutating the externally visible scores. sort_by_score_desc(results); results.sort_by(|a, b| { - let a_is_exact = scoring::is_exact_name_match(&a.node.name, query); - let b_is_exact = scoring::is_exact_name_match(&b.node.name, query); + let a_is_exact = scoring::is_exact_name_match(&a.node.name, query) + && !deprioritize.is_some_and(|matcher| matcher.is_match(&a.node.file_path)); + let b_is_exact = scoring::is_exact_name_match(&b.node.name, query) + && !deprioritize.is_some_and(|matcher| matcher.is_match(&b.node.file_path)); b_is_exact.cmp(&a_is_exact) }); } +fn seed_named_nodes( + store: &Store, + seed_names: &[String], + kinds: &[NodeKind], + languages: &[Language], + results: &mut Vec, +) -> rusqlite::Result<()> { + if seed_names.is_empty() { + return Ok(()); + } + let mut existing = results + .iter() + .map(|result| result.node.id.clone()) + .collect::>(); + for name in seed_names { + for node in store.nodes_by_exact_name_nocase(name, kinds, languages)? { + if existing.insert(node.id.clone()) { + results.push(SearchResult { + node, + score: INFIX_SEED_SCORE * 0.6, + }); + } + } + } + Ok(()) +} + /// Multi-hump field-name seeding (upstream `1de7e8f` #1319). /// /// A query token that names a multi-SEGMENT field (`userProfileId`, @@ -197,7 +254,7 @@ fn seed_multi_segment_definers( continue; } let mut seeded = 0usize; - for node in store.callable_nodes_by_name_infix(&needle, INFIX_CANDIDATE_CAP)? { + for node in store.seedable_nodes_by_name_infix(&needle, INFIX_CANDIDATE_CAP)? { if seeded >= INFIX_SEED_CAP { break; } diff --git a/crates/codegraph-graph/src/query/scoring.rs b/crates/codegraph-graph/src/query/scoring.rs index 59c3dd18..422b0c45 100644 --- a/crates/codegraph-graph/src/query/scoring.rs +++ b/crates/codegraph-graph/src/query/scoring.rs @@ -408,6 +408,7 @@ pub fn score_path_relevance( file_path: &str, query: &str, project_name_tokens: &HashSet, + is_deprioritized: bool, ) -> f64 { let path_lower = file_path.to_lowercase(); let file_name = basename(file_path).to_lowercase(); @@ -451,7 +452,9 @@ pub fn score_path_relevance( let query_lower = query.to_lowercase(); let is_test_query = query_lower.contains("test") || query_lower.contains("spec"); - if !is_test_query && is_test_file(file_path) { + // A configured peripheral path stays penalized even for a test-oriented + // query. A path that is both test-like and configured is docked once. + if (!is_test_query && is_test_file(file_path)) || is_deprioritized { score -= 15.0; } @@ -855,9 +858,9 @@ mod tests { fn score_path_relevance_filename_beats_dir_beats_path() { let empty = HashSet::new(); // token in filename → +10 - let fname = score_path_relevance("src/auth/login.ts", "login", &empty); + let fname = score_path_relevance("src/auth/login.ts", "login", &empty, false); // token in dir → +5 - let dir = score_path_relevance("src/login/handler.ts", "login", &empty); + let dir = score_path_relevance("src/login/handler.ts", "login", &empty, false); assert!( fname > dir, "filename hit outranks dir hit: {fname} vs {dir}" @@ -869,33 +872,35 @@ mod tests { fn score_path_relevance_path_only_hit_scores_three() { let empty = HashSet::new(); // "widget" appears only deep in the path segment, not filename/immediate dir. - let score = score_path_relevance("app/widget/nested/main.ts", "widget", &empty); + let score = score_path_relevance("app/widget/nested/main.ts", "widget", &empty, false); assert!(score >= 3.0); } #[test] fn score_path_relevance_empty_query_is_zero() { let empty = HashSet::new(); - assert_eq!(score_path_relevance("src/a.ts", "", &empty), 0.0); - assert_eq!(score_path_relevance("src/a.ts", " ", &empty), 0.0); + assert_eq!(score_path_relevance("src/a.ts", "", &empty, false), 0.0); + assert_eq!(score_path_relevance("src/a.ts", " ", &empty, false), 0.0); } #[test] fn score_path_relevance_test_file_penalized_unless_test_query() { let empty = HashSet::new(); // Non-test query on a test file → -15 penalty pushes it negative. - let penalized = score_path_relevance("src/tests/foo.rs", "foo", &empty); + let penalized = score_path_relevance("src/tests/foo.rs", "foo", &empty, false); assert!(penalized < 10.0, "test file penalized: {penalized}"); // Test query on a test file → no penalty. - let ok = score_path_relevance("src/tests/foo.rs", "test foo", &empty); + let ok = score_path_relevance("src/tests/foo.rs", "test foo", &empty, false); assert!(ok >= penalized); + let configured = score_path_relevance("src/tests/foo.rs", "test foo", &empty, true); + assert_eq!(configured, ok - 15.0); } #[test] fn score_path_relevance_project_tokens_filtered_out() { // If every query word is a project token, fall back to all_words. let project = tokens(&["myproj"]); - let s = score_path_relevance("src/myproj/login.ts", "myproj login", &project); + let s = score_path_relevance("src/myproj/login.ts", "myproj login", &project, false); assert!(s > 0.0); } diff --git a/crates/codegraph-cli/src/segment_match.rs b/crates/codegraph-graph/src/segment_match.rs similarity index 98% rename from crates/codegraph-cli/src/segment_match.rs rename to crates/codegraph-graph/src/segment_match.rs index 6a1faf9a..a30bf721 100644 --- a/crates/codegraph-cli/src/segment_match.rs +++ b/crates/codegraph-graph/src/segment_match.rs @@ -1,4 +1,5 @@ -//! Query-time graph-derived segment matching for the prompt-hook MEDIUM tier. +//! Query-time graph-derived segment matching for explore retrieval and the +//! prompt-hook MEDIUM tier. //! //! Rust port of upstream `CodeGraph.getSegmentMatches` (`src/index.ts:908-1030`, //! #1136 `e699ee9` + #1144/#1145/#1146 hardening from `35611b9`). Instead of diff --git a/crates/codegraph-cli/src/segments.rs b/crates/codegraph-graph/src/segments.rs similarity index 90% rename from crates/codegraph-cli/src/segments.rs rename to crates/codegraph-graph/src/segments.rs index 0e361cfd..3d70e0a6 100644 --- a/crates/codegraph-cli/src/segments.rs +++ b/crates/codegraph-graph/src/segments.rs @@ -1,4 +1,5 @@ -//! Identifier-segment utilities for the prompt hook's graph-derived MEDIUM tier. +//! Identifier-segment utilities shared by graph search and the prompt hook's +//! graph-derived MEDIUM tier. //! //! Rust port of the PURE functions from upstream `src/search/identifier-segments.ts` //! (#1136 `e699ee9` + #1145 plural-folding correctness from `35611b9`). NO @@ -375,6 +376,35 @@ pub fn extract_prose_candidates(prompt: &str) -> Vec { out } +/// Search-query words for the segment matcher: prose candidates plus the +/// constituent words of camelCase/PascalCase tokens. FTS stores a camelCase +/// identifier as one token, so `atBottom` must also seed `bottom`. +pub fn extract_segment_search_words(query: &str) -> Vec { + if query.is_empty() { + return Vec::new(); + } + let mut out = extract_prose_candidates(query); + let mut seen = out.iter().cloned().collect::>(); + let mut segments = Vec::new(); + for run_match in WORD_RUN_RE.find_iter(query) { + let run = run_match.as_str(); + let chars = run.chars().collect::>(); + if !chars + .windows(2) + .any(|pair| (pair[0].is_lowercase() || pair[0].is_numeric()) && pair[1].is_uppercase()) + { + continue; + } + segments.extend(split_identifier_segments(run)); + } + for word in extract_prose_candidates(&segments.join(" ")) { + if seen.insert(word.clone()) { + out.push(word); + } + } + out +} + /// Lookup variants for a prose word: the word itself plus light English-plural /// folding, so common plurals still hit their singular segment. Keyed on /// English plural spelling (#1145), in three classes: @@ -481,4 +511,11 @@ mod tests { assert!(cands.contains(&"state".to_string())); assert!(cands.contains(&"machine".to_string())); } + + #[test] + fn search_words_add_camel_segments() { + let words = extract_segment_search_words("where is atBottom reset"); + assert!(words.contains(&"bottom".to_string())); + assert!(words.contains(&"reset".to_string())); + } } diff --git a/crates/codegraph-graph/tests/search.rs b/crates/codegraph-graph/tests/search.rs index cdac3b45..9518ec2e 100644 --- a/crates/codegraph-graph/tests/search.rs +++ b/crates/codegraph-graph/tests/search.rs @@ -1,5 +1,7 @@ use std::collections::HashSet; +use std::sync::Arc; +use codegraph_core::deprioritize::DeprioritizeMatcher; use codegraph_core::types::{Language, Node, NodeKind}; use codegraph_graph::query::{SearchOptions, parse_query, search_nodes}; use codegraph_store::Store; @@ -285,6 +287,8 @@ fn options_from_golden(opts: &GoldenOptions) -> SearchOptions { languages: Vec::new(), limit: opts.limit, offset: None, + seed_names: Vec::new(), + deprioritize: None, } } @@ -624,3 +628,139 @@ fn search_multiple_exact_matches_preserve_score_order_deterministically() { "limit must retain exact rows in score order" ); } + +#[test] +fn search_deprioritize_damps_ranking_without_hiding_results() { + let mut store = Store::open(&temp_db_path("deprioritize-ranking")).expect("open temp store"); + store + .upsert_nodes(&[ + node( + "function:product", + NodeKind::Function, + "usage", + "product::usage", + "src/product/usage.rs", + Language::Rust, + 1, + 2, + None, + None, + true, + ), + node( + "function:vendor", + NodeKind::Function, + "usage", + "vendor::usage", + "vendor/sdk/usage.rs", + Language::Rust, + 1, + 2, + None, + None, + true, + ), + node( + "function:prefix", + NodeKind::Function, + "usageHelper", + "product::usageHelper", + "src/product/usage_helper.rs", + Language::Rust, + 1, + 2, + None, + None, + true, + ), + ]) + .expect("insert ranking corpus"); + let opts = SearchOptions { + deprioritize: Some(Arc::new(DeprioritizeMatcher::from_patterns([ + "vendor/**".to_string() + ]))), + ..SearchOptions::default() + }; + let results = search_nodes(&store, "usage", &opts, &HashSet::new()).unwrap(); + assert_eq!(results.len(), 3, "deprioritized source remains findable"); + assert_eq!(results[0].node.id, "function:product"); + assert_eq!(results[1].node.id, "function:vendor"); + assert_eq!(results[2].node.id, "function:prefix"); + assert!( + (results[0].score - results[1].score - 35.0).abs() < f64::EPSILON, + "-15 path penalty plus 0.75x exact-name bonus must produce a 35-point discount" + ); + assert!( + results[1].score > results[2].score, + "discount must not erase an exact match behind a mere prefix" + ); +} + +#[test] +fn search_accepts_variable_and_constant_seed_names() { + let mut store = Store::open(&temp_db_path("camel-variable-seed")).expect("open temp store"); + let mut variable = node( + "variable:bottom", + NodeKind::Variable, + "feedAtBottom", + "feedAtBottom", + "src/feed.ts", + Language::TypeScript, + 1, + 1, + None, + None, + false, + ); + variable.signature = Some("let feedAtBottom = false".to_string()); + let constant = node( + "constant:pin", + NodeKind::Constant, + "PIN_FEED_IF_NEAR_BOTTOM", + "PIN_FEED_IF_NEAR_BOTTOM", + "src/feed.ts", + Language::TypeScript, + 2, + 2, + None, + None, + false, + ); + store + .upsert_nodes(&[variable, constant]) + .expect("insert camel corpus"); + + let results = search_nodes( + &store, + "atBottom", + &SearchOptions { + seed_names: vec!["feedAtBottom".to_string()], + ..SearchOptions::default() + }, + &HashSet::new(), + ) + .unwrap(); + assert!( + results + .iter() + .any(|result| result.node.id == "variable:bottom"), + "multi-hump variable must be a seed candidate" + ); + + let results = search_nodes( + &store, + "pinFeedNearBottom", + &SearchOptions { + seed_names: vec!["PIN_FEED_IF_NEAR_BOTTOM".to_string()], + ..SearchOptions::default() + }, + &HashSet::new(), + ) + .unwrap(); + assert!( + results + .iter() + .any(|result| result.node.id == "constant:pin"), + "multi-hump constant must be a seed candidate" + ); +} diff --git a/crates/codegraph-mcp/src/engine.rs b/crates/codegraph-mcp/src/engine.rs index f5d07990..1e412130 100644 --- a/crates/codegraph-mcp/src/engine.rs +++ b/crates/codegraph-mcp/src/engine.rs @@ -14,17 +14,21 @@ use std::sync::Arc; use std::time::{Duration, Instant, UNIX_EPOCH}; use codegraph_core::config::Config; +use codegraph_core::deprioritize::DeprioritizeMatcher; use codegraph_core::file_class::{is_generated_file, is_test_file}; use codegraph_core::node_id::hash_content; use codegraph_core::types::{EdgeKind, FileRecord, Node, NodeKind}; use codegraph_graph::graph::{GodotReach, GraphTraverser, NodeEdge}; use codegraph_graph::query::{SearchOptions, search_nodes}; +use codegraph_graph::segment_match::get_segment_matches; +use codegraph_graph::segments::extract_segment_search_words; use codegraph_store::Store; use serde_json::Value; use crate::dynamic_boundaries::scan_dynamic_dispatch; use crate::explore_budget::{ExploreOutputBudget, get_explore_output_budget}; use crate::protocol::ToolResult; +use crate::query_paths::{extract_query_paths, query_might_contain_paths}; /// Default caller/callee recursion depth for callers/callees tools. The upstream /// `getCallers`/`getCallees` default to `maxDepth: 1` (`traversal.ts` callers @@ -54,6 +58,16 @@ const POINTER_MAX_FILES: usize = 10; /// caller then drops the file whole rather than emitting a stub. const MIN_WINDOW_LINES: usize = 12; +/// Minimum source budget protected for every path explicitly named in an +/// explore query. Pinned files are ordered first and reserve this much for one +/// another before incidental files can spend the remaining envelope. +const PINNED_MIN_SOURCE_CHARS: usize = 1024; + +/// Bounded symbol admission for one explicitly pinned file. The file itself +/// remains pinned and source-renderable beyond this cap; only the auxiliary +/// graph-node gather is bounded. +const PINNED_FILE_NODE_CAP: usize = 300; + /// Fixed cost of `render_explore_file`'s section template around the header, the /// language tag and the cluster body: `"\n\n"` + ``"```"`` + `"\n"` + `"\n"` + /// ``"```"`` + `"\n"`. @@ -103,6 +117,10 @@ pub struct CodeGraphEngine { /// global HTTP process) therefore honors each project's own settings without /// reading a process-global value or another project's config. config: Arc, + /// Ranking-only project path policy. It is reloaded whenever the + /// request-scoped engine opens, so long-lived MCP processes observe + /// config mtime/size changes on the next search/explore call. + deprioritize: Arc, /// Request-local source/freshness memo. `execute` clears it before every tool /// call so repeated render passes stat and read each referenced file once. source_probes: RefCell>, @@ -124,12 +142,14 @@ impl CodeGraphEngine { std::env::var("CODEGRAPH_DIR").ok().as_deref(), )?; let config = Config::load_for_paths(None, &paths)?; + let deprioritize = Arc::new(DeprioritizeMatcher::load_for_paths(&paths, &config)); let store = Store::open_for_read(&paths, Instant::now() + Self::READ_LEASE_TIMEOUT, || false)?; Ok(Self { store, project_root: project_root.to_path_buf(), config, + deprioritize, source_probes: RefCell::new(HashMap::new()), source_citations: RefCell::new(HashSet::new()), }) @@ -380,6 +400,8 @@ impl CodeGraphEngine { languages: Vec::new(), limit: Some(limit), offset: None, + seed_names: Vec::new(), + deprioritize: Some(Arc::clone(&self.deprioritize)), }; let results = search_nodes(&self.store, &query, &options, &self.project_name_tokens())?; if results.is_empty() { @@ -922,13 +944,6 @@ impl CodeGraphEngine { Err(msg) => return Ok(ToolResult::error(msg)), }; - let subgraph = self.find_relevant_context(&query)?; - if subgraph.nodes.is_empty() { - return Ok(ToolResult::text(format!( - "No relevant code found for \"{query}\"" - ))); - } - // Resolve the adaptive output budget from the project's indexed file // count (`tools.ts:2024-2029`), falling back to the largest tier if // stats are unavailable. @@ -946,12 +961,39 @@ impl CodeGraphEngine { .unwrap_or(budget.default_max_files) .clamp(1, 20); - let file_count = subgraph - .nodes - .iter() - .map(|n| n.file_path.as_str()) - .collect::>() - .len(); + let path_extraction = if query_might_contain_paths(&query) { + let indexed_paths = self + .store + .all_files()? + .into_iter() + .map(|file| file.path) + .collect::>(); + extract_query_paths(&query, &indexed_paths, max_files.min(8)) + } else { + extract_query_paths(&query, &[], max_files.min(8)) + }; + let match_query = path_extraction.stripped_query.trim().to_string(); + let subgraph = + self.find_relevant_context_with_pins(&match_query, &path_extraction.pinned_files)?; + let unresolved_note = (!path_extraction.unresolved_path_spans.is_empty()).then(|| { + path_extraction + .unresolved_path_spans + .iter() + .map(|span| format!("`{span}`")) + .collect::>() + .join(", ") + }); + if subgraph.nodes.is_empty() && subgraph.file_order.is_empty() { + let miss_note = unresolved_note + .as_ref() + .map(|spans| format!(" (no indexed file uniquely matches {spans})")) + .unwrap_or_default(); + return Ok(ToolResult::text(format!( + "No relevant code found for \"{query}\"{miss_note}" + ))); + } + + let file_count = subgraph.file_order.len(); let mut lines = vec![ format!("## Exploration: {query}"), @@ -960,8 +1002,22 @@ impl CodeGraphEngine { "Found {} symbols across {file_count} files.", subgraph.nodes.len() ), - String::new(), ]; + if !path_extraction.pinned_files.is_empty() { + lines.push(format!( + "{} file{} pinned from the query.", + path_extraction.pinned_files.len(), + if path_extraction.pinned_files.len() == 1 { + "" + } else { + "s" + } + )); + } + if let Some(spans) = &unresolved_note { + lines.push(format!("No indexed file uniquely matches {spans}.")); + } + lines.push(String::new()); if let Some(blast) = self.build_blast_radius(&subgraph)? { lines.push(blast); @@ -991,10 +1047,10 @@ impl CodeGraphEngine { // icon/i18n files unless the query mentions tests, and only when >=2 // non-low-value files remain (else tests are the only signal here). let mut file_order: Vec<&String> = subgraph.file_order.iter().collect(); - if budget.exclude_low_value_files && !query_mentions_tests(&query) { + if budget.exclude_low_value_files && !query_mentions_tests(&match_query) { let non_low: Vec<&String> = file_order .iter() - .filter(|f| !is_low_value_file(f)) + .filter(|f| !is_low_value_file(f) || subgraph.is_pinned(f)) .copied() .collect(); if non_low.len() >= 2 { @@ -1012,7 +1068,7 @@ impl CodeGraphEngine { let mut any_file_trimmed = false; let mut excluded_files: Vec<&String> = Vec::new(); let mut rendered_sources: Vec<(String, usize)> = Vec::new(); - let precise_tokens = precise_query_tokens(&query); + let precise_tokens = precise_query_tokens(&match_query); let indexed_file_count = self.store.counts().ok().map(|c| c.file_count); let effective_budget = budget .max_output_chars @@ -1023,7 +1079,7 @@ impl CodeGraphEngine { indexed_file_count, )); - for file_path in &file_order { + for (file_index, file_path) in file_order.iter().enumerate() { if files_included >= max_files { excluded_files.push(file_path); continue; @@ -1038,11 +1094,21 @@ impl CodeGraphEngine { let lang = subgraph.file_language(file_path); let focuses = subgraph.file_focus_lines(file_path, &precise_tokens); + let available = effective_budget + .saturating_sub(total_chars) + .saturating_sub(1); + let pending_pins = file_order[file_index + 1..] + .iter() + .filter(|path| subgraph.is_pinned(path)) + .count(); + let reserved_for_pins = pending_pins.saturating_mul(PINNED_MIN_SOURCE_CHARS); + let mut funded_headroom = available.saturating_sub(reserved_for_pins); + if subgraph.is_pinned(file_path) { + funded_headroom = funded_headroom.max(PINNED_MIN_SOURCE_CHARS.min(available)); + } let ctx = RenderCtx { budget: &budget, - funded_headroom: effective_budget - .saturating_sub(total_chars) - .saturating_sub(1), + funded_headroom, focuses: &focuses, drifted: possibly_drifted, }; @@ -1951,26 +2017,64 @@ impl CodeGraphEngine { /// callers/callees + `contains` children into the subgraph so the blast /// radius, relationship map, and source section have content. The RWR /// relevance re-ranking / file gating is NOT ported (see KNOWN_DIFFS.md). + #[cfg(test)] fn find_relevant_context(&self, query: &str) -> anyhow::Result { + self.find_relevant_context_with_pins(query, &[]) + } + + fn find_relevant_context_with_pins( + &self, + query: &str, + pinned_files: &[String], + ) -> anyhow::Result { let mut sub = ExploreSubgraph::default(); let traverser = GraphTraverser::new(&self.store); let mut seed_ids: Vec = Vec::new(); - let results = search_nodes( - &self.store, - query, - &SearchOptions { - limit: Some(8), - ..Default::default() - }, - &self.project_name_tokens(), - )?; + let results = if query.trim().is_empty() { + Vec::new() + } else { + let seed_names = + get_segment_matches(&self.store, &extract_segment_search_words(query), 8) + .into_iter() + .map(|matched| matched.name) + .collect(); + search_nodes( + &self.store, + query, + &SearchOptions { + limit: Some(8), + seed_names, + deprioritize: Some(Arc::clone(&self.deprioritize)), + ..Default::default() + }, + &self.project_name_tokens(), + )? + }; for r in results { if sub.insert(r.node.clone()) { seed_ids.push(r.node.id.clone()); sub.roots.push(r.node.id.clone()); } } + for file_path in pinned_files { + sub.pin_file(file_path); + let mut file_nodes = self.store.nodes_by_file_path(file_path)?; + file_nodes.sort_by(|a, b| { + (a.start_line, a.start_column, a.id.as_str()).cmp(&( + b.start_line, + b.start_column, + b.id.as_str(), + )) + }); + for node in file_nodes + .into_iter() + .filter(|node| !matches!(node.kind, NodeKind::File | NodeKind::Import)) + .take(PINNED_FILE_NODE_CAP) + { + sub.insert(node); + } + } for root_id in seed_ids.clone() { for c in traverser.get_callers(&root_id, CALL_DEPTH)? { @@ -1996,10 +2100,16 @@ impl CodeGraphEngine { .iter() .map(|n| n.file_path.clone()) .collect::>(); + candidate_paths.extend(pinned_files.iter().cloned()); candidate_paths.sort_unstable(); candidate_paths.dedup(); sub.generated_files = self.store.generated_paths_among(&candidate_paths)?; sub.ambient_files = self.ambient_declaration_files(&candidate_paths, query)?; + sub.deprioritized_files = candidate_paths + .iter() + .filter(|path| self.deprioritize.is_match(path)) + .cloned() + .collect(); sub.finalize(); Ok(sub) @@ -2573,6 +2683,9 @@ struct ExploreSubgraph { edge_seen: HashSet<(String, String, String)>, roots: Vec, file_order: Vec, + /// Query-named paths in appearance order. They remain ahead of every + /// inferred file and are exempt from low-value filtering. + pinned_files: Vec, /// Files rescued by the #1064 change-surface pass: a tiered callable seed's /// buried signature-type file. `finalize` floats these to the TOP tier so a /// lexically-dissimilar answer file (grpc's `dialoptions.go`) is not buried @@ -2594,9 +2707,25 @@ struct ExploreSubgraph { /// accounts for the exemption, which is why the same file can be damped for a /// prose query and undamped for one naming its type. ambient_files: HashSet, + /// Project-configured ranking-only paths. They stay in the subgraph and + /// source output, but lose same-tier ordering ties to first-party files. + deprioritized_files: HashSet, } impl ExploreSubgraph { + fn pin_file(&mut self, file_path: &str) { + if !self.pinned_files.iter().any(|path| path == file_path) { + self.pinned_files.push(file_path.to_string()); + } + if !self.file_order.iter().any(|path| path == file_path) { + self.file_order.push(file_path.to_string()); + } + } + + fn is_pinned(&self, file_path: &str) -> bool { + self.pinned_files.iter().any(|path| path == file_path) + } + fn insert(&mut self, node: Node) -> bool { if self.index.contains_key(&node.id) { return false; @@ -2653,7 +2782,7 @@ impl ExploreSubgraph { neighbor_files.insert(n.file_path.as_str()); } } - let scores: HashMap = self + let scores: HashMap = self .file_order .iter() .map(|fp| { @@ -2702,11 +2831,24 @@ impl ExploreSubgraph { .iter() .filter(|n| &n.file_path == fp && n.kind != NodeKind::File) .count(); - (fp.clone(), (tier, weight, sym_count)) + let first_party = u8::from(!self.deprioritized_files.contains(fp)); + (fp.clone(), (tier, first_party, weight, sym_count)) }) .collect(); - self.file_order - .sort_by(|a, b| scores[b].cmp(&scores[a]).then_with(|| a.cmp(b))); + let pin_order = self + .pinned_files + .iter() + .enumerate() + .map(|(index, path)| (path.as_str(), index)) + .collect::>(); + self.file_order.sort_by(|a, b| { + match (pin_order.get(a.as_str()), pin_order.get(b.as_str())) { + (Some(a_index), Some(b_index)) => a_index.cmp(b_index), + (Some(_), None) => std::cmp::Ordering::Less, + (None, Some(_)) => std::cmp::Ordering::Greater, + (None, None) => scores[b].cmp(&scores[a]).then_with(|| a.cmp(b)), + } + }); } fn file_language(&self, file_path: &str) -> String { @@ -3749,6 +3891,7 @@ mod tests { store, project_root: base, config: Arc::new(Config::default()), + deprioritize: Arc::new(DeprioritizeMatcher::default()), source_probes: RefCell::new(HashMap::new()), source_citations: RefCell::new(HashSet::new()), } @@ -5444,6 +5587,166 @@ mod tests { assert!(text_of(&tr).contains("No relevant code found for")); } + #[test] + fn ext_explore_pins_explicit_path_and_strips_path_shrapnel() { + let mut engine = test_engine(); + let target = "src/routes/m/projects/[id]/runs/[runId]/+page.ts"; + put_indexed_source( + &engine, + target, + "const feedAtBottom = true;\nfunction pinFeedIfNearBottom() {}\n", + Language::TypeScript, + 2, + ); + put_indexed_source( + &engine, + "src/lib/runs-store.ts", + "function runId() {}\n", + Language::TypeScript, + 1, + ); + put_nodes( + &mut engine, + &[ + node_lang( + "feedAtBottom", + "feedAtBottom", + target, + 1, + 1, + NodeKind::Variable, + Language::TypeScript, + ), + node_lang( + "runId", + "runId", + "src/lib/runs-store.ts", + 1, + 1, + NodeKind::Function, + Language::TypeScript, + ), + ], + ); + + let text = text_of(&engine.execute( + "codegraph_explore", + &serde_json::json!({"query": target, "maxFiles": 1}), + )); + assert!( + text.contains("1 file pinned from the query."), + "got: {text}" + ); + assert!(text.contains(target), "got: {text}"); + assert!(text.contains("feedAtBottom"), "got: {text}"); + assert!( + !text.contains("src/lib/runs-store.ts"), + "path fragments must not seed the decoy: {text}" + ); + } + + #[test] + fn ext_explore_camel_segment_seeds_variables_and_constants() { + let mut engine = test_engine(); + put_indexed_source( + &engine, + "src/feed.ts", + "let feedAtBottom = false;\nconst PIN_FEED_IF_NEAR_BOTTOM = true;\n", + Language::TypeScript, + 2, + ); + put_nodes( + &mut engine, + &[ + node_lang( + "feedAtBottom", + "feedAtBottom", + "src/feed.ts", + 1, + 1, + NodeKind::Variable, + Language::TypeScript, + ), + node_lang( + "PIN_FEED_IF_NEAR_BOTTOM", + "PIN_FEED_IF_NEAR_BOTTOM", + "src/feed.ts", + 2, + 2, + NodeKind::Constant, + Language::TypeScript, + ), + ], + ); + + let text = text_of(&engine.execute( + "codegraph_explore", + &serde_json::json!({"query": "feed bottom behavior"}), + )); + assert!(text.contains("feedAtBottom"), "got: {text}"); + assert!(text.contains("PIN_FEED_IF_NEAR_BOTTOM"), "got: {text}"); + } + + #[test] + fn ext_explore_pinned_low_value_file_wins_max_files_and_reports_misses() { + let mut engine = test_engine(); + put_indexed_source( + &engine, + "tests/target.rs", + "fn only_in_target() {}\n", + Language::Rust, + 1, + ); + put_indexed_source( + &engine, + "src/scroll.rs", + "fn scroll() {}\n", + Language::Rust, + 1, + ); + put_nodes( + &mut engine, + &[ + node_lang( + "target", + "only_in_target", + "tests/target.rs", + 1, + 1, + NodeKind::Function, + Language::Rust, + ), + node_lang( + "scroll", + "scroll", + "src/scroll.rs", + 1, + 1, + NodeKind::Function, + Language::Rust, + ), + ], + ); + let pinned = text_of(&engine.execute( + "codegraph_explore", + &serde_json::json!({"query": "scroll tests/target.rs", "maxFiles": 1}), + )); + assert!(pinned.contains("tests/target.rs"), "got: {pinned}"); + assert!( + !pinned.contains("#### `src/scroll.rs`"), + "pinned file must own the sole source slot: {pinned}" + ); + + let missing = text_of(&engine.execute( + "codegraph_explore", + &serde_json::json!({"query": "crash in src/gone/missing.rs"}), + )); + assert!( + missing.contains("No indexed file uniquely matches `src/gone/missing.rs`"), + "got: {missing}" + ); + } + #[test] fn ext_explore_full_render_with_blast_relationships_and_source() { let mut engine = test_engine(); @@ -6926,6 +7229,77 @@ mod tests { let _ = std::fs::remove_dir_all(&base); } + #[test] + fn ext_request_scoped_open_reloads_deprioritize_config() { + let base = std::env::temp_dir().join(format!( + "cg-mcp-ranking-reload-{}-{}", + std::process::id(), + TEMP_SEQ.fetch_add(1, Ordering::Relaxed) + )); + std::fs::create_dir_all(&base).unwrap(); + let paths = codegraph_core::IndexPaths::resolve(&base, None).unwrap(); + std::fs::create_dir_all(paths.current_root()).unwrap(); + { + let mut store = Store::open(&paths.current_db()).unwrap(); + store + .upsert_nodes(&[ + node_lang( + "usage", + "product::usage", + "src/product.rs", + 1, + 2, + NodeKind::Function, + Language::Rust, + ), + node_lang( + "usage", + "vendor::usage", + "vendor/sdk.rs", + 1, + 2, + NodeKind::Function, + Language::Rust, + ), + ]) + .unwrap(); + } + codegraph_store::test_support::finalize_current_test_fixture(&paths).unwrap(); + + std::fs::write( + paths.config_toml(), + "[app]\nname = \"ranking-reload\"\n\n[indexing]\ndeprioritize = [\"vendor/**\"]\n", + ) + .unwrap(); + let first = CodeGraphEngine::open(&base) + .unwrap() + .execute("codegraph_search", &serde_json::json!({"query": "usage"})); + let first = text_of(&first); + assert!( + first.find("src/product.rs").unwrap() < first.find("vendor/sdk.rs").unwrap(), + "vendor rule should rank product first: {first}" + ); + + // A long-lived server opens a request-scoped engine for every tool + // call. Rewriting the project config must therefore affect the next + // request in the same process without restarting or rebuilding. + std::fs::write( + paths.config_toml(), + "[app]\nname = \"ranking-reload\"\n\n[indexing]\ndeprioritize = [\"src/**\"]\n", + ) + .unwrap(); + let second = CodeGraphEngine::open(&base) + .unwrap() + .execute("codegraph_search", &serde_json::json!({"query": "usage"})); + let second = text_of(&second); + assert!( + second.find("vendor/sdk.rs").unwrap() < second.find("src/product.rs").unwrap(), + "reloaded src rule should rank vendor first: {second}" + ); + + let _ = std::fs::remove_dir_all(&base); + } + #[test] fn ext_check_reports_cycle() { let mut engine = test_engine(); diff --git a/crates/codegraph-mcp/src/lib.rs b/crates/codegraph-mcp/src/lib.rs index ce70e5c0..8a03f4df 100644 --- a/crates/codegraph-mcp/src/lib.rs +++ b/crates/codegraph-mcp/src/lib.rs @@ -15,6 +15,7 @@ pub mod engine; pub mod explore_budget; pub mod instructions; pub mod protocol; +pub(crate) mod query_paths; pub mod rmcp_handler; pub mod rmcp_session; pub(crate) mod roots; diff --git a/crates/codegraph-mcp/src/query_paths.rs b/crates/codegraph-mcp/src/query_paths.rs new file mode 100644 index 00000000..46668b52 --- /dev/null +++ b/crates/codegraph-mcp/src/query_paths.rs @@ -0,0 +1,391 @@ +//! Pure file-path recognition for `codegraph_explore` queries. +//! +//! Explicit paths are resolved against the indexed file list, pinned, and +//! removed from the normal text query so bracketed routes and common basenames +//! cannot dissolve into noisy symbol seeds. + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::LazyLock; + +use regex::Regex; + +const MAX_PREFIX_DROPS: usize = 8; +const MAX_CANDIDATE_SPANS: usize = 8; +const MAX_PINS: usize = 8; +const MAX_UNRESOLVED: usize = 4; + +static DOTTED_BASENAME: LazyLock = LazyLock::new(|| { + Regex::new(r"^[^\s/\\]+\.[A-Za-z][A-Za-z0-9]{0,7}$").expect("dotted basename regex is valid") +}); +static KEBAB_BASENAME: LazyLock = LazyLock::new(|| { + Regex::new(r"^[A-Za-z0-9]+(?:-[A-Za-z0-9]+)+$").expect("kebab basename regex is valid") +}); +static LINE_REFERENCE: LazyLock = LazyLock::new(|| { + Regex::new(r"(?::\d+(?:-\d+)?|#L\d+(?:-L?\d+)?)$").expect("line-reference regex is valid") +}); +static LAST_EXTENSION: LazyLock = LazyLock::new(|| { + Regex::new(r"\.[A-Za-z][A-Za-z0-9]{0,7}$").expect("last-extension regex is valid") +}); + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct QueryPathExtraction { + pub stripped_query: String, + pub pinned_files: Vec, + pub unresolved_path_spans: Vec, +} + +pub(crate) fn query_might_contain_paths(query: &str) -> bool { + query.split_whitespace().any(|token| { + let stripped = strip_wrapping(token); + stripped.contains(['/', '\\']) + || DOTTED_BASENAME.is_match(&stripped) + || KEBAB_BASENAME.is_match(&stripped) + }) +} + +pub(crate) fn extract_query_paths( + query: &str, + indexed_paths: &[String], + max_pins: usize, +) -> QueryPathExtraction { + let passthrough = || QueryPathExtraction { + stripped_query: query.to_string(), + pinned_files: Vec::new(), + unresolved_path_spans: Vec::new(), + }; + if query.trim().is_empty() || indexed_paths.is_empty() { + return passthrough(); + } + let max_pins = max_pins.clamp(1, MAX_PINS); + let lower_to_original = indexed_paths + .iter() + .map(|path| (path.to_lowercase(), path.clone())) + .collect::>(); + let tokens = query + .split_whitespace() + .map(str::to_string) + .collect::>(); + let mut consumed = BTreeSet::new(); + let mut pinned = Vec::new(); + let mut pinned_seen = BTreeSet::new(); + let mut unresolved = Vec::new(); + let mut candidates_examined = 0usize; + + for (index, token) in tokens.iter().enumerate() { + if pinned.len() >= max_pins || candidates_examined >= MAX_CANDIDATE_SPANS { + break; + } + let stripped = strip_wrapping(token); + if stripped.chars().count() < 4 { + continue; + } + let has_slash = stripped.contains(['/', '\\']); + if !has_slash && !DOTTED_BASENAME.is_match(&stripped) { + continue; + } + let normalized = normalize_span(&stripped); + if normalized.is_empty() { + continue; + } + candidates_examined += 1; + let resolved = resolve_span(&normalized.to_lowercase(), &lower_to_original, 3); + if !resolved.matches.is_empty() { + consumed.insert(index); + for path in resolved.matches { + if pinned.len() >= max_pins { + break; + } + if pinned_seen.insert(path.clone()) { + pinned.push(path); + } + } + } else if resolved.ambiguous || is_clearly_path_shaped(&normalized) { + consumed.insert(index); + if unresolved.len() < MAX_UNRESOLVED { + unresolved.push(normalized); + } + } + } + + // Extensionless kebab basenames run after explicit paths, which therefore + // own the shared pin budget. + let stems = build_basename_stems(indexed_paths); + for (index, token) in tokens.iter().enumerate() { + if pinned.len() >= max_pins || candidates_examined >= MAX_CANDIDATE_SPANS { + break; + } + if consumed.contains(&index) { + continue; + } + let stripped = strip_wrapping(token); + if stripped.chars().count() < 4 || !KEBAB_BASENAME.is_match(&stripped) { + continue; + } + candidates_examined += 1; + let Some(matches) = stems.get(&stripped.to_lowercase()) else { + continue; + }; + if matches.len() > 3 { + continue; + } + consumed.insert(index); + for path in matches { + if pinned.len() >= max_pins { + break; + } + if pinned_seen.insert(path.clone()) { + pinned.push(path.clone()); + } + } + } + + if consumed.is_empty() { + return passthrough(); + } + QueryPathExtraction { + stripped_query: tokens + .into_iter() + .enumerate() + .filter_map(|(index, token)| (!consumed.contains(&index)).then_some(token)) + .collect::>() + .join(" "), + pinned_files: pinned, + unresolved_path_spans: unresolved, + } +} + +struct SpanResolution { + matches: Vec, + ambiguous: bool, +} + +fn resolve_span( + normalized_lower: &str, + lower_to_original: &BTreeMap, + max_matches: usize, +) -> SpanResolution { + if let Some(exact) = lower_to_original.get(normalized_lower) { + return SpanResolution { + matches: vec![exact.clone()], + ambiguous: false, + }; + } + let segments = normalized_lower + .split('/') + .filter(|segment| !segment.is_empty()) + .collect::>(); + let max_drop = MAX_PREFIX_DROPS.min(segments.len().saturating_sub(1)); + for drop_count in 0..=max_drop { + let suffix = segments[drop_count..].join("/"); + if suffix.is_empty() { + break; + } + let with_slash = format!("/{suffix}"); + let mut matches = Vec::new(); + for (lower, original) in lower_to_original { + if lower == &suffix || lower.ends_with(&with_slash) { + matches.push(original.clone()); + if matches.len() > max_matches { + return SpanResolution { + matches: Vec::new(), + ambiguous: true, + }; + } + } + } + if !matches.is_empty() { + return SpanResolution { + matches, + ambiguous: false, + }; + } + } + SpanResolution { + matches: Vec::new(), + ambiguous: false, + } +} + +fn build_basename_stems(indexed_paths: &[String]) -> BTreeMap> { + let mut stems = BTreeMap::>::new(); + for path in indexed_paths { + let basename = path.rsplit(['/', '\\']).next().unwrap_or(path.as_str()); + if !basename.contains('-') { + continue; + } + let stem = LAST_EXTENSION.replace(basename, "").to_lowercase(); + if stem.is_empty() { + continue; + } + stems.entry(stem).or_default().push(path.clone()); + } + stems +} + +fn strip_wrapping(token: &str) -> String { + let mut value = token.to_string(); + while let Some(first) = value.chars().next() { + let strip = matches!(first, '\'' | '"' | '`' | '<') + || (first == '(' && !value.contains(')')) + || (first == '[' && !value.contains(']')) + || (first == '{' && !value.contains('}')); + if !strip { + break; + } + value.remove(0); + } + while let Some(last) = value.chars().last() { + let strip = matches!(last, '\'' | '"' | '`' | '>' | '.' | ',' | ';' | '!' | '?') + || (last == ')' && !value.contains('(')) + || (last == ']' && !value.contains('[')) + || (last == '}' && !value.contains('{')); + if !strip { + break; + } + value.pop(); + } + LINE_REFERENCE.replace(&value, "").to_string() +} + +fn normalize_span(span: &str) -> String { + let mut normalized = span.replace('\\', "/"); + while let Some(rest) = normalized.strip_prefix("./") { + normalized = rest.to_string(); + } + while normalized.contains("//") { + normalized = normalized.replace("//", "/"); + } + normalized.trim_end_matches('/').to_string() +} + +fn is_clearly_path_shaped(normalized: &str) -> bool { + let Some(slash) = normalized.rfind('/') else { + return false; + }; + slash > 0 && DOTTED_BASENAME.is_match(&normalized[slash + 1..]) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn index() -> Vec { + [ + "src/routes/m/projects/[id]/runs/[runId]/+page.svelte", + "src/routes/m/projects/[id]/chat/[scope]/+page.svelte", + "src/routes/m/projects/[id]/+page.svelte", + "src/routes/(protected)/chat-window/+page.svelte", + "src/lib/chat-manager.ts", + "src/lib/task-runner-manager.ts", + "src/components/training-set-page/training-set-page.tsx", + "src/components/training-set-page/training-set-page.module.scss", + "src/components/training-set-page/background-image-table.tsx", + "src/x/generic-modal.tsx", + "src/y/generic-modal.tsx", + "scripts/pre-commit", + "src/a/user-profile.tsx", + "src/b/user-profile.tsx", + "src/c/user-profile.tsx", + "src/d/user-profile.tsx", + ] + .into_iter() + .map(str::to_string) + .collect() + } + + #[test] + fn gate_distinguishes_paths_from_plain_prose_and_flags() { + assert!(query_might_contain_paths("see src/lib/chat-manager.ts")); + assert!(query_might_contain_paths("see background-image-table")); + assert!(!query_might_contain_paths("how does scroll pinning work")); + assert!(!query_might_contain_paths("use --no-cache")); + } + + #[test] + fn resolves_bracketed_absolute_windows_and_line_referenced_paths() { + let paths = index(); + let bracketed = extract_query_paths( + "scroll in src/routes/m/projects/[id]/runs/[runId]/+page.svelte atBottom", + &paths, + 8, + ); + assert_eq!( + bracketed.pinned_files, + vec!["src/routes/m/projects/[id]/runs/[runId]/+page.svelte"] + ); + assert_eq!(bracketed.stripped_query, "scroll in atBottom"); + + let absolute = + extract_query_paths(r#"fix C:\dev\repo\src\lib\chat-manager.ts:243"#, &paths, 8); + assert_eq!(absolute.pinned_files, vec!["src/lib/chat-manager.ts"]); + + let hash = extract_query_paths("see src/lib/task-runner-manager.ts#L88-L120", &paths, 8); + assert_eq!(hash.pinned_files, vec!["src/lib/task-runner-manager.ts"]); + + let eight_prefixes = + extract_query_paths("see /a/b/c/d/e/f/g/h/src/lib/chat-manager.ts", &paths, 8); + assert_eq!( + eight_prefixes.pinned_files, + vec!["src/lib/chat-manager.ts"], + "exactly eight discarded prefix segments must stay within the bound" + ); + + let nine_prefixes = + extract_query_paths("see /a/b/c/d/e/f/g/h/i/src/lib/chat-manager.ts", &paths, 8); + assert!(nine_prefixes.pinned_files.is_empty()); + assert_eq!( + nine_prefixes.unresolved_path_spans, + vec!["/a/b/c/d/e/f/g/h/i/src/lib/chat-manager.ts"] + ); + } + + #[test] + fn reports_only_unambiguous_path_misses() { + let paths = index(); + let ambiguous = extract_query_paths("why all +page.svelte files flash", &paths, 8); + assert_eq!(ambiguous.unresolved_path_spans, vec!["+page.svelte"]); + assert_eq!(ambiguous.stripped_query, "why all files flash"); + + let missing = extract_query_paths( + "crash in src/routes/gone/missing-page.svelte on load", + &paths, + 8, + ); + assert_eq!( + missing.unresolved_path_spans, + vec!["src/routes/gone/missing-page.svelte"] + ); + assert_eq!(missing.stripped_query, "crash in on load"); + + let prose = "does gen_server:call/2 block and/or timeout"; + assert_eq!(extract_query_paths(prose, &paths, 8).stripped_query, prose); + } + + #[test] + fn kebab_basenames_are_bounded_and_explicit_paths_win() { + let paths = index(); + let one = extract_query_paths("background-image-table Source", &paths, 8); + assert_eq!( + one.pinned_files, + vec!["src/components/training-set-page/background-image-table.tsx"] + ); + assert_eq!(one.stripped_query, "Source"); + + let shared = extract_query_paths("generic-modal close", &paths, 8); + assert_eq!( + shared.pinned_files, + vec!["src/x/generic-modal.tsx", "src/y/generic-modal.tsx"] + ); + + let hot = "refactor user-profile rendering"; + assert_eq!(extract_query_paths(hot, &paths, 8).stripped_query, hot); + + let capped = extract_query_paths( + "background-image-table then src/lib/chat-manager.ts", + &paths, + 1, + ); + assert_eq!(capped.pinned_files, vec!["src/lib/chat-manager.ts"]); + assert_eq!(capped.stripped_query, "background-image-table then"); + } +} diff --git a/crates/codegraph-store/src/queries.rs b/crates/codegraph-store/src/queries.rs index 37c500bf..8aaee18c 100644 --- a/crates/codegraph-store/src/queries.rs +++ b/crates/codegraph-store/src/queries.rs @@ -239,12 +239,14 @@ impl Store { } /// Ports `getNodesByLowerName` from `upstream db/queries.ts:754-765`. - /// Callers pass `lowercase`; the SQL shape stays `lower(name) = ?` for `idx_nodes_lower_name`. - pub fn nodes_by_lower_name(&self, lower_name: &str) -> rusqlite::Result> { + /// The SQL expression mirrors `idx_nodes_lower_name`, so SQLite performs an + /// index seek. `lower(?)` keeps parameter folding on SQLite's ASCII rules + /// and does not require callers to lowercase first. + pub fn nodes_by_lower_name(&self, name: &str) -> rusqlite::Result> { query_nodes( &self.conn, - "SELECT * FROM nodes WHERE lower(name) = ?", - [lower_name], + "SELECT * FROM nodes WHERE lower(name) = lower(?)", + [name], ) } @@ -496,14 +498,16 @@ impl Store { } /// Ports the exact-name supplement query from `upstream db/queries.ts:834-845`. - /// `name = ? COLLATE NOCASE` plus optional kind/language `IN (...)`; `LIMIT 20`. + /// `lower(name) = lower(?)` plus optional kind/language `IN (...)`; + /// `LIMIT 20`. The matching expression pins the planner to + /// `idx_nodes_lower_name`, including misses. pub fn nodes_by_exact_name_nocase( &self, name: &str, kinds: &[NodeKind], languages: &[Language], ) -> rusqlite::Result> { - let mut sql = String::from("SELECT * FROM nodes WHERE name = ? COLLATE NOCASE"); + let mut sql = String::from("SELECT * FROM nodes WHERE lower(name) = lower(?)"); let mut params: Vec> = vec![Box::new(name.to_string())]; append_kind_language_filters(&mut sql, &mut params, "kind", "language", kinds, languages); // upstream db/queries.ts:844 — LIMIT 20. @@ -533,10 +537,9 @@ impl Store { rows.collect() } - /// Callable definitions whose name CONTAINS `needle` as a substring, fetched - /// as a SEPARATE batch from the FTS/LIKE ladder (upstream `1de7e8f` #1319 - /// "kind whitelist" — hot single-word terms must not crowd definers out of - /// the length-ordered batch). + /// Callable/variable/constant definitions whose name CONTAINS `needle` as a + /// substring, fetched as a SEPARATE batch from the FTS/LIKE ladder + /// (upstream `1de7e8f` #1319 plus v1.6.0 variable seeding). /// /// `needle` is matched lowercase against both `lower(name)` and its /// separator-stripped form, so a `user_profile_id` query reaches a @@ -546,7 +549,7 @@ impl Store { /// an incidental namesake never becomes a match. `ORDER BY` is fully /// specified — shortest name first, then name/path/line — so the candidate /// order never depends on SQLite's incidental row order. - pub fn callable_nodes_by_name_infix( + pub fn seedable_nodes_by_name_infix( &self, needle: &str, limit: i64, @@ -555,7 +558,7 @@ impl Store { let mut stmt = self.conn.prepare( r#" SELECT * FROM nodes - WHERE kind IN ('function', 'method', 'component') + WHERE kind IN ('function', 'method', 'component', 'variable', 'constant') AND ( lower(name) LIKE ? OR replace(replace(replace(lower(name), '_', ''), '-', ''), '.', '') LIKE ? @@ -2573,12 +2576,14 @@ mod tests { } #[test] - fn callable_infix_returns_callables_only_shortest_first_and_folds_separators() { + fn seedable_infix_includes_variables_constants_and_folds_separators() { let mut store = store("callable-infix"); let mut cls = node("class:holder", "UserProfileIdHolder", "src/holder.rs"); cls.kind = NodeKind::Class; let mut var = node("variable:v", "userProfileId", "src/var.rs"); var.kind = NodeKind::Variable; + let mut constant = node("constant:c", "USER_PROFILE_ID", "src/constant.rs"); + constant.kind = NodeKind::Constant; store .upsert_nodes(&[ node("function:long", "updateUserProfileIdMapping", "src/b.rs"), @@ -2587,26 +2592,29 @@ mod tests { node("function:other", "unrelated", "src/d.rs"), cls, var, + constant, ]) .unwrap(); let hits = store - .callable_nodes_by_name_infix("userprofileid", 50) + .seedable_nodes_by_name_infix("userprofileid", 50) .unwrap(); let names: Vec<&str> = hits.iter().map(|n| n.name.as_str()).collect(); assert_eq!( names, vec![ + "userProfileId", "userProfileId2", + "USER_PROFILE_ID", "read_user_profile_id", "updateUserProfileIdMapping" ], - "callables only, shortest name first; the separator-stripped form matches snake_case" + "seedable kinds only, shortest name first; the separator-stripped form matches snake_case" ); assert!( store - .callable_nodes_by_name_infix("userprofileid", 1) + .seedable_nodes_by_name_infix("userprofileid", 1) .unwrap() .len() == 1, @@ -2614,7 +2622,7 @@ mod tests { ); assert!( store - .callable_nodes_by_name_infix("nosuchneedle", 50) + .seedable_nodes_by_name_infix("nosuchneedle", 50) .unwrap() .is_empty() ); @@ -2706,6 +2714,57 @@ mod tests { ); } + #[test] + fn exact_name_queries_seek_lower_name_index_for_hits_misses_and_filters() { + let mut store = store("exact-name-plan"); + store + .upsert_nodes(&[ + node("function:x", "HandleRequest", "src/x.rs"), + node("function:y", "handlerequest", "src/y.rs"), + ]) + .unwrap(); + + let ids = store + .nodes_by_lower_name("HANDLEREQUEST") + .unwrap() + .into_iter() + .map(|node| node.id) + .collect::>(); + assert_eq!(ids.len(), 2, "caller casing must not affect lookup"); + + fn plan(store: &Store, sql: &str, params: &[&dyn ToSql]) -> String { + let explained = format!("EXPLAIN QUERY PLAN {sql}"); + let mut stmt = store.conn.prepare(&explained).unwrap(); + let details = stmt + .query_map(params, |row| row.get::<_, String>(3)) + .unwrap() + .collect::>>() + .unwrap(); + details.join(" | ") + } + + let miss = plan( + &store, + "SELECT * FROM nodes WHERE lower(name) = lower(?) LIMIT 20", + &[&"not-a-symbol"], + ); + assert!( + miss.contains("SEARCH nodes") && miss.contains("idx_nodes_lower_name"), + "miss regressed to a scan: {miss}" + ); + + let filtered = plan( + &store, + "SELECT * FROM nodes WHERE lower(name) = lower(?) \ + AND kind IN (?) AND language IN (?) LIMIT 20", + &[&"HandleRequest", &"function", &"rust"], + ); + assert!( + filtered.contains("SEARCH nodes") && filtered.contains("idx_nodes_lower_name"), + "filtered lookup regressed to a scan: {filtered}" + ); + } + #[test] fn parse_language_accepts_nix() { assert_eq!(parse_language("nix".to_string()).unwrap(), Language::Nix); diff --git a/docs/cli.md b/docs/cli.md index c198f808..8c89a0c2 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -1,14 +1,14 @@ # CLI Subcommand Reference -`codegraph` ships 22 subcommands. All commands accept `--help` for usage details. +`codegraph` ships 26 subcommands. All commands accept `--help` for usage details. ## Path Convention - **Positional or `-p/--path`:** `init`, `uninit`, `index`, `sync`, `status`, `callers`, `callees`, `impact`, `affected`, `unlock`, `check`, `export`. -- **`-p/--path` only:** `search`, `files`, `serve`, `audit`. +- **`-p/--path` only:** `search`, `files`, `serve`, `audit`, `explore`, `node`. - **No project path:** `install`, `uninstall`, `skill`, `version`, `self-update`, - `completions`. + `completions`, `http`, `mcp`. --- @@ -39,6 +39,10 @@ | `check` | Detect circular dependencies (each cycle as `a.ts -> b.ts -> a.ts`) | `[path]`, `-j/--json` | | `audit` | Read-only Godot resource audit: orphan resources, dangling references, impact | `-p`, `--orphans`, `--dangling`, `--impact ` (≥1 required), `--verify-plan`, `--include `, `--exclude `, `-j/--json` | | `export` | Export the whole code graph as NetworkX node-link JSON | `[path]`, `-o/--out `, `--no-centrality` | +| `explore` | Explore an area with the same deterministic engine/output as `codegraph_explore` | ``, `-p`, `--max-files <1..20>`, `-j/--json` | +| `node` | Read a symbol or indexed file with the same engine/output as `codegraph_node` | ``, `-p`, `-f/--file`, `--symbols-only`, `-j/--json`, `--strict` | +| `http` | Inspect or stop detached HTTP MCP servers | `` (`list`, `status`, `stop`) | +| `mcp` | Inspect foreground stdio MCP processes | `list`, optional `--json` | | `version` | Print the codegraph version (same as `--version`) | — | | `self-update` | Update the binary in place from the latest GitHub release | `--check`, `--force`, `--tag ` | | `completions` | Print or install shell completions | `` (bash, zsh, fish, powershell, elvish), `--install` | @@ -46,6 +50,11 @@ `query` remains a visible backward-compatible alias for `search`; new scripts, documentation, and diagnostics should use `search`. +Case-insensitive exact-name search probes seek through `idx_nodes_lower_name` +rather than scanning `nodes`. Explore separately supplements its context seeds +with camelCase/snake-case segments, including Variable and Constant definitions. +These are query-time changes only; they do not change the stored graph. + > **Note:** `serve --no-watch` and `CODEGRAPH_NO_WATCH=1` are fully equivalent — > both disable the live file watcher. See > [Daemon, watch & environment variables](#daemon-watch--environment-variables) @@ -944,6 +953,44 @@ Rules: so adding or removing an extension override takes effect without restarting the MCP server. +### Ranking-only path de-prioritization + +Keep peripheral source indexed and directly retrievable while ranking it below +first-party code: + +```toml +[app] +name = "my-project" + +[indexing] +deprioritize = ["vendor/**", "generated/**"] +``` + +The compatibility JSON form shares `.codegraph/codegraph.json` with extension +overrides: + +```jsonc +{ + "extensions": { + ".x": "lua", + }, + "deprioritize": ["vendor/**"], +} +``` + +Rules are project-root-relative, evaluated in order, and use last-match-wins; +prefix a rule with `!` to clear an earlier match. JSON rules are evaluated +first, then TOML rules, so TOML has final authority. Blank, invalid, or +non-string JSON rules are ignored. + +This policy affects only `search`/`explore` ranking: path relevance loses 15 +points and the exact-name bonus is multiplied by `0.75`. A file that is both +test-like and explicitly de-prioritized is docked once; a test-oriented query +may waive only the inferred test penalty, never an explicit rule. Files, nodes, +edges, exact path pins, and direct node/file reads are unchanged. MCP engines +reload the addressed project's config for every request, so a long-running +server observes edits on the next search/explore call. + ### Extraction-version upgrades After upgrading the binary, let diagnostics choose the least destructive path: diff --git a/docs/equivalence.md b/docs/equivalence.md index 26a49d52..09c4e9af 100644 --- a/docs/equivalence.md +++ b/docs/equivalence.md @@ -1173,6 +1173,22 @@ self-equivalence. As for every fixture below the Godot caveats, do not compare `colby.db` bytes; compare the four JSON artifacts byte-for-byte and compare `schema.sql` as a normalized statement set when statement order differs. +### Retrieval and ranking wave (golden-neutral) + +Explicit-path pinning in `codegraph_explore`, camel/segment explore seeding, +case-insensitive exact-name index seeks, and `[indexing].deprioritize` are +query/render-time behavior only. They do not modify extraction, resolution, +node IDs, files, nodes, edges, unresolved references, FTS schema, or any +committed golden artifact. This wave therefore keeps extraction version `12` +and has no golden regeneration recipe: a changed file under +`reference/golden/` is a regression, not expected output. + +Planner tests lock exact-name hit, miss, and filtered probes to +`idx_nodes_lower_name`; search/MCP tests cover Variable/Constant seeds, +last-match-wins ranking rules, explicit-path bounds, pinned-file budgeting, and +same-process config reload. The normal equivalence suite remains the authority +that every extraction golden is byte-stable. + ### KNOWN_DIFFS.md format Tier-3 differences are allowlisted by grep-able lines in diff --git a/docs/mcp.md b/docs/mcp.md index e143ed0a..c5d04bee 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -318,6 +318,23 @@ returns the symbols relevant to a query, their verbatim source grouped by file, plus the call/impact graph around them. Prefer it over individual `callers`/ `callees` chains when surveying an unfamiliar area. +Explicit source paths in the query are resolved before fuzzy search and pinned +to the front of the result. Quoted/backticked paths, `./`, Windows separators, +`:123`, and `#L12` are normalized; exact matches precede segment-aligned suffix +matches. Extensionless kebab basenames are accepted only when they resolve to at +most three indexed files, so ordinary hyphenated prose remains prose. The +resolver examines at most eight path spans, drops at most eight leading +segments, pins at most eight files (also bounded by `maxFiles`), and reports at +most four unresolved explicit paths. Resolved or clearly missing explicit paths +are removed from the normal query, preventing route parameters and basenames +from becoming noisy symbol seeds. Pinned files survive low-score filtering and +receive a protected source budget. + +Explore also resolves prose and camelCase query segments against indexed symbol +names, then merges the resulting callable, Variable, and Constant names as +dampened exact seeds. This recovers names such as `feedAtBottom` from "feed +bottom" while keeping ordinary `codegraph_search` on its fast SQL/FTS path. + The blast-radius block that `explore` prints reports **measured** test coverage. When no test file calls a root directly, codegraph walks UP the caller graph before saying anything — up to 3 hops (direct callers are hop 1), bounded by 64 @@ -383,6 +400,10 @@ and trigger one full scan that supersedes queued path deltas. Invalid TOML keeps the last valid scope and is reported; malformed JSON keeps the existing tolerant empty-override semantics. Every later incremental sync re-checks the current scope, preventing stale events from re-admitting excluded files. +Ranking-only `deprioritize` rules are loaded separately from watcher scope. +Each request-scoped engine loads the addressed project's JSON rules followed by +its authoritative TOML rules, so a long-lived stdio or HTTP process observes +edits on its next search/explore request without changing the graph. When the resolved root is exactly `$HOME` or the filesystem root (`/`), the server first disables the daemon, the file watcher, AND catch-up sync — not just the watcher. This happens when an IDE or agent (e.g. Kiro) launches diff --git a/docs/readme/README.zh-CN.md b/docs/readme/README.zh-CN.md index 78dd475d..61bc1126 100644 --- a/docs/readme/README.zh-CN.md +++ b/docs/readme/README.zh-CN.md @@ -119,7 +119,8 @@ codegraph install --yes # 检测已安装的代理并接线 **经验法则**：读文件**之前**先用 `codegraph_explore`；信任它的结果（完整 AST 解析， 别用 grep 复核）；重构影响半径用 `codegraph_impact` 而非手工遍历调用者。索引比文件 -写入滞后约 1 秒；工具响应会标注过期文件。 +写入滞后约 1 秒；工具响应会标注过期文件。问题若明确提到源码路径，就把路径直接放进 +explore 查询：显式路径会被标准化并固定在模糊结果之前，无法解析的路径会明确报告。 @@ -413,6 +414,18 @@ codegraph http list / stop # 管理运行中的 HTTP 服务 文件会被加入，无需重启 MCP 服务。TOML 损坏时保留最后一次有效 scope 并报告错误； JSON 继续沿用宽容的空 override 语义。 +若希望 vendor、生成代码等继续被索引和直接查询，但在排序中低于一方代码，可配置： + +```toml +[indexing] +deprioritize = ["vendor/**", "generated/**"] +``` + +兼容格式 `.codegraph/codegraph.json` 也接受顶层 `deprioritize` 数组。JSON 规则先执行， +TOML 规则后执行并最终生效；有序 `!pattern` 例外采用 last-match-wins。该配置只影响 +search/explore 排名，不删除文件、节点或边；长驻 MCP 进程会在每次请求重新读取当前 +项目策略，无需重启。 + 完整参考——守护进程生命周期、所有环境变量、HTTP 注册表、扩展名映射、Claude hook： [`../cli.md`](../cli.md#daemon-watch--environment-variables)。 @@ -420,8 +433,9 @@ JSON 继续沿用宽容的空 override 语义。 ## CLI 子命令 -核心命令：`init`、`index`、`sync`、`search`、`files`、`status`、`serve`、 -`callers`、`callees`、`impact`、`affected`、`check`、`export`、`unlock`。 +核心命令：`init`、`index`、`sync`、`search`、`explore`、`node`、`files`、 +`status`、`serve`、`callers`、`callees`、`impact`、`affected`、`check`、 +`audit`、`export`、`unlock`、`http`、`mcp`。 代理 / 安装命令：`install`、`uninstall`、`skill`、`self-update`、`completions`。 diff --git a/skills/codegraph/SKILL.md b/skills/codegraph/SKILL.md index 8f60d804..91884a63 100644 --- a/skills/codegraph/SKILL.md +++ b/skills/codegraph/SKILL.md @@ -98,6 +98,20 @@ without restarting the MCP server. Invalid TOML keeps the last valid scope and reports an error; malformed JSON retains CodeGraph's tolerant empty-override behavior. +To keep vendored or generated source indexed but rank it below first-party code, +configure ranking-only rules: + +```toml +[indexing] +deprioritize = ["vendor/**", "generated/**"] +``` + +The compatibility form in `.codegraph/codegraph.json` is +`{"deprioritize":["vendor/**"]}`. JSON rules run first, TOML rules run last, +and ordered `!pattern` exceptions make matching last-rule-wins. This never +removes files or changes nodes/edges. Search and explore reload the addressed +project's policy on each request, so a long-running MCP process needs no restart. + Auto-register into all detected agents (Claude Code, Cursor, Codex, opencode, Hermes, Gemini CLI, Antigravity, Kiro, Trae, Qoder, Zed, Zuno, VS Code, Copilot CLI, JetBrains): @@ -125,10 +139,11 @@ Pick your entry point based on what you're trying to answer: | "What files are under path P?" | `codegraph_files` | | "Is the index up to date?" | `codegraph_status` | -These are MCP tool names. When MCP is unavailable and you are invoking the -binary directly, use `codegraph search "" -p /path/to/project` for -symbol lookup. `codegraph query` remains a compatibility alias; prefer -`codegraph search` in new commands and guidance. +These are MCP tool names. When MCP is unavailable, the binary exposes the same +engine through `codegraph explore "" -p /path/to/project`, +`codegraph node "" -p /path/to/project`, and +`codegraph search "" -p /path/to/project`. `codegraph query` remains a +compatibility alias; prefer `codegraph search` in new commands and guidance. ### `codegraph_explore` — start here @@ -143,6 +158,17 @@ relevant to your query, their verbatim source grouped by file, and the call/impact graph connecting them. It replaces the grep-then-read-then-grep loop that would otherwise take 10-20 round-trips. +When the question names a source path, include that path in the explore query. +CodeGraph normalizes quoted/backticked, `./`, Windows, `:line`, and `#Lline` +forms, pins matching files ahead of fuzzy results, and removes path fragments +from normal symbol search. An unresolved explicit path is reported instead of +being dissolved into noisy query tokens. Path recognition is deliberately +bounded; ordinary hyphenated prose is left alone. + +Explore also supplements FTS with camelCase/snake-case identifier segments, so +prose such as "feed bottom" can seed callable, Variable, and Constant names such +as `feedAtBottom` without slowing the ordinary `codegraph_search` path. + Trust the results — they come from a full AST parse, not text matching. Don't re-verify with grep. @@ -153,6 +179,7 @@ suspect) a symbol's name but not where it lives. The results are FTS5-scored across multiple signals (name, kind, path). This is the closest thing to "semantic" lookup available, but it is **fully deterministic full-text scoring** — there are no embeddings, no vector index, no neural model of any kind. +Case-insensitive exact-name lookups use the dedicated lower-name SQLite index. CLI equivalent: