diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 05872489..8faf4d3d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -229,6 +229,29 @@ jobs: cargo test -p codegraph-extract --locked --test scan_symlinks cargo test -p codegraph-watch --locked symlink + ui: + name: UI + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "24" + cache: npm + cache-dependency-path: ui/package-lock.json + + # npm ci from the committed lockfile, svelte-check, the vitest suites, a + # production build, and a byte check that the embedded bundle under + # crates/codegraph-ui/viewer is exactly this build. + - name: Check, test and rebuild the viewer + run: make ui-check + audit: name: Security Audit needs: workspace-version @@ -306,7 +329,7 @@ jobs: ci-success: name: CI Success - needs: [workspace-version, linux, windows-clippy, windows-test, macos-watcher, audit] + needs: [workspace-version, linux, windows-clippy, windows-test, macos-watcher, ui, audit] if: always() runs-on: ubuntu-24.04 timeout-minutes: 5 diff --git a/.gitignore b/.gitignore index a43e3fb2..3c1211ba 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,6 @@ target/llvm-cov/ .aws/*.local .aws/*.rendered.local.json .aws/pipeline.rendered* + +# The browser viewer's npm install (ui/); its build output is committed under crates/codegraph-ui/viewer/. +node_modules/ diff --git a/.oxfmtignore b/.oxfmtignore index a659ed97..3589e8a7 100644 --- a/.oxfmtignore +++ b/.oxfmtignore @@ -14,6 +14,10 @@ release-please-config.json docs/upstream-sync/UPSTREAM.md docs/upstream-sync/V1_5_COMMIT_MANIFEST.md +# The selected viewer design, transcribed from the Penpot boards: its tables +# are numeric specs read against the board exports, so their layout is frozen. +docs/design/viewer-d.md + # Golden artifacts are byte-compared by the equivalence/golden oracle tests # (include_str! / fs::read). Reformatting them breaks the test suite. reference/ @@ -40,6 +44,13 @@ KNOWN_DIFFS.md # Rust sources are formatted by `cargo fmt`. *.rs +# The browser viewer is vendored from upstream colbymchenry/codegraph (ui/, +# MIT, see ui/LICENSE) and keeps upstream's formatting so it stays diffable +# against the release it was imported from; svelte-check and the UI tests are +# its gate. Its committed build output is generated, not authored. +ui/ +crates/codegraph-ui/viewer/ + # Build artifacts / vendored deps. target/ dist/ diff --git a/AGENTS.md b/AGENTS.md index 2ab410bd..e307c7cc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -71,18 +71,19 @@ or required by that guidance. The workspace members are declared in root `Cargo.toml`; that manifest is the authority when the list changes. -| Crate | Owns | -| -------------------------------- | ------------------------------------------------------------------ | -| `codegraph-core` | shared types, config, IDs, file classification, logging | -| `codegraph-extract` | language detection, tree-sitter/custom extraction, scan policy | -| `codegraph-store` | SQLite schema/migrations, FTS5, persistence and queries | -| `codegraph-resolve` | import/name resolution and framework resolvers | -| `codegraph-graph` | traversal, impact, search scoring and query parsing | -| `codegraph-mcp` | MCP schemas, rmcp transports, project resolution, tool rendering | -| `codegraph-watch` | incremental synchronization and filesystem watching | -| `codegraph-daemon` | shared process lifecycle, IPC, registries and detach behavior | -| `codegraph-cli` (`codegraph-rs`) | CLI, installer, orchestration and shipped binary | -| `codegraph-bench` | equivalence oracle and reproducible benchmark harness; not shipped | +| Crate | Owns | +| -------------------------------- | --------------------------------------------------------------------- | +| `codegraph-core` | shared types, config, IDs, file classification, logging | +| `codegraph-extract` | language detection, tree-sitter/custom extraction, scan policy | +| `codegraph-store` | SQLite schema/migrations, FTS5, persistence and queries | +| `codegraph-resolve` | import/name resolution and framework resolvers | +| `codegraph-graph` | traversal, impact, search scoring and query parsing | +| `codegraph-mcp` | MCP schemas, rmcp transports, project resolution, tool rendering | +| `codegraph-watch` | incremental synchronization and filesystem watching | +| `codegraph-daemon` | shared process lifecycle, IPC, registries and detach behavior | +| `codegraph-ui` | browser viewer: loopback JSON API, live channel, embedded `ui/` build | +| `codegraph-cli` (`codegraph-rs`) | CLI, installer, orchestration and shipped binary | +| `codegraph-bench` | equivalence oracle and reproducible benchmark harness; not shipped | Keep dependency direction acyclic and lower layers independent of presentation. Extraction must not depend on store/graph. Query rendering must not leak into core @@ -93,17 +94,18 @@ semantics. See [`docs/architecture.md`](docs/architecture.md) for the current gr Run the narrowest relevant tests while iterating, then the complete gate before handoff. Do not use a narrow test to claim a workspace-wide property. -| Change | Minimum focused proof | Required documentation | -| -------------------------- | ---------------------------------------------------------------------- | ---------------------------------------------------------------------------- | -| extraction/language rules | extractor tests; affected golden corpus; incremental/full equivalence | `languages.md`, `grammar-manifest.md`, and golden recipe when behavior moves | -| resolution/framework rules | resolver unit/integration tests; ambiguity negatives; affected golden | `equivalence.md` or framework reference when public behavior moves | -| schema/migrations/store | schema parity; migration replay; state/lease tests; golden equivalence | `data-model.md` | -| graph/search | graph/query tests; deterministic ordering and limit cases | CLI/MCP docs for public output changes | -| MCP/protocol | engine tests; structural MCP goldens; rmcp stdio/HTTP/version tests | `mcp.md` | -| CLI/installer | command tests; installer round trips; script contract fixtures | `cli.md`, README only for landing-page behavior | -| daemon/watch/concurrency | lifecycle, lock, recovery, watcher and platform-focused tests | architecture/CLI/MCP lifecycle sections | -| release/install/checksum | shell/PowerShell fixtures, asset-name checks, archive smoke | README install section and release workflow contract | -| docs/community files | `python3 scripts/docs-check.py`; formatter; link/anchor checks | update the canonical page, not a duplicate summary | +| Change | Minimum focused proof | Required documentation | +| ------------------------------ | ------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------- | +| extraction/language rules | extractor tests; affected golden corpus; incremental/full equivalence | `languages.md`, `grammar-manifest.md`, and golden recipe when behavior moves | +| resolution/framework rules | resolver unit/integration tests; ambiguity negatives; affected golden | `equivalence.md` or framework reference when public behavior moves | +| schema/migrations/store | schema parity; migration replay; state/lease tests; golden equivalence | `data-model.md` | +| graph/search | graph/query tests; deterministic ordering and limit cases | CLI/MCP docs for public output changes | +| MCP/protocol | engine tests; structural MCP goldens; rmcp stdio/HTTP/version tests | `mcp.md` | +| CLI/installer | command tests; installer round trips; script contract fixtures | `cli.md`, README only for landing-page behavior | +| daemon/watch/concurrency | lifecycle, lock, recovery, watcher and platform-focused tests | architecture/CLI/MCP lifecycle sections | +| release/install/checksum | shell/PowerShell fixtures, asset-name checks, archive smoke | README install section and release workflow contract | +| viewer (`codegraph-ui`, `ui/`) | crate tests over indexed fixtures; `cli_ui`; `make ui-check` (rebuilds and byte-checks the committed bundle) | `ui.md`; `cli.md` for the command | +| docs/community files | `python3 scripts/docs-check.py`; formatter; link/anchor checks | update the canonical page, not a duplicate summary | If a change alters nodes, edges, reference resolution, file classification, or stored graph meaning, decide explicitly whether the extraction version must move. @@ -136,7 +138,10 @@ The authoritative local/CI entry point is: make check ``` -`make ci` is a compatibility alias for the same complete gate. `make pre-ci` adds a package/unpack/execute smoke over the built release bytes. +`make ci` is a compatibility alias for the same complete gate. `make pre-ci` adds the +viewer frontend gate (`make ui-check`: `npm ci`, svelte-check, vitest, a production +build, and a byte check of the committed bundle under `crates/codegraph-ui/viewer`) +and a package/unpack/execute smoke over the built release bytes. It validates workspace-version consistency before any Cargo subprocess, required tool versions, Rust and repository-text formatting, workflow/shell linting, Clippy with warnings denied, locked tests, a locked release build, guardrails, diff --git a/Cargo.lock b/Cargo.lock index 39afa1a6..fe1a0c6b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -551,6 +551,7 @@ dependencies = [ "rusqlite", "serde", "serde_json", + "tempfile", "unicode-normalization", ] @@ -609,6 +610,7 @@ dependencies = [ "codegraph-mcp", "codegraph-resolve", "codegraph-store", + "codegraph-ui", "codegraph-watch", "indicatif", "interprocess", @@ -642,6 +644,36 @@ dependencies = [ "tracing", ] +[[package]] +name = "codegraph-ui" +version = "0.52.2" +dependencies = [ + "anyhow", + "axum", + "codegraph-core", + "codegraph-extract", + "codegraph-graph", + "codegraph-mcp", + "codegraph-resolve", + "codegraph-store", + "codegraph-watch", + "form_urlencoded", + "futures-util", + "http-body-util", + "httpdate", + "notify", + "percent-encoding", + "rusqlite", + "serde", + "serde_json", + "sha2 0.11.0", + "tempfile", + "tokio", + "tokio-stream", + "tower", + "tracing", +] + [[package]] name = "codegraph-watch" version = "0.52.2" @@ -2879,6 +2911,16 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + [[package]] name = "signature" version = "2.2.0" @@ -3187,6 +3229,7 @@ dependencies = [ "libc", "mio", "pin-project-lite", + "signal-hook-registry", "socket2", "tokio-macros", "windows-sys 0.61.2", @@ -3222,6 +3265,7 @@ dependencies = [ "futures-core", "pin-project-lite", "tokio", + "tokio-util", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 5929f9f3..cc03b2d2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ members = [ "crates/codegraph-mcp", "crates/codegraph-daemon", "crates/codegraph-watch", + "crates/codegraph-ui", "crates/codegraph-cli", "crates/codegraph-bench", ] diff --git a/Makefile b/Makefile index 08e8a7d2..c69dfc3b 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ OXFMT_ARGS := --no-error-on-unmatched-pattern --ignore-path .oxfmtignore . .PHONY: all build build-dev build-prod release release-target install uninstall \ fmt fmt-rust fmt-oxfmt fmt-check fmt-rust-check fmt-oxfmt-check \ tools-check workflow-lint docs-check workspace-version typecheck lint test guardrail script-tests archive-smoke \ - check ci pre-ci hooks setup-hooks clean size-compare help \ + check ci pre-ci ui ui-check hooks setup-hooks clean size-compare help \ coverage coverage-html coverage-lcov coverage-open coverage-clean all: build @@ -139,7 +139,23 @@ check: @echo "All checks passed." ci: check -pre-ci: check archive-smoke +pre-ci: check ui-check archive-smoke + +# Rebuild the embedded viewer bundle (crates/codegraph-ui/viewer) from ui/. +ui: + cd ui && npm ci && npm run build + +# The browser viewer's frontend (`ui/`, Node + npm): install from the lockfile, +# type-check, test, rebuild, and require the committed bundle under +# crates/codegraph-ui/viewer to be exactly that build. CI runs it as its own job. +ui-check: + cd ui && npm ci && npm run check && npm test && npm run build + git diff --exit-code -- crates/codegraph-ui/viewer + @stale="$$(git status --porcelain --untracked-files=all -- crates/codegraph-ui/viewer)"; \ + if [ -n "$$stale" ]; then \ + echo "the committed viewer bundle is not this build; commit crates/codegraph-ui/viewer:"; \ + echo "$$stale"; exit 1; \ + fi archive-smoke: bash scripts/smoke-release-archive.sh @@ -189,7 +205,8 @@ help: @printf '%s\n' \ 'Targets:' \ ' check / ci complete quality gate (ci is an alias)' \ - ' pre-ci complete gate plus local archive smoke' \ + ' pre-ci complete gate, viewer frontend, and local archive smoke' \ + ' ui / ui-check rebuild the viewer bundle / also check, test, and byte-check it' \ ' fmt / fmt-check Rust plus repository text formatting' \ ' typecheck locked cargo check for the workspace' \ ' lint / test / release locked Rust gates and shipped build' \ diff --git a/README.md b/README.md index 374cd64d..389a9dbd 100644 --- a/README.md +++ b/README.md @@ -291,6 +291,7 @@ make check - [`docs/architecture.md`](docs/architecture.md) — workspace and runtime design - [`docs/cli.md`](docs/cli.md) — complete command reference - [`docs/mcp.md`](docs/mcp.md) — MCP transports, tools, and clients +- [`docs/ui.md`](docs/ui.md) — local browser viewer (preview, `CODEGRAPH_UI=1`) - [`docs/languages.md`](docs/languages.md) — language coverage and boundaries - [`docs/equivalence.md`](docs/equivalence.md) — deterministic golden contract - [`docs/upstream-sync/UPSTREAM.md`](docs/upstream-sync/UPSTREAM.md) — upstream ledger diff --git a/crates/codegraph-cli/Cargo.toml b/crates/codegraph-cli/Cargo.toml index 7e7170a2..fee53a8e 100644 --- a/crates/codegraph-cli/Cargo.toml +++ b/crates/codegraph-cli/Cargo.toml @@ -25,6 +25,7 @@ codegraph-graph = { path = "../codegraph-graph" } codegraph-mcp = { path = "../codegraph-mcp" } codegraph-resolve = { path = "../codegraph-resolve" } codegraph-store = { path = "../codegraph-store" } +codegraph-ui = { path = "../codegraph-ui" } codegraph-watch = { path = "../codegraph-watch" } serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/codegraph-cli/src/main.rs b/crates/codegraph-cli/src/main.rs index b0985f38..310f49a7 100644 --- a/crates/codegraph-cli/src/main.rs +++ b/crates/codegraph-cli/src/main.rs @@ -14,7 +14,7 @@ use std::sync::mpsc; use std::time::{SystemTime, UNIX_EPOCH}; use anyhow::{Context, Result, anyhow, bail}; -use clap::{CommandFactory, Parser, Subcommand, ValueEnum}; +use clap::{CommandFactory, FromArgMatches, Parser, Subcommand, ValueEnum}; use clap_complete::{Shell, generate}; use codegraph_core::config::Config; use codegraph_core::deprioritize::DeprioritizeMatcher; @@ -40,6 +40,7 @@ use time::format_description::well_known::Rfc3339; mod diagnostics; mod installer; mod structural_gate; +mod viewer_gate; /// Test-only: the ONE process-wide environment lock for this binary. /// @@ -170,7 +171,17 @@ fn main() { } fn cli_main() { - let cli = Cli::parse(); + // The browser viewer is not part of a release yet: refuse `ui` / `web` + // (also as `help ui` or `ui --help`) before any startup work unless + // CODEGRAPH_UI=1 opts in. + let raw_args: Vec = std::env::args().skip(1).collect(); + if let Some(command) = viewer_gate::requested_viewer_command(&raw_args) + && !viewer_gate::viewer_enabled() + { + eprintln!("{}", viewer_gate::refusal(command)); + std::process::exit(1); + } + let cli = parse_cli(); // Process bootstrap has no addressed project yet, so this config is // `APP_CONFIG`-or-defaults ONLY and may configure NOTHING but the logger // below. Every project operation (index, sync, watch, an MCP request) loads @@ -216,6 +227,14 @@ fn cli_main() { } } +/// `Cli::parse()`, with the viewer listed in `--help` only when it is enabled. +fn parse_cli() -> Cli { + let enabled = viewer_gate::viewer_enabled(); + let command = Cli::command().mut_subcommand("ui", |ui| ui.hide(!enabled)); + let matches = command.get_matches(); + Cli::from_arg_matches(&matches).unwrap_or_else(|err| err.exit()) +} + #[derive(Debug, Parser)] #[command(name = "codegraph")] #[command(version = VERSION)] @@ -578,6 +597,21 @@ enum Command { #[command(subcommand)] action: McpAction, }, + /// Open the CodeGraph viewer in your browser — read your indexed project as a graph. + #[command(hide = true, visible_alias = "web", after_help = UI_AFTER_HELP)] + Ui { + /// The indexed project to read (default: the one you're standing in). + path: Option, + /// Port to listen on (default: 4747, or the next free one). + #[arg(long, value_name = "NUMBER")] + port: Option, + /// Print the URL instead of opening a browser. + #[arg(long = "no-open")] + no_open: bool, + /// Refuse every write — saved trails can be opened but not saved or deleted. + #[arg(long = "read-only")] + read_only: bool, + }, /// Print the codegraph version. Version, /// Generate shell completion scripts (bash, zsh, fish, powershell, elvish). @@ -926,6 +960,12 @@ fn run(cli: Cli) -> Result<()> { Command::Mcp { action } => match action { McpAction::List { json } => cmd_mcp_list(json), }, + Command::Ui { + path, + port, + no_open, + read_only, + } => cmd_ui(path, port, no_open, read_only), Command::Version => { println!("codegraph {VERSION}"); Ok(()) @@ -6808,6 +6848,141 @@ fn warn_ancestor_index_retarget(requested: &Path, resolved: &Path) { ); } +const UI_AFTER_HELP: &str = "Examples: + $ codegraph ui Read the project you're standing in + $ codegraph ui ~/code/my-app Read a specific indexed project + $ codegraph ui --port 8080 Use one specific port (fails if it's taken) + $ codegraph ui --no-open Just print the URL (headless boxes, SSH) + $ codegraph web Same command under its alias + +Pick a symbol and you see who calls it on the left, its source in the middle, +and what it calls on the right at the height of the line that calls it. Search +with / (or Cmd-K), click a file path for the file's outline and its imports. + +Ask \"how does main reach open_store\" (or \"main -> open_store\") in the search +box for the flow between two symbols: one card per hop, opened at the line that +makes the next call, with dynamic-dispatch hops drawn dashed and named. The Map +tab draws the whole project by module, with dependencies pointing down. + +Never opened this codebase before? The Entry points tab lists the routes with +the symbols that serve them, the files that run something when they load, the +tests, and what the most code depends on — and starts a flow from any of them. + +The page keeps up with the project while it is open: save a file and it says so +within about half a second, and whatever is on screen re-reads the graph when +something re-indexes it. It watches for that; it never polls. + +Save a walk you want to keep: name the trail and it is written under the index +directory (.codegraph/ui/trails/, already gitignored) as plain JSON, listed on +the empty screen, and reopened at the symbol you left. Hops are remembered by +name rather than by position, so a saved trail survives re-indexing and says +which hop moved when one does. Pass --read-only to refuse every write. + +The viewer listens on 127.0.0.1 only, so nothing on your network can reach it. +It opens an index that already exists, never indexes, and never changes a line +of your code — the one thing it writes is a trail you asked it to save. +Requests from any other host are refused, and nothing is sent anywhere: no code, +no paths, no analytics. + +Without --port it takes 4747, or the next free port if that one is busy. + +Set CODEGRAPH_BROWSER= to choose which browser opens, or +CODEGRAPH_BROWSER=none to never open one."; + +/// `codegraph ui [path]` (alias `web`): serve the embedded viewer over +/// loopback and open it. Reads an index that already exists; never indexes. +fn cmd_ui( + path: Option, + port: Option, + no_open: bool, + read_only: bool, +) -> Result<()> { + // An explicit --port stays explicit: a scripted `--port 8080` that quietly + // lands on 8081 is worse than one that says the port is busy. + let requested_port = match port { + None => None, + Some(raw) => match raw.trim().parse::() { + Ok(port) => Some(port), + Err(_) => ui_refusal(format!( + "--port must be a whole number between 0 and 65535 (got \"{raw}\")." + )), + }, + }; + let start = absolute_path(path.unwrap_or_else(|| PathBuf::from("."))); + let indexed = codegraph_ui::find_indexed_root(&start); + let project = indexed.clone().unwrap_or_else(|| start.clone()); + // The same sensitive-directory refusal the MCP entry points use, before + // anything opens: `codegraph ui /etc` is turned away here. + if let Err(reason) = codegraph_ui::security::validate_project_path(&project) { + ui_refusal(reason); + } + if indexed.is_none() { + eprintln!("✗ No CodeGraph index found for {}", project.display()); + eprintln!(); + eprintln!(" The viewer reads an index that already exists — it never creates one."); + eprintln!(" To index this project:"); + eprintln!(); + eprintln!(" codegraph init"); + eprintln!(); + eprintln!(" Already indexed somewhere else? Point the viewer at it:"); + eprintln!(); + eprintln!(" codegraph ui /path/to/indexed/project"); + eprintln!(); + std::process::exit(1); + } + if !codegraph_ui::has_viewer() { + ui_refusal( + "The CodeGraph viewer assets are missing from this build.\n\ + If you installed CodeGraph normally, reinstall it — the release binary embeds the viewer.\n\ + If you are working from a source checkout, run: npm run build --prefix ui", + ); + } + let options = codegraph_ui::UiOptions { + project_root: project.clone(), + port: requested_port, + read_only, + }; + let served = codegraph_ui::serve_until_shutdown(options, |url| { + println!(); + println!("CodeGraph viewer"); + println!(); + println!(" Reading {}", project.display()); + println!(" URL {url}"); + println!( + " Access this machine only — {}", + if read_only { + "read-only, nothing leaves your computer" + } else { + "nothing leaves your computer; saved trails are the only thing written" + } + ); + println!(); + let opened = !no_open && codegraph_ui::browser::open_browser(url); + println!( + "{}", + if opened { + " Opening your browser... press Ctrl+C to stop." + } else { + " Open that URL in a browser. Press Ctrl+C to stop." + } + ); + println!(); + }); + // Both startup failures (no free port, a pinned port taken) carry their own + // remediation: print it plainly. + if let Err(err) = served { + ui_refusal(err); + } + Ok(()) +} + +/// A refusal the way upstream's `codegraph ui` prints one: a cross and the +/// sentence — never a stack trace, never an `Error:` prefix. +fn ui_refusal(message: impl std::fmt::Display) -> ! { + eprintln!("✗ {message}"); + std::process::exit(1); +} + fn resolve_project_path_optional(start: &Path) -> PathBuf { if has_lifecycle_namespace(start) { return start.to_path_buf(); diff --git a/crates/codegraph-cli/src/viewer_gate.rs b/crates/codegraph-cli/src/viewer_gate.rs new file mode 100644 index 00000000..fe8064f3 --- /dev/null +++ b/crates/codegraph-cli/src/viewer_gate.rs @@ -0,0 +1,73 @@ +//! The browser viewer (`codegraph ui`, alias `web`) ships in the binary but is +//! not part of a release yet. Until it launches, both spellings — also as +//! `help ui` or `ui --help` — are refused before any startup work unless +//! `CODEGRAPH_UI=1` is set, which keeps it usable for the people testing it. A +//! port of upstream `src/bin/viewer-gate.ts`; `main` runs it before +//! `Cli::parse()`. + +/// Setting this to `1` enables the viewer commands. +pub const VIEWER_ENV: &str = "CODEGRAPH_UI"; + +const VIEWER_COMMANDS: &[&str] = &["ui", "web"]; +/// Program-level flags that take no value and may precede the command. +const PROGRAM_FLAGS: &[&str] = &["--color", "--no-color"]; + +pub fn viewer_enabled() -> bool { + std::env::var(VIEWER_ENV).is_ok_and(|v| v == "1") +} + +/// The viewer command these arguments ask for (`ui` or `web`), whether run +/// directly or through `help`, or `None` when they ask for something else. +pub fn requested_viewer_command(args: &[String]) -> Option<&str> { + let positional: Vec<&str> = args + .iter() + .map(String::as_str) + .filter(|arg| !PROGRAM_FLAGS.contains(arg)) + .collect(); + let command = if positional.first() == Some(&"help") { + positional.get(1) + } else { + positional.first() + }; + command.copied().filter(|c| VIEWER_COMMANDS.contains(c)) +} + +/// The refusal upstream prints, word for word. +pub fn refusal(command: &str) -> String { + format!( + "error: 'codegraph {command}' is not in this release yet. The browser viewer is coming in an upcoming release." + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn args(list: &[&str]) -> Vec { + list.iter().map(|s| s.to_string()).collect() + } + + #[test] + fn the_viewer_is_recognised_however_it_is_asked_for() { + assert_eq!(requested_viewer_command(&args(&["ui"])), Some("ui")); + assert_eq!(requested_viewer_command(&args(&["web", "."])), Some("web")); + assert_eq!(requested_viewer_command(&args(&["help", "ui"])), Some("ui")); + assert_eq!( + requested_viewer_command(&args(&["ui", "--help"])), + Some("ui") + ); + assert_eq!( + requested_viewer_command(&args(&["--no-color", "web"])), + Some("web") + ); + } + + #[test] + fn everything_else_passes() { + assert_eq!(requested_viewer_command(&args(&[])), None); + assert_eq!(requested_viewer_command(&args(&["status"])), None); + assert_eq!(requested_viewer_command(&args(&["help"])), None); + assert_eq!(requested_viewer_command(&args(&["search", "ui"])), None); + assert_eq!(requested_viewer_command(&args(&["--help", "ui"])), None); + } +} diff --git a/crates/codegraph-cli/tests/cli_ui.rs b/crates/codegraph-cli/tests/cli_ui.rs new file mode 100644 index 00000000..47d6eee9 --- /dev/null +++ b/crates/codegraph-cli/tests/cli_ui.rs @@ -0,0 +1,486 @@ +//! `codegraph ui` end to end — a port of upstream `__tests__/cli-ui-command.test.ts` +//! (`v1.6.1`) plus its viewer gate (`src/bin/viewer-gate.ts`): the binary is +//! run for real, the server it starts is asked over a real socket, and every +//! wait is bounded. + +use std::io::{BufRead, BufReader, Read, Write}; +use std::net::TcpStream; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::sync::mpsc; +use std::time::{Duration, Instant}; + +struct TestDir { + path: PathBuf, +} + +impl TestDir { + fn new(label: &str) -> Self { + let path = std::env::temp_dir().join(format!( + "codegraph-cli-ui-{label}-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::create_dir_all(&path).unwrap(); + Self { path } + } +} + +impl Drop for TestDir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.path); + } +} + +fn command(args: &[&str], viewer: bool) -> Command { + let mut command = Command::new(env!("CARGO_BIN_EXE_codegraph")); + command + .args(args) + .env("CODEGRAPH_NO_DAEMON", "1") + .env("NO_COLOR", "1") + .env_remove("CODEGRAPH_BROWSER"); + if viewer { + command.env("CODEGRAPH_UI", "1"); + } else { + command.env_remove("CODEGRAPH_UI"); + } + command +} + +/// Run to completion: (exit code, stdout + stderr). +fn run(args: &[&str], viewer: bool) -> (i32, String) { + let output = command(args, viewer).output().expect("run codegraph"); + let mut text = String::from_utf8_lossy(&output.stdout).into_owned(); + text.push_str(&String::from_utf8_lossy(&output.stderr)); + (output.status.code().unwrap_or(-1), text) +} + +fn indexed_project(label: &str) -> TestDir { + let dir = TestDir::new(label); + std::fs::create_dir_all(dir.path.join("src")).unwrap(); + std::fs::write( + dir.path.join("src/auth.ts"), + "export function parseToken(t: string){ return t.trim(); }\n", + ) + .unwrap(); + let (code, output) = run(&["init", dir.path.to_str().unwrap()], false); + assert_eq!(code, 0, "init: {output}"); + dir +} + +/// A running viewer and everything it has printed so far. +struct Viewer { + child: Child, + port: u16, + lines: mpsc::Receiver, + output: String, +} + +impl Viewer { + fn start(args: &[&str], env: &[(&str, &str)]) -> Self { + let mut command = command(&[&["ui"], args].concat(), true); + for (key, value) in env { + command.env(key, value); + } + let mut child = command + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("spawn codegraph ui"); + let (tx, lines) = mpsc::channel(); + for stream in [ + Box::new(child.stdout.take().unwrap()) as Box, + Box::new(child.stderr.take().unwrap()), + ] { + let tx = tx.clone(); + std::thread::spawn(move || { + for line in BufReader::new(stream).lines().map_while(Result::ok) { + let _ = tx.send(line); + } + }); + } + let mut viewer = Self { + child, + port: 0, + lines, + output: String::new(), + }; + let deadline = Instant::now() + Duration::from_secs(30); + loop { + let left = deadline.saturating_duration_since(Instant::now()); + match viewer.lines.recv_timeout(left) { + Ok(line) => { + viewer.output.push_str(&line); + viewer.output.push('\n'); + if let Some(at) = line.find("http://127.0.0.1:") { + let rest = &line[at + "http://127.0.0.1:".len()..]; + let digits: String = + rest.chars().take_while(char::is_ascii_digit).collect(); + viewer.port = digits.parse().unwrap(); + break; + } + } + Err(_) => { + let _ = viewer.child.kill(); + panic!("codegraph ui never printed a URL:\n{}", viewer.output); + } + } + } + viewer + } + + /// Everything printed until `marker` appears, waiting at most `timeout`. + fn read_until(&mut self, marker: &str, timeout: Duration) -> bool { + let deadline = Instant::now() + timeout; + while !self.output.contains(marker) { + let left = deadline.saturating_duration_since(Instant::now()); + match self.lines.recv_timeout(left) { + Ok(line) => { + self.output.push_str(&line); + self.output.push('\n'); + } + Err(_) => return false, + } + } + true + } + + fn stop(mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +/// One HTTP/1.1 GET with the `Host` header given: (status, body). +fn get(port: u16, path: &str, host: Option<&str>) -> (u16, String) { + let mut stream = TcpStream::connect(("127.0.0.1", port)).expect("connect"); + stream + .set_read_timeout(Some(Duration::from_secs(30))) + .unwrap(); + let host = host + .map(str::to_string) + .unwrap_or_else(|| format!("127.0.0.1:{port}")); + write!( + stream, + "GET {path} HTTP/1.1\r\nHost: {host}\r\nConnection: close\r\n\r\n" + ) + .unwrap(); + let mut response = String::new(); + stream.read_to_string(&mut response).unwrap(); + let status = response + .split_whitespace() + .nth(1) + .and_then(|s| s.parse().ok()) + .unwrap_or(0); + let body = response + .split_once("\r\n\r\n") + .map(|(_, b)| b.to_string()) + .unwrap_or_default(); + (status, body) +} + +/* ------------------------------------------------------------------- gate -- */ + +#[test] +fn the_viewer_is_refused_unless_codegraph_ui_is_1() { + for args in [ + &["ui"][..], + &["web", "."], + &["help", "ui"], + &["ui", "--help"], + &["help", "web"], + ] { + let (code, output) = run(args, false); + assert_eq!(code, 1, "{args:?}: {output}"); + let command = if args.contains(&"web") { "web" } else { "ui" }; + assert!( + output.contains(&format!( + "error: 'codegraph {command}' is not in this release yet. The browser viewer is coming in an upcoming release." + )), + "{args:?}: {output}" + ); + } + let (code, top) = run(&["--help"], false); + assert_eq!(code, 0); + assert!(!top.contains("Open the CodeGraph viewer"), "{top}"); +} + +/* ------------------------------------------------------------------- help -- */ + +#[test] +fn reads_well_and_documents_the_flags() { + let (code, output) = run(&["ui", "--help"], true); + assert_eq!(code, 0, "{output}"); + for needle in [ + "--port", + "--no-open", + "4747", + "127.0.0.1", + "read-only", + "Examples:", + "CODEGRAPH_BROWSER", + ] { + assert!(output.contains(needle), "{needle} in {output}"); + } +} + +#[test] +fn works_through_codegraph_help_ui() { + let (code, via_help) = run(&["help", "ui"], true); + let (_, via_flag) = run(&["ui", "--help"], true); + assert_eq!(code, 0); + assert_eq!(via_help, via_flag); +} + +#[test] +fn is_listed_in_the_top_level_help_and_web_is_an_alias() { + let (_, top) = run(&["--help"], true); + assert!(top.contains("ui") && top.contains("web"), "{top}"); + assert!(top.contains("Open the CodeGraph viewer")); + let (code, via_alias) = run(&["help", "web"], true); + assert_eq!(code, 0, "{via_alias}"); + assert!(via_alias.contains("--no-open")); +} + +#[test] +fn rejects_a_nonsense_port_with_a_plain_message_not_a_stack_trace() { + let (code, output) = run(&["ui", "--port", "banana"], true); + assert_eq!(code, 1); + assert!(output.contains("--port must be a whole number"), "{output}"); + assert!(!output.contains("panicked")); +} + +/* --------------------------------------------------------------- refusals -- */ + +#[test] +fn gives_friendly_guidance_never_a_stack_trace_when_there_is_no_index() { + let dir = TestDir::new("unindexed"); + std::fs::write(dir.path.join("a.ts"), "export const a = 1;\n").unwrap(); + let (code, output) = run(&["ui", dir.path.to_str().unwrap()], true); + assert_eq!(code, 1); + assert!(output.contains("No CodeGraph index found"), "{output}"); + assert!(output.contains("codegraph init")); + assert!(!output.contains("Error:"), "{output}"); + assert!(!output.contains("panicked")); +} + +#[cfg(unix)] +#[test] +fn refuses_a_sensitive_system_directory() { + let (code, output) = run(&["ui", "/etc"], true); + assert_eq!(code, 1); + assert!( + output.contains("Refusing to operate on sensitive"), + "{output}" + ); +} + +/* ---------------------------------------------------------------- serving -- */ + +#[test] +fn serves_the_viewer_and_prints_where_it_is() { + let project = indexed_project("serve"); + let root = project.path.to_str().unwrap(); + let mut viewer = Viewer::start(&["--no-open", "--port", "0", root], &[]); + assert!( + viewer.read_until("Press Ctrl+C to stop", Duration::from_secs(10)), + "{}", + viewer.output + ); + let (status, body) = get(viewer.port, "/", None); + assert_eq!(status, 200); + assert!(body.contains(r#"
"#)); + assert!( + viewer.output.contains("CodeGraph viewer"), + "{}", + viewer.output + ); + let canonical = Path::new(root).canonicalize().unwrap(); + assert!( + viewer.output.contains(&canonical.display().to_string()), + "{}", + viewer.output + ); + assert!(viewer.output.contains("this machine only")); + viewer.stop(); +} + +/// A stand-in browser that records the URL it was launched with. +fn opener(dir: &TestDir) -> (PathBuf, PathBuf) { + let marker = dir.path.join("opened.txt"); + #[cfg(windows)] + let script = { + let script = dir.path.join("open.cmd"); + std::fs::write(&script, format!("@echo %1 > \"{}\"\r\n", marker.display())).unwrap(); + script + }; + #[cfg(not(windows))] + let script = { + use std::os::unix::fs::PermissionsExt; + let script = dir.path.join("open.sh"); + std::fs::write( + &script, + format!("#!/bin/sh\nprintf '%s' \"$1\" > \"{}\"\n", marker.display()), + ) + .unwrap(); + std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o755)).unwrap(); + script + }; + (script, marker) +} + +fn wait_for_file(path: &Path, timeout: Duration) -> Option { + let deadline = Instant::now() + timeout; + loop { + if let Ok(text) = std::fs::read_to_string(path) + && !text.trim().is_empty() + { + return Some(text); + } + if Instant::now() > deadline { + return None; + } + std::thread::sleep(Duration::from_millis(50)); + } +} + +#[test] +fn honours_no_open_no_browser_is_launched() { + let project = indexed_project("no-open"); + let tools = TestDir::new("opener-a"); + let (script, marker) = opener(&tools); + let mut viewer = Viewer::start( + &["--no-open", "--port", "0", project.path.to_str().unwrap()], + &[("CODEGRAPH_BROWSER", script.to_str().unwrap())], + ); + assert!( + viewer.read_until("Press Ctrl+C to stop", Duration::from_secs(10)), + "{}", + viewer.output + ); + assert_eq!(get(viewer.port, "/", None).0, 200); + assert!(wait_for_file(&marker, Duration::from_millis(1500)).is_none()); + assert!( + viewer.output.contains("Open that URL in a browser"), + "{}", + viewer.output + ); + assert!(!viewer.output.contains("Opening your browser")); + viewer.stop(); +} + +#[test] +fn opens_the_browser_at_the_served_url_when_no_open_is_absent() { + let project = indexed_project("open"); + let tools = TestDir::new("opener-b"); + let (script, marker) = opener(&tools); + let mut viewer = Viewer::start( + &["--port", "0", project.path.to_str().unwrap()], + &[("CODEGRAPH_BROWSER", script.to_str().unwrap())], + ); + let opened = wait_for_file(&marker, Duration::from_secs(10)).expect("the opener ran"); + assert!( + opened + .trim() + .contains(&format!("http://127.0.0.1:{}", viewer.port)), + "{opened}" + ); + assert!( + viewer.read_until("Opening your browser", Duration::from_secs(10)), + "{}", + viewer.output + ); + viewer.stop(); +} + +#[test] +fn codegraph_browser_none_suppresses_the_launch_like_no_open() { + let project = indexed_project("browser-none"); + let mut viewer = Viewer::start( + &["--port", "0", project.path.to_str().unwrap()], + &[("CODEGRAPH_BROWSER", "none")], + ); + assert!( + viewer.read_until("Press Ctrl+C to stop", Duration::from_secs(10)), + "{}", + viewer.output + ); + assert_eq!(get(viewer.port, "/", None).0, 200); + assert!( + viewer.output.contains("Open that URL in a browser"), + "{}", + viewer.output + ); + viewer.stop(); +} + +#[test] +fn moves_off_the_default_port_when_it_is_busy() { + let project = indexed_project("busy"); + // Occupy 4747 so the fallback has something to fall back FROM; if another + // viewer already holds it the assertion below is still the right one. + let blocker = std::net::TcpListener::bind(("127.0.0.1", 4747)).ok(); + let viewer = Viewer::start(&["--no-open", project.path.to_str().unwrap()], &[]); + assert_ne!(viewer.port, 4747); + assert_eq!(get(viewer.port, "/", None).0, 200); + viewer.stop(); + drop(blocker); +} + +#[test] +fn refuses_to_move_off_a_port_the_user_pinned_with_port() { + let project = indexed_project("pinned"); + let blocker = std::net::TcpListener::bind(("127.0.0.1", 0)).unwrap(); + let taken = blocker.local_addr().unwrap().port().to_string(); + let (code, output) = run( + &[ + "ui", + "--no-open", + "--port", + &taken, + project.path.to_str().unwrap(), + ], + true, + ); + assert_eq!(code, 1); + assert!(output.contains("already in use"), "{output}"); + assert!(!output.contains("panicked")); +} + +#[test] +fn refuses_a_foreign_host_end_to_end() { + let project = indexed_project("host"); + let viewer = Viewer::start( + &["--no-open", "--port", "0", project.path.to_str().unwrap()], + &[], + ); + let (status, body) = get(viewer.port, "/", Some("evil.example")); + assert_eq!(status, 403); + assert!(!body.contains(r#"
"#)); + viewer.stop(); +} + +#[cfg(unix)] +#[test] +fn shuts_down_cleanly_on_sigterm() { + let project = indexed_project("sigterm"); + let mut viewer = Viewer::start( + &["--no-open", "--port", "0", project.path.to_str().unwrap()], + &[], + ); + let pid = viewer.child.id().to_string(); + let killed = Command::new("kill").args(["-TERM", &pid]).status().unwrap(); + assert!(killed.success()); + let deadline = Instant::now() + Duration::from_secs(15); + let status = loop { + if let Some(status) = viewer.child.try_wait().unwrap() { + break status; + } + assert!(Instant::now() < deadline, "the viewer ignored SIGTERM"); + std::thread::sleep(Duration::from_millis(50)); + }; + assert!(status.success(), "{status:?}"); +} diff --git a/crates/codegraph-extract/src/lib.rs b/crates/codegraph-extract/src/lib.rs index dcbc261f..bcc2a9a0 100644 --- a/crates/codegraph-extract/src/lib.rs +++ b/crates/codegraph-extract/src/lib.rs @@ -15,6 +15,7 @@ pub mod function_ref; pub mod lang; mod links; pub mod spec; +pub mod syntax_tokens; pub mod walker; pub use codegraph_core::source_file::{ diff --git a/crates/codegraph-extract/src/syntax_tokens.rs b/crates/codegraph-extract/src/syntax_tokens.rs new file mode 100644 index 00000000..3ca662b4 --- /dev/null +++ b/crates/codegraph-extract/src/syntax_tokens.rs @@ -0,0 +1,581 @@ +//! Syntax classification from the engine's own tree-sitter parse — a port of +//! upstream `src/extraction/syntax-tokens.ts` (CG-57, `v1.6.1`), used by the +//! browser viewer's code blocks. +//! +//! The classes are deliberately few and the rules language-agnostic: a node +//! whose type mentions `comment` is a comment, whole; inside a string node every +//! leaf is string except below an interpolation; a numeric literal is a number; +//! an anonymous leaf is a keyword when its text is a bare word and punctuation +//! otherwise; a named identifier-shaped leaf is an identifier unless the grammar +//! called it a type name, or the extractor's own definition tables say it is the +//! name of a definition. That last rule reuses [`LanguageSpec`]'s node-type +//! lists, so a language that learns a declaration form gets its name bolded for +//! free. +//! +//! Offsets are BYTE offsets into the source (tree-sitter's), not upstream's +//! UTF-16 indices; the viewer only ever receives the classified text. + +use codegraph_core::types::Language; +use tree_sitter::{Node, Parser}; + +use crate::lang::spec_for_language; +use crate::spec::LanguageSpec; + +/// Every class a token can carry, in wire order. +pub const SYNTAX_TOKEN_CLASSES: [&str; 8] = [ + "other", "ident", "comment", "string", "keyword", "number", "type", "def", +]; + +/// A token class, by its wire index. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[repr(u8)] +pub enum SyntaxClass { + Other = 0, + Ident = 1, + Comment = 2, + String = 3, + Keyword = 4, + Number = 5, + Type = 6, + Def = 7, +} + +impl SyntaxClass { + pub fn id(self) -> u8 { + self as u8 + } + + pub fn name(self) -> &'static str { + SYNTAX_TOKEN_CLASSES[self as usize] + } +} + +/// A classified run of the source, by byte offset. Half-open. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SyntaxSpan { + pub start: usize, + pub end: usize, + pub class: SyntaxClass, +} + +fn is_comment_type(kind: &str) -> bool { + kind.contains("comment") +} + +fn is_string_type(kind: &str) -> bool { + kind.contains("string") + || kind.contains("heredoc") + || kind.contains("regex") + || matches!( + kind, + "char_literal" + | "character" + | "character_literal" + | "rune_literal" + | "quoted_attribute_value" + ) +} + +fn is_interpolation_type(kind: &str) -> bool { + kind.contains("interpolation") + || kind.contains("substitution") + || matches!( + kind, + "template_substitution" | "string_interpolation" | "format_expression" + ) +} + +fn is_number_type(kind: &str) -> bool { + matches!( + kind, + "number" + | "integer" + | "float" + | "number_literal" + | "integer_literal" + | "float_literal" + | "decimal_integer_literal" + | "decimal_floating_point_literal" + | "hex_integer_literal" + | "real_literal" + | "numeric_literal" + | "int_literal" + | "imaginary_literal" + ) +} + +fn is_type_name_type(kind: &str) -> bool { + kind.contains("type_identifier") || kind == "type_name" || kind == "class_type" +} + +/// Built-in type words, emitted whole and undescended with the `type` class. +fn is_builtin_type_type(kind: &str) -> bool { + matches!( + kind, + "primitive_type" | "predefined_type" | "builtin_type" | "sized_type_specifier" + ) +} + +/// Literal constants a theme groups with numbers. +fn is_constant_type(kind: &str) -> bool { + matches!( + kind, + "true" + | "false" + | "null" + | "nil" + | "none" + | "undefined" + | "null_literal" + | "nil_literal" + | "boolean_literal" + | "true_literal" + | "false_literal" + ) +} + +/// Upstream's `IDENT_SHAPE`: `[A-Za-z_$À-￿][\w$À-￿-]*`. +/// Every char at or above U+00C0 counts, astral ones included (JavaScript sees +/// those as two surrogates, both inside the range). +fn is_ident_shape(text: &str) -> bool { + let mut chars = text.chars(); + let Some(first) = chars.next() else { + return false; + }; + let start = |c: char| c.is_ascii_alphabetic() || c == '_' || c == '$' || c >= '\u{C0}'; + start(first) && chars.all(|c| start(c) || c.is_ascii_digit() || c == '-') +} + +/// Upstream's `WORD_SHAPE`: `[A-Za-z_][A-Za-z_0-9-]*` — separates a keyword from +/// punctuation among anonymous nodes. +fn is_word_shape(text: &str) -> bool { + let mut chars = text.chars(); + let Some(first) = chars.next() else { + return false; + }; + (first.is_ascii_alphabetic() || first == '_') + && chars.all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-') +} + +fn definition_types(spec: &dyn LanguageSpec) -> Vec<&'static str> { + let mut out = Vec::new(); + out.extend_from_slice(spec.function_types()); + out.extend_from_slice(spec.class_types()); + out.extend_from_slice(spec.method_types()); + out.extend_from_slice(spec.interface_types()); + out.extend_from_slice(spec.struct_types()); + out.extend_from_slice(spec.enum_types()); + out.extend_from_slice(spec.type_alias_types()); + out.extend_from_slice(spec.union_types()); + out.extend_from_slice(spec.extra_class_node_types()); + out +} + +struct Walk<'a> { + source: &'a [u8], + out: Vec, + def_types: Vec<&'static str>, + name_field: &'static str, + def_starts: std::collections::HashSet, + offset: usize, +} + +impl Walk<'_> { + fn emit(&mut self, start: usize, end: usize, class: SyntaxClass) { + if end <= start { + return; + } + let from = start + self.offset; + let to = end + self.offset; + if let Some(last) = self.out.last_mut() + && last.class == class + && last.end == from + { + last.end = to; + return; + } + self.out.push(SyntaxSpan { + start: from, + end: to, + class, + }); + } + + fn text(&self, node: Node<'_>) -> &str { + std::str::from_utf8(&self.source[node.start_byte()..node.end_byte()]).unwrap_or("") + } + + fn visit(&mut self, node: Node<'_>, in_string: bool) { + let kind = node.kind(); + if node.is_named() && is_comment_type(kind) { + self.emit(node.start_byte(), node.end_byte(), SyntaxClass::Comment); + return; + } + if node.is_named() && is_builtin_type_type(kind) { + self.emit(node.start_byte(), node.end_byte(), SyntaxClass::Type); + return; + } + // Mark the definition's own name before descending to it. + if self.def_types.contains(&kind) + && let Some(name) = node.child_by_field_name(self.name_field) + { + self.def_starts.insert(name.start_byte()); + } + let count = node.child_count(); + if count == 0 { + let class = self.leaf_class(node, in_string); + self.emit(node.start_byte(), node.end_byte(), class); + return; + } + let nested = if is_interpolation_type(kind) { + false + } else { + in_string || is_string_type(kind) + }; + for i in 0..count { + if let Some(child) = node.child(i) { + self.visit(child, nested); + } + } + } + + fn leaf_class(&self, node: Node<'_>, in_string: bool) -> SyntaxClass { + let kind = node.kind(); + // An anonymous node's kind is its own text: keyword or punctuation by shape. + if !node.is_named() { + if in_string { + return SyntaxClass::String; + } + if is_constant_type(kind) { + return SyntaxClass::Number; + } + return if is_word_shape(kind) { + SyntaxClass::Keyword + } else { + SyntaxClass::Other + }; + } + if in_string || is_string_type(kind) { + return SyntaxClass::String; + } + if is_number_type(kind) || is_constant_type(kind) { + return SyntaxClass::Number; + } + let text = self.text(node); + if self.def_starts.contains(&node.start_byte()) && is_ident_shape(text) { + return SyntaxClass::Def; + } + if is_type_name_type(kind) { + return SyntaxClass::Type; + } + if is_ident_shape(text) { + SyntaxClass::Ident + } else { + SyntaxClass::Other + } + } +} + +/// Classify one parsed tree. `offset` shifts every span (a component's script +/// block sits inside its file). +pub fn classify_tree( + root: Node<'_>, + source: &str, + language: Language, + offset: usize, +) -> Vec { + let spec = spec_for_language(language); + let mut walk = Walk { + source: source.as_bytes(), + out: Vec::new(), + def_types: spec.map(definition_types).unwrap_or_default(), + name_field: spec.map(|s| s.name_field()).unwrap_or("name"), + def_starts: std::collections::HashSet::new(), + offset, + }; + walk.visit(root, false); + walk.out +} + +/// A stretch of a file written in a different language from the file itself: +/// a single-file component's script block (or Astro's frontmatter). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SyntaxRegion { + pub start: usize, + pub end: usize, + pub language: Language, +} + +fn find_ci(haystack: &str, needle: &str, from: usize) -> Option { + let lower = haystack.get(from..)?.to_ascii_lowercase(); + lower.find(&needle.to_ascii_lowercase()).map(|i| i + from) +} + +/// The sub-language regions of a file, or `None` when the file is one language. +/// `Some(vec![])` means "a grammar covers none of it". +pub fn syntax_regions_for(source: &str, language: Language) -> Option> { + if !matches!(language, Language::Svelte | Language::Vue | Language::Astro) { + return None; + } + let mut regions = Vec::new(); + if language == Language::Astro + && let Some(rest) = source.strip_prefix("---") + { + let fence = if rest.starts_with("\r\n") { + Some(5) + } else if rest.starts_with('\n') { + Some(4) + } else { + None + }; + if let Some(start) = fence { + // The body runs to the next line that is exactly `---`. + let body = &source[start..]; + let mut end = None; + let mut cursor = 0; + for line in body.split_inclusive('\n') { + let trimmed = line.trim_end_matches(['\n', '\r']); + if trimmed == "---" { + end = Some(cursor); + break; + } + cursor += line.len(); + } + if let Some(end) = end { + let mut body_end = start + end; + // Upstream's lazy `([\s\S]*?)\r?\n---` stops before the newline. + if source[..body_end].ends_with('\n') { + body_end -= 1; + if source[..body_end].ends_with('\r') { + body_end -= 1; + } + } + if body_end > start { + regions.push(SyntaxRegion { + start, + end: body_end, + language: Language::TypeScript, + }); + } + } + } + } + let mut from = 0; + while let Some(open) = find_ci(source, "`. + let next = source[after..].chars().next(); + if !matches!(next, Some(c) if c == '>' || c.is_whitespace()) { + from = after; + continue; + } + let Some(tag_end) = source[after..].find('>').map(|i| i + after) else { + break; + }; + let attrs = &source[after..tag_end]; + let body_start = tag_end + 1; + let Some(close) = find_ci(source, "", body_start) else { + break; + }; + let body = &source[body_start..close]; + if !body.trim().is_empty() { + let lower = attrs.to_ascii_lowercase(); + let is_ts = [ + "lang=\"ts\"", + "lang='ts'", + "lang=\"typescript\"", + "lang='typescript'", + ] + .iter() + .any(|p| lower.replace(' ', "").contains(p)); + regions.push(SyntaxRegion { + start: body_start, + end: close, + language: if is_ts { + Language::TypeScript + } else { + Language::JavaScript + }, + }); + } + from = close + "".len(); + } + Some(regions) +} + +/// The classified spans and the grammar(s) that produced them. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TokenizeResult { + pub spans: Vec, + pub grammars: Vec, +} + +fn tokenize_region(source: &str, language: Language, offset: usize) -> Option> { + let spec = spec_for_language(language)?; + let mut parser = Parser::new(); + parser + .set_language(&spec.tree_sitter_language_for_source(source)) + .ok()?; + let tree = parser.parse(source, None)?; + Some(classify_tree(tree.root_node(), source, language, offset)) +} + +/// Parse `source` and classify it; `None` when nothing in it has a grammar. +/// Never panics on a bad parse: a grammar that will not load is the same +/// outcome as not having one. +pub fn tokenize_source(source: &str, language: Language) -> Option { + match syntax_regions_for(source, language) { + None => { + let spans = tokenize_region(source, language, 0)?; + Some(TokenizeResult { + spans, + grammars: vec![language], + }) + } + Some(regions) => { + if regions.is_empty() { + return None; + } + let mut spans = Vec::new(); + let mut grammars: Vec = Vec::new(); + for region in regions { + let Some(part) = tokenize_region( + &source[region.start..region.end], + region.language, + region.start, + ) else { + continue; + }; + if !grammars.contains(®ion.language) { + grammars.push(region.language); + } + spans.extend(part); + } + if spans.is_empty() { + return None; + } + spans.sort_by_key(|s| s.start); + Some(TokenizeResult { spans, grammars }) + } + } +} + +/// Whether a grammar covers `language` here (component languages read their +/// script through TypeScript), as upstream's `grammarFor`. +pub fn grammar_for(language: Language) -> Option { + if matches!(language, Language::Svelte | Language::Vue | Language::Astro) { + return Some(Language::TypeScript); + } + spec_for_language(language).map(|_| language) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn classes(source: &str, language: Language) -> Vec<(String, &'static str)> { + let result = tokenize_source(source, language).expect("grammar"); + result + .spans + .iter() + .map(|s| (source[s.start..s.end].to_string(), s.class.name())) + .collect() + } + + fn class_of<'a>(spans: &'a [(String, &'static str)], text: &str) -> Option<&'a str> { + spans.iter().find(|(t, _)| t == text).map(|(_, c)| *c) + } + + // Ported from upstream __tests__/ui-highlight.test.ts "classification". + #[test] + fn reads_typescript_with_the_classes_the_theme_paints() { + let spans = classes( + "const total = add(1, 2); // sum\nfunction add(a: number, b: number) { return a + b; }\n", + Language::TypeScript, + ); + assert_eq!(class_of(&spans, "const"), Some("keyword")); + assert_eq!(class_of(&spans, "total"), Some("ident")); + assert_eq!(class_of(&spans, "1"), Some("number")); + assert_eq!(class_of(&spans, "// sum"), Some("comment")); + assert_eq!(class_of(&spans, "number"), Some("type")); + assert!( + spans.iter().any(|(t, c)| t == "add" && *c == "def"), + "the declared name is bold" + ); + } + + #[test] + fn a_hash_comment_is_a_comment_in_python() { + let spans = classes("x = 1 # note\n", Language::Python); + assert_eq!(class_of(&spans, "# note"), Some("comment")); + } + + #[test] + fn a_type_annotation_string_is_not_a_string_literal() { + let spans = classes("let key: string = \"v\";\n", Language::TypeScript); + assert_eq!(class_of(&spans, "string"), Some("type")); + assert_eq!(class_of(&spans, "\"v\""), Some("string")); + } + + #[test] + fn a_template_literal_interpolation_stays_code() { + let spans = classes("const s = `a ${user.name()} b`;\n", Language::TypeScript); + assert_eq!(class_of(&spans, "user"), Some("ident")); + assert_eq!(class_of(&spans, "name"), Some("ident")); + } + + #[test] + fn go_keywords_land_without_naming_them() { + let spans = classes( + "package main\nfunc main() { var x int = 1; _ = x }\n", + Language::Go, + ); + assert_eq!(class_of(&spans, "func"), Some("keyword")); + assert_eq!(class_of(&spans, "package"), Some("keyword")); + } + + #[test] + fn rust_declarations_and_builtin_types() { + let spans = classes("pub fn resolve(p: &str) -> u32 { 0 }\n", Language::Rust); + assert_eq!(class_of(&spans, "resolve"), Some("def")); + assert_eq!(class_of(&spans, "u32"), Some("type")); + assert_eq!(class_of(&spans, "fn"), Some("keyword")); + } + + #[test] + fn spans_are_ordered_and_non_overlapping() { + let source = "fn a() { let b = \"c\"; /* d */ }\n"; + let result = tokenize_source(source, Language::Rust).unwrap(); + assert!(result.spans.windows(2).all(|w| w[0].end <= w[1].start)); + assert!(result.spans.iter().all(|s| s.end <= source.len())); + } + + #[test] + fn a_component_is_read_through_its_script_block() { + let source = + "\n\n"; + let regions = syntax_regions_for(source, Language::Vue).unwrap(); + assert_eq!(regions.len(), 1); + assert_eq!(regions[0].language, Language::TypeScript); + let spans = classes(source, Language::Vue); + assert_eq!(class_of(&spans, "const"), Some("keyword")); + assert!( + !spans.iter().any(|(t, _)| t.contains("template")), + "markup stays unclassified" + ); + } + + #[test] + fn an_empty_script_block_has_no_grammar() { + let regions = syntax_regions_for("", Language::Svelte).unwrap(); + assert!(regions.is_empty()); + assert!(tokenize_source("", Language::Svelte).is_none()); + } + + #[test] + fn languages_without_a_grammar_answer_none() { + assert!(tokenize_source("a: 1\n", Language::Yaml).is_none()); + assert_eq!(grammar_for(Language::Yaml), None); + assert_eq!(grammar_for(Language::Svelte), Some(Language::TypeScript)); + } +} diff --git a/crates/codegraph-graph/Cargo.toml b/crates/codegraph-graph/Cargo.toml index 93140f44..4ad062c9 100644 --- a/crates/codegraph-graph/Cargo.toml +++ b/crates/codegraph-graph/Cargo.toml @@ -24,6 +24,7 @@ regex = "1.10" unicode-normalization = "0.1" [dev-dependencies] +tempfile = "3" serde_json = { workspace = true } codegraph-extract = { path = "../codegraph-extract" } codegraph-resolve = { path = "../codegraph-resolve" } diff --git a/crates/codegraph-graph/src/collation.rs b/crates/codegraph-graph/src/collation.rs new file mode 100644 index 00000000..782d04d7 --- /dev/null +++ b/crates/codegraph-graph/src/collation.rs @@ -0,0 +1,77 @@ +//! A deterministic stand-in for JavaScript's `String.prototype.localeCompare`, +//! so lists ported from upstream sort the way upstream's viewer sorts them. + +/// CLDR root order for ASCII punctuation and symbols. Whitespace sorts before +/// all of them; digits, then letters, after. +const PUNCTUATION_ORDER: &str = "_-,;:!?.'\"()[]{}@*/\\&#%`^+<=>|~$"; + +fn primary_weight(c: char) -> (u8, u32) { + if c.is_whitespace() { + (0, c as u32) + } else if let Some(at) = PUNCTUATION_ORDER.find(c) { + (1, at as u32) + } else if c.is_ascii_digit() { + (3, c as u32) + } else if c.is_alphanumeric() { + (4, c.to_lowercase().next().unwrap_or(c) as u32) + } else { + (2, c as u32) + } +} + +/// `String.prototype.localeCompare` (ICU root collation) for the paths and +/// names the viewer sorts: whitespace, punctuation, digits, then letters with +/// case ignored; then lower case before upper case; then code points. Accented +/// letters are not folded onto their base letter — they sort after `z`. +pub fn locale_compare(a: &str, b: &str) -> std::cmp::Ordering { + a.chars() + .map(primary_weight) + .cmp(b.chars().map(primary_weight)) + .then_with(|| { + a.chars() + .map(char::is_uppercase) + .cmp(b.chars().map(char::is_uppercase)) + }) + .then_with(|| a.cmp(b)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn locale_compare_follows_icu_root_order() { + // `node -e` with String.prototype.localeCompare, Node 26 / ICU. + let expected = [ + "_x", + "-x", + "#x", + "~x", + "1", + "a", + "A", + "a b", + "a_b", + "a-b", + "a.b", + "a/b", + "a1", + "a10", + "a2", + "ab", + "b", + "B", + "f", + "src/_a", + "src/(root files)", + "src/a", + "src/A", + "z", + "Z", + ]; + let mut shuffled = expected.to_vec(); + shuffled.reverse(); + shuffled.sort_by(|a, b| locale_compare(a, b)); + assert_eq!(shuffled, expected); + } +} diff --git a/crates/codegraph-graph/src/dead_code.rs b/crates/codegraph-graph/src/dead_code.rs new file mode 100644 index 00000000..a834b4bf --- /dev/null +++ b/crates/codegraph-graph/src/dead_code.rs @@ -0,0 +1,883 @@ +//! Dead code — one derivation of "nothing in this repository reaches here", +//! a port of upstream `src/graph/dead-code.ts` (`buildDeadCodeReport`, `v1.6.1`). +//! +//! `unreferenced` is a fact: no edge in the index other than the `contains` +//! edge from whatever holds it points at the symbol. `dead` is an inference on +//! top of that fact, and every step of it is subtractive — a candidate leaves +//! the list the moment there is any reason to believe something outside the +//! graph reaches it: exported or declared in a header; in a test, generated or +//! vendored file; abstract or declared on an interface; decorated; overriding +//! an ancestor's member (or an ancestor we cannot read); named something the +//! language calls by itself; in an in-file test scope; in a component file +//! whose markup can reference it invisibly; in a file nothing reaches; sharing +//! a name the resolver failed to follow, or a name another referenced symbol +//! carries; in a language the index records no export marker for; or written +//! more than once in a file that can reach it. +//! +//! Every subtraction is counted in [`DeadCodeExclusions`]: that is the sentence +//! under the list, because eight rows drawn from four thousand candidates mean +//! something different from eight drawn from nine. Query-time and read-only. + +use std::collections::{HashMap, HashSet}; +use std::path::{Path, PathBuf}; + +use codegraph_core::types::{EdgeKind, Node, NodeKind}; +use codegraph_store::Store; + +use crate::collation::locale_compare; +use crate::query::scoring::is_test_file; + +/// Kinds asked about by default: callables and types. Values are out on +/// purpose — their `references` coverage is the most language-dependent thing +/// in the resolver. +pub const DEAD_CODE_KINDS: &[NodeKind] = &[ + NodeKind::Function, + NodeKind::Method, + NodeKind::Class, + NodeKind::Component, + NodeKind::Interface, + NodeKind::Struct, + NodeKind::Trait, + NodeKind::Protocol, + NodeKind::Enum, + NodeKind::Union, + NodeKind::TypeAlias, +]; + +/// Kinds a `kinds=` request may ask for. +pub fn is_allowed_dead_code_kind(kind: NodeKind) -> bool { + DEAD_CODE_KINDS.contains(&kind) + || matches!( + kind, + NodeKind::Variable + | NodeKind::Constant + | NodeKind::Property + | NodeKind::Field + | NodeKind::EnumMember + | NodeKind::Namespace + | NodeKind::Module + ) +} + +/// Candidates pulled out of SQL before any exclusion runs. +pub const MAX_DEAD_CODE_CANDIDATES: i64 = 20_000; +/// Levels walked up looking for an ancestor that declares the same member. +pub const MAX_OVERRIDE_ANCESTOR_DEPTH: usize = 8; +/// Files read for the corroboration pass, and the biggest one read. A file +/// skipped for either reason counts as NOT corroborated, which drops the row. +pub const MAX_CORROBORATION_FILES: usize = 600; +pub const MAX_CORROBORATION_BYTES: u64 = 2_000_000; + +fn is_container_kind(kind: NodeKind) -> bool { + matches!( + kind, + NodeKind::Class + | NodeKind::Interface + | NodeKind::Struct + | NodeKind::Trait + | NodeKind::Protocol + | NodeKind::Enum + | NodeKind::Union + | NodeKind::TypeAlias + ) +} + +/// Container kinds whose members are declarations, never call targets. +fn is_declaration_container_kind(kind: NodeKind) -> bool { + matches!( + kind, + NodeKind::Interface | NodeKind::Trait | NodeKind::Protocol + ) +} + +/// Member kinds an override can be declared on. +fn is_overridable_kind(kind: NodeKind) -> bool { + matches!( + kind, + NodeKind::Method | NodeKind::Function | NodeKind::Property | NodeKind::Field + ) +} + +/// Names the language or runtime itself calls. +const IMPLICIT_ENTRY_NAMES: &[&str] = &[ + "constructor", + "main", + "init", + "deinit", + "finalize", + "destructor", + "dispose", + "drop", + "default", + "tostring", + "equals", + "gethashcode", + "hashcode", +]; + +/// Qualified-name segments that mean "inside a test scope the path does not +/// reveal" — a Rust `#[cfg(test)] mod tests`, a nested `Tests` class. +const TEST_SCOPE_SEGMENTS: &[&str] = &["test", "tests", "__tests__", "spec", "specs", "testing"]; + +/// Languages whose files are markup with a script block inside them: a handler +/// passed by reference in the template is not extracted. +const MARKUP_HOST_LANGUAGES: &[&str] = &[ + "svelte", + "vue", + "astro", + "liquid", + "html", + "razor", + "twig", + "blade", + "erb", + "handlebars", +]; + +/// Extensions whose contents are declarations for somebody else. +const HEADER_EXTENSIONS: &[&str] = &[ + ".h", ".hh", ".hpp", ".hxx", ".h++", ".inc", ".d.ts", ".d.mts", ".d.cts", ".pyi", ".pxd", +]; + +/// Directory names that mean "this code is carried, not written here", +/// matched as whole path segments. +const VENDOR_SEGMENTS: &[&str] = &[ + "vendor", + "vendored", + "third_party", + "third-party", + "thirdparty", + "external", + "externals", + "node_modules", + "bower_components", + "site-packages", + "godeps", + "pods", + ".venv", + "venv", +]; + +/// One symbol nothing reaches, and what it takes with it. +#[derive(Debug, Clone)] +pub struct DeadCodeEntry { + pub node: Node, + /// Unreferenced members folded in: a class nobody instantiates takes its + /// methods with it. + pub members: Vec, + /// Source lines the entry spans, members included. + pub lines: i64, + /// Only ever true when the caller asked for exported symbols. + pub exported: bool, +} + +/// How many candidates each rule removed. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct DeadCodeExclusions { + pub tests: usize, + pub generated: usize, + pub exported: usize, + pub exports_unknown: usize, + pub declarations: usize, + pub decorated: usize, + pub overriding: usize, + pub implicit: usize, + pub vendored: usize, + pub test_scope: usize, + pub markup: usize, + pub unreachable_file: usize, + pub unresolved_name: usize, + pub ambiguous_name: usize, + pub mentioned: usize, + pub unreadable: usize, + pub nested: usize, +} + +impl DeadCodeExclusions { + /// Every rule with its wire name (upstream's camelCase key), in rule order. + pub fn entries(&self) -> [(&'static str, usize); 17] { + [ + ("tests", self.tests), + ("generated", self.generated), + ("exported", self.exported), + ("exportsUnknown", self.exports_unknown), + ("declarations", self.declarations), + ("decorated", self.decorated), + ("overriding", self.overriding), + ("implicit", self.implicit), + ("vendored", self.vendored), + ("testScope", self.test_scope), + ("markup", self.markup), + ("unreachableFile", self.unreachable_file), + ("unresolvedName", self.unresolved_name), + ("ambiguousName", self.ambiguous_name), + ("mentioned", self.mentioned), + ("unreadable", self.unreadable), + ("nested", self.nested), + ] + } +} + +#[derive(Debug, Clone)] +pub struct DeadCodeReport { + /// Ranked, capped. + pub entries: Vec, + /// Entries before the limit — always the real number. + pub total: usize, + /// Symbols with no incoming reference at all, before any exclusion ran. + pub candidates: usize, + pub excluded: DeadCodeExclusions, + /// The kinds actually asked about. + pub kinds: Vec, + pub include_exported: bool, + /// The candidate scan stopped at [`MAX_DEAD_CODE_CANDIDATES`]. + pub bounded: bool, + /// Every surviving row was checked against the text of the files that can + /// reach it. False when no reader was given. + pub corroborated: bool, +} + +/// Reads a project-relative file for the corroboration pass; `None` for +/// anything unreadable. +pub type SourceReader<'a> = &'a dyn Fn(&str) -> Option; + +pub struct DeadCodeQuery<'a> { + pub kinds: Option>, + pub include_exported: bool, + pub include_tests: bool, + pub include_generated: bool, + pub limit: usize, + /// `None` turns the corroboration pass off, and the report then says so. + pub read_source: Option>, +} + +impl Default for DeadCodeQuery<'_> { + fn default() -> Self { + Self { + kinds: None, + include_exported: false, + include_tests: false, + include_generated: false, + limit: 200, + read_source: None, + } + } +} + +/// The dead code report: unreferenced symbols, minus every reason to doubt +/// them, plus a count of every doubt. +pub fn build_dead_code_report( + store: &Store, + query: &DeadCodeQuery<'_>, +) -> rusqlite::Result { + let kinds = normalize_kinds(query.kinds.as_deref()); + let include_exported = query.include_exported; + let include_tests = query.include_tests; + let include_generated = query.include_generated; + let limit = query.limit.max(1); + let mut excluded = DeadCodeExclusions::default(); + + let mut raw = store.unreferenced_nodes(&kinds, MAX_DEAD_CODE_CANDIDATES + 1)?; + let bounded = raw.len() as i64 > MAX_DEAD_CODE_CANDIDATES; + raw.truncate(MAX_DEAD_CODE_CANDIDATES as usize); + let candidates = raw; + + // Whether the exported filter can run at all, per language — skipped when + // the caller has already accepted outside-reachability. + let languages_with_exports = if include_exported { + HashSet::new() + } else { + let languages: Vec = candidates + .iter() + .map(|(node, _)| node.language.as_str().to_string()) + .collect(); + store.languages_with_exports(&languages)? + }; + + // ---- the cheap, per-row rules ------------------------------------------ + let mut surviving: Vec<&Node> = Vec::new(); + for (node, generated) in &candidates { + if !include_tests && is_test_file(&node.file_path) { + excluded.tests += 1; + } else if !include_generated && *generated { + excluded.generated += 1; + } else if !include_exported && (node.is_exported || is_header_file(&node.file_path)) { + excluded.exported += 1; + } else if !include_exported && !languages_with_exports.contains(node.language.as_str()) { + excluded.exports_unknown += 1; + } else if node.is_abstract { + excluded.declarations += 1; + } else if is_implicit_entry_name(&node.name) { + excluded.implicit += 1; + } else if is_vendored_path(&node.file_path) { + excluded.vendored += 1; + } else if !include_tests && is_test_scope(&node.qualified_name) { + excluded.test_scope += 1; + } else if MARKUP_HOST_LANGUAGES.contains(&node.language.as_str()) { + excluded.markup += 1; + } else { + surviving.push(node); + } + } + + // ---- the rules that need the graph ------------------------------------- + // A `decorates` edge runs FROM the decorated symbol to the decorator. + let surviving_ids: Vec = surviving.iter().map(|n| n.id.clone()).collect(); + let decorated: HashSet = store + .outgoing_edges_from(&surviving_ids, &[EdgeKind::Decorates])? + .into_iter() + .map(|e| e.source) + .collect(); + let containers = containers_of(store, &surviving)?; + let overriding = override_candidates(store, &surviving, &containers)?; + // A file nothing reaches is an island: its symbols' zero fan-in describes + // the file, not the symbol. + let unreachable_files = files_nothing_reaches(store, &surviving)?; + surviving.retain(|node| { + if unreachable_files.contains(&node.file_path) { + excluded.unreachable_file += 1; + false + } else { + true + } + }); + + let names: Vec = surviving.iter().map(|n| n.name.clone()).collect(); + let unresolved = store.unresolved_names_among(&names)?; + let ambiguous = store.ambiguous_referenced_names(&names)?; + + let mut kept: Vec<&Node> = Vec::new(); + for node in surviving { + if decorated.contains(&node.id) || !node.decorators.is_empty() { + excluded.decorated += 1; + } else if containers + .get(&node.id) + .is_some_and(|c| is_declaration_container_kind(c.kind)) + { + excluded.declarations += 1; + } else if overriding.contains(&node.id) { + excluded.overriding += 1; + } else if unresolved.contains(&node.name) { + excluded.unresolved_name += 1; + } else if ambiguous.contains(&node.name) { + excluded.ambiguous_name += 1; + } else { + kept.push(node); + } + } + + // ---- the file's own text has the last word ------------------------------ + let mut confirmed: Vec<&Node> = Vec::new(); + match query.read_source { + Some(read_source) => { + let mut sources: HashMap> = HashMap::new(); + let mut read = |file: &str| -> Option { + if !sources.contains_key(file) { + let text = if sources.len() >= MAX_CORROBORATION_FILES { + None + } else { + read_source(file) + }; + sources.insert(file.to_string(), text); + } + sources.get(file).cloned().flatten() + }; + // The declaring file plus everything the index says reaches into + // it — the same set a call could have come from. Once per file. + let mut scopes: HashMap> = HashMap::new(); + for node in &kept { + if !scopes.contains_key(&node.file_path) { + let mut scope = vec![node.file_path.clone()]; + let mut dependents = store.dependent_file_paths(&node.file_path)?; + dependents.sort(); + scope.extend(dependents); + scopes.insert(node.file_path.clone(), scope); + } + } + for node in kept { + let scope = scopes + .get(&node.file_path) + .cloned() + .unwrap_or_else(|| vec![node.file_path.clone()]); + let mut own_readable = false; + let mut mentions = 0; + for file in &scope { + let source = read(file); + if *file == node.file_path { + own_readable = source.is_some(); + } + let Some(source) = source else { + continue; + }; + mentions += mention_count(&source, &node.name, 2 - mentions); + if mentions >= 2 { + break; + } + } + // The declaration itself is one of the two mentions, so an + // unreadable declaring file makes the count meaningless. + if !own_readable { + excluded.unreadable += 1; + } else if mentions >= 2 { + excluded.mentioned += 1; + } else { + confirmed.push(node); + } + } + } + None => confirmed = kept, + } + + // ---- fold members into a container that is itself dead ------------------ + let kept_ids: HashSet<&str> = confirmed.iter().map(|n| n.id.as_str()).collect(); + let mut order: Vec = Vec::new(); + let mut entries: HashMap = HashMap::new(); + let mut pending: Vec<(&Node, String)> = Vec::new(); + for node in &confirmed { + if let Some(container) = containers.get(&node.id) + && kept_ids.contains(container.id.as_str()) + { + pending.push((node, container.id.clone())); + excluded.nested += 1; + continue; + } + order.push(node.id.clone()); + entries.insert( + node.id.clone(), + DeadCodeEntry { + node: (*node).clone(), + members: Vec::new(), + lines: (node.end_line - node.start_line + 1).max(1), + exported: node.is_exported, + }, + ); + } + for (member, container_id) in pending { + // A member whose container was itself folded away has no entry to hang + // off; it was still counted as nested. + if let Some(entry) = entries.get_mut(&container_id) { + entry.members.push(member.clone()); + } + } + let mut ranked: Vec = order + .into_iter() + .filter_map(|id| entries.remove(&id)) + .collect(); + for entry in &mut ranked { + entry.members.sort_by(|a, b| { + a.start_line + .cmp(&b.start_line) + .then_with(|| locale_compare(&a.name, &b.name)) + }); + } + // Biggest first; file and line break the tie so the order is stable. + ranked.sort_by(|a, b| { + b.lines + .cmp(&a.lines) + .then_with(|| locale_compare(&a.node.file_path, &b.node.file_path)) + .then_with(|| a.node.start_line.cmp(&b.node.start_line)) + }); + let total = ranked.len(); + ranked.truncate(limit); + + Ok(DeadCodeReport { + entries: ranked, + total, + candidates: candidates.len(), + excluded, + kinds, + include_exported, + bounded, + corroborated: query.read_source.is_some(), + }) +} + +/// A reader for callers with no chokepoint of their own: resolves against the +/// project root, refuses anything that escapes it, and skips files over the +/// corroboration size cap. +pub fn default_source_reader(project_root: &Path) -> impl Fn(&str) -> Option + use<> { + let root: PathBuf = project_root.to_path_buf(); + move |file_path: &str| { + let absolute = root.join(file_path); + let canonical = absolute.canonicalize().ok()?; + let canonical_root = root.canonicalize().ok()?; + if !canonical.starts_with(&canonical_root) { + return None; + } + let meta = std::fs::metadata(&canonical).ok()?; + if !meta.is_file() || meta.len() > MAX_CORROBORATION_BYTES { + return None; + } + std::fs::read_to_string(&canonical).ok() + } +} + +/// How many times `name` is written in `source` as a whole identifier, counting +/// no further than `stop_at`. Deliberately dumb: a mention in a comment or a +/// string is exactly what turns out to be a reflective call, and the count only +/// ever makes the report say LESS. +pub fn mention_count(source: &str, name: &str, stop_at: usize) -> usize { + if name.is_empty() || stop_at == 0 { + return 0; + } + let mut count = 0; + let mut from = 0; + while let Some(offset) = source[from..].find(name) { + let at = from + offset; + from = at + name.len(); + let before = source[..at].chars().next_back(); + let after = source[from..].chars().next(); + if !before.is_some_and(is_identifier_char) && !after.is_some_and(is_identifier_char) { + count += 1; + if count >= stop_at { + return count; + } + } + } + count +} + +fn is_identifier_char(c: char) -> bool { + c == '_' || c == '$' || c.is_alphabetic() || c.is_numeric() +} + +/// A file whose contents are declarations for somebody else. +pub fn is_header_file(file_path: &str) -> bool { + let lower = file_path.to_lowercase(); + HEADER_EXTENSIONS.iter().any(|ext| lower.ends_with(ext)) +} + +/// A qualified name that runs through a test scope. +pub fn is_test_scope(qualified_name: &str) -> bool { + qualified_name + .split(['.', ':', '/', '\\', '#', '>']) + .any(|segment| TEST_SCOPE_SEGMENTS.contains(&segment.to_lowercase().as_str())) +} + +/// Code the repository carries rather than owns. +pub fn is_vendored_path(file_path: &str) -> bool { + file_path + .replace('\\', "/") + .split('/') + .any(|segment| VENDOR_SEGMENTS.contains(&segment.to_lowercase().as_str())) +} + +/// Python and Ruby call `__enter__`, `__iter__`, `__init__` by protocol. +fn is_dunder(name: &str) -> bool { + name.len() > 4 + && name.starts_with("__") + && name.ends_with("__") + && name[2..name.len() - 2] + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'_') +} + +/// A name the language calls by itself, so no source file could name it. +pub fn is_implicit_entry_name(name: &str) -> bool { + is_dunder(name) || IMPLICIT_ENTRY_NAMES.contains(&name.to_lowercase().as_str()) +} + +fn normalize_kinds(requested: Option<&[NodeKind]>) -> Vec { + let Some(requested) = requested.filter(|r| !r.is_empty()) else { + return DEAD_CODE_KINDS.to_vec(); + }; + let mut kinds: Vec = Vec::new(); + for kind in requested { + if is_allowed_dead_code_kind(*kind) && !kinds.contains(kind) { + kinds.push(*kind); + } + } + if kinds.is_empty() { + DEAD_CODE_KINDS.to_vec() + } else { + kinds + } +} + +/// The type each member candidate is declared in. Only type-ish containers are +/// returned: a function's container is the file, which tells us nothing. +fn containers_of(store: &Store, nodes: &[&Node]) -> rusqlite::Result> { + let member_ids: Vec = nodes + .iter() + .filter(|n| is_overridable_kind(n.kind)) + .map(|n| n.id.clone()) + .collect(); + let mut out = HashMap::new(); + if member_ids.is_empty() { + return Ok(out); + } + let mut by_member: HashMap = HashMap::new(); + let mut edges = store.incoming_edges_to(&member_ids, &[EdgeKind::Contains])?; + // First container in a stable order when a member has several. + edges.sort_by(|a, b| { + a.target + .cmp(&b.target) + .then_with(|| a.source.cmp(&b.source)) + }); + for edge in edges { + by_member.entry(edge.target).or_insert(edge.source); + } + let container_ids: Vec = by_member.values().cloned().collect(); + let container_nodes = store.nodes_by_ids(&container_ids)?; + for (member_id, container_id) in by_member { + if let Some(container) = container_nodes.get(&container_id) + && is_container_kind(container.kind) + { + out.insert(member_id, container.clone()); + } + } + Ok(out) +} + +/// Which candidates override something — matched by NAME within a chain the +/// graph already links. An ancestor with no readable members, or one outside +/// the index (a failed `extends` / `implements` row, #1973), counts as a match: +/// "cannot tell" is answered with an exclusion. +fn override_candidates( + store: &Store, + nodes: &[&Node], + containers: &HashMap, +) -> rusqlite::Result> { + let mut dropped = HashSet::new(); + let mut container_ids: Vec = containers.values().map(|n| n.id.clone()).collect(); + container_ids.sort(); + container_ids.dedup(); + if container_ids.is_empty() { + return Ok(dropped); + } + + // Level-by-level upward walk over EVERY container at once. `reach` maps an + // ancestor back to the containers it is an ancestor of. + let mut reach: HashMap> = HashMap::new(); + let mut seen: HashMap> = container_ids + .iter() + .map(|id| (id.clone(), HashSet::from([id.clone()]))) + .collect(); + let mut frontier: Vec<(String, HashSet)> = container_ids + .iter() + .map(|id| (id.clone(), HashSet::from([id.clone()]))) + .collect(); + let mut depth = 0; + while depth < MAX_OVERRIDE_ANCESTOR_DEPTH && !frontier.is_empty() { + let roots_of: HashMap<&str, &HashSet> = + frontier.iter().map(|(id, r)| (id.as_str(), r)).collect(); + let ids: Vec = frontier.iter().map(|(id, _)| id.clone()).collect(); + let mut edges = + store.outgoing_edges_from(&ids, &[EdgeKind::Extends, EdgeKind::Implements])?; + edges.sort_by(|a, b| { + a.source + .cmp(&b.source) + .then_with(|| a.target.cmp(&b.target)) + }); + let mut next: Vec<(String, HashSet)> = Vec::new(); + let mut next_at: HashMap = HashMap::new(); + for edge in edges { + if edge.target == edge.source { + continue; + } + let Some(roots) = roots_of.get(edge.source.as_str()) else { + continue; + }; + let visited = seen.entry(edge.target.clone()).or_default(); + let known = reach.entry(edge.target.clone()).or_default(); + let mut merged: HashSet = HashSet::new(); + for root in roots.iter() { + if visited.insert(root.clone()) { + merged.insert(root.clone()); + known.insert(root.clone()); + } + } + if known.is_empty() { + reach.remove(&edge.target); + } + if merged.is_empty() { + continue; + } + match next_at.get(&edge.target) { + Some(&at) => next[at].1.extend(merged), + None => { + next_at.insert(edge.target.clone(), next.len()); + next.push((edge.target.clone(), merged)); + } + } + } + frontier = next; + depth += 1; + } + + let mut opaque: HashSet = HashSet::new(); + let mut probe: Vec = container_ids.clone(); + probe.extend(reach.keys().cloned()); + let external_based = store.unresolved_supertype_sources_among(&probe)?; + for id in &container_ids { + if external_based.contains(id) { + opaque.insert(id.clone()); + } + } + for (ancestor, roots) in &reach { + if external_based.contains(ancestor) { + opaque.extend(roots.iter().cloned()); + } + } + + // What each ancestor declares, and whether it declares anything at all. + let ancestor_ids: Vec = reach.keys().cloned().collect(); + let member_edges = store.outgoing_edges_from(&ancestor_ids, &[EdgeKind::Contains])?; + let mut member_ids_by_ancestor: HashMap<&str, Vec<&str>> = HashMap::new(); + for edge in &member_edges { + member_ids_by_ancestor + .entry(edge.source.as_str()) + .or_default() + .push(edge.target.as_str()); + } + let member_targets: Vec = member_edges.iter().map(|e| e.target.clone()).collect(); + let member_nodes = store.nodes_by_ids(&member_targets)?; + let mut names_by_container: HashMap> = HashMap::new(); + for (ancestor, roots) in &reach { + let names: Vec<&str> = member_ids_by_ancestor + .get(ancestor.as_str()) + .map(|ids| { + ids.iter() + .filter_map(|id| member_nodes.get(*id)) + .filter(|m| is_overridable_kind(m.kind)) + .map(|m| m.name.as_str()) + .collect() + }) + .unwrap_or_default(); + for root in roots { + if names.is_empty() { + opaque.insert(root.clone()); + continue; + } + names_by_container + .entry(root.clone()) + .or_default() + .extend(names.iter().map(|n| n.to_string())); + } + } + + for node in nodes { + let Some(container) = containers.get(&node.id) else { + continue; + }; + if opaque.contains(&container.id) + || names_by_container + .get(&container.id) + .is_some_and(|names| names.contains(&node.name)) + { + dropped.insert(node.id.clone()); + } + } + Ok(dropped) +} + +/// Files in the candidate set nothing else in the index reaches. +fn files_nothing_reaches(store: &Store, nodes: &[&Node]) -> rusqlite::Result> { + let mut paths: Vec = nodes.iter().map(|n| n.file_path.clone()).collect(); + paths.sort(); + paths.dedup(); + if paths.is_empty() { + return Ok(HashSet::new()); + } + let dependents: HashMap = + store.file_dependent_counts(&paths)?.into_iter().collect(); + Ok(paths + .into_iter() + .filter(|p| dependents.get(p).copied().unwrap_or(0) == 0) + .collect()) +} + +#[cfg(test)] +mod tests { + use super::*; + + // Upstream `__tests__/dead-code.test.ts` "the rules that are pure". + + #[test] + fn counts_whole_identifier_mentions_only() { + assert_eq!( + mention_count("const load = 1; loader(); reload();", "load", usize::MAX), + 1 + ); + assert_eq!(mention_count("a.load(); load();", "load", usize::MAX), 2); + assert_eq!(mention_count("nothing here", "load", usize::MAX), 0); + // Stops early: the caller only ever needs "one, or more than one". + assert_eq!(mention_count("x x x x x", "x", 2), 2); + } + + #[test] + fn matches_vendored_directories_as_whole_segments() { + assert!(is_vendored_path("vendor/lib/a.go")); + assert!(is_vendored_path("a/node_modules/b/c.js")); + assert!(!is_vendored_path("src/vendored-parser.ts")); + } + + #[test] + fn recognises_headers_as_declaration_surfaces() { + assert!(is_header_file("src/tree_sitter/parser.h")); + assert!(is_header_file("types/global.d.ts")); + assert!(!is_header_file("src/parser.c")); + } + + #[test] + fn recognises_a_test_scope_inside_a_file() { + assert!(is_test_scope("tests::row_sizes_match")); + assert!(is_test_scope("Fixtures.Tests.Helper")); + assert!(!is_test_scope("Latest.value")); + } + + #[test] + fn recognises_names_the_language_calls_by_itself() { + assert!(is_implicit_entry_name("constructor")); + assert!(is_implicit_entry_name("__enter__")); + assert!(is_implicit_entry_name("ToString")); + assert!(!is_implicit_entry_name("mainHandler")); + } + + #[test] + fn mentions_count_whole_identifiers_only() { + let source = + "fn greet() {}\nlet a = greet;\n// greeting\nregreet(); greet_x; $greet; greet2"; + assert_eq!(mention_count(source, "greet", usize::MAX), 2); + assert_eq!(mention_count(source, "greet", 1), 1); + assert_eq!(mention_count("été greetété greet", "greet", usize::MAX), 1); + assert_eq!(mention_count("", "x", 2), 0); + assert_eq!(mention_count("x", "", 2), 0); + } + + #[test] + fn the_cheap_rules_read_paths_and_names() { + assert!(is_header_file("include/a.H")); + assert!(is_header_file("types/index.d.ts")); + assert!(!is_header_file("src/a.ts")); + assert!(is_test_scope("crate::store::tests::helper")); + assert!(is_test_scope("Foo.Tests.Bar")); + assert!(!is_test_scope("crate::contest::run")); + assert!(is_vendored_path("third_party/zlib/inflate.c")); + assert!(is_vendored_path("a\\node_modules\\b.js")); + assert!(!is_vendored_path("src/vendored-parser.ts")); + assert!(is_implicit_entry_name("__enter__")); + assert!(is_implicit_entry_name("ToString")); + assert!(is_implicit_entry_name("main")); + assert!(!is_implicit_entry_name("____")); + assert!(!is_implicit_entry_name("__x")); + assert!(!is_implicit_entry_name("run")); + } + + #[test] + fn requested_kinds_are_filtered_and_deduplicated() { + assert_eq!(normalize_kinds(None), DEAD_CODE_KINDS.to_vec()); + assert_eq!( + normalize_kinds(Some(&[ + NodeKind::Variable, + NodeKind::Variable, + NodeKind::Import + ])), + vec![NodeKind::Variable] + ); + assert_eq!( + normalize_kinds(Some(&[NodeKind::Import])), + DEAD_CODE_KINDS.to_vec() + ); + } +} diff --git a/crates/codegraph-graph/src/flow_boundary.rs b/crates/codegraph-graph/src/flow_boundary.rs new file mode 100644 index 00000000..aeb61a17 --- /dev/null +++ b/crates/codegraph-graph/src/flow_boundary.rs @@ -0,0 +1,87 @@ +//! Where a flow stops: the calls out of the last symbol a path reached, split +//! into the ones the resolver was sure of and the name-only guesses under the +//! confidence floor that the search refused to follow. A port of upstream +//! `continuationsFrom` (`src/graph/dynamic-boundary-report.ts`, `v1.6.1`). +//! +//! The uncertain half is the honest one: an unfollowed guess left invisible +//! reads as "there is nothing here". + +use std::collections::{HashMap, HashSet}; + +use codegraph_core::types::{EdgeKind, Node}; +use codegraph_store::Store; + +/// Confidence below which a resolved edge is a name-only guess. +pub const UNCERTAIN_BELOW: f64 = 0.6; + +/// A call out of the stopping symbol, and how sure the resolver was. +#[derive(Debug, Clone)] +pub struct BoundaryContinuation { + pub node: Node, + pub line: Option, + pub confidence: Option, +} + +#[derive(Debug, Clone, Default)] +pub struct BoundaryContinuations { + pub resolved: Vec, + pub uncertain: Vec, +} + +/// The kinds that continue a flow (`navigates` has no Rust edge kind yet). +fn is_continuation_kind(kind: EdgeKind) -> bool { + matches!(kind, EdgeKind::Calls | EdgeKind::Instantiates) +} + +/// One row per target and bucket, in line order; `exclude` drops the symbols +/// already on the path. +pub fn continuations_from( + store: &Store, + node: &Node, + exclude: &HashSet, +) -> rusqlite::Result { + let mut resolved: Vec = Vec::new(); + let mut uncertain: Vec = Vec::new(); + let mut seen_resolved: HashSet = HashSet::new(); + let mut seen_uncertain: HashSet = HashSet::new(); + let edges = store.edges_by_source_kind(&node.id, None)?; + let targets: Vec = edges.iter().map(|e| e.target.clone()).collect(); + let nodes: HashMap = store.nodes_by_ids(&targets)?; + for edge in edges { + if !is_continuation_kind(edge.kind) + || edge.target == node.id + || exclude.contains(&edge.target) + { + continue; + } + let confidence = edge + .metadata + .as_ref() + .and_then(|m| m.get("confidence")) + .and_then(serde_json::Value::as_f64); + let is_uncertain = confidence.is_some_and(|c| c < UNCERTAIN_BELOW); + let (bucket, seen) = if is_uncertain { + (&mut uncertain, &mut seen_uncertain) + } else { + (&mut resolved, &mut seen_resolved) + }; + if seen.contains(&edge.target) { + continue; + } + let Some(target) = nodes.get(&edge.target) else { + continue; + }; + seen.insert(edge.target.clone()); + bucket.push(BoundaryContinuation { + node: target.clone(), + line: edge.line, + confidence, + }); + } + resolved.sort_by_key(|c| c.line.unwrap_or(0)); + uncertain.sort_by_key(|c| c.line.unwrap_or(0)); + Ok(BoundaryContinuations { + resolved, + uncertain, + }) +} diff --git a/crates/codegraph-graph/src/hierarchy.rs b/crates/codegraph-graph/src/hierarchy.rs new file mode 100644 index 00000000..19bee876 --- /dev/null +++ b/crates/codegraph-graph/src/hierarchy.rs @@ -0,0 +1,401 @@ +//! A type's supertypes and subtypes as the viewer draws them — a port of +//! upstream `src/graph/type-hierarchy.ts` (`buildTypeHierarchy`, `v1.6.1`). +//! +//! Supertypes walk `extends`/`implements` edges outward from the focus (nearest +//! first), subtypes walk them inward breadth-first so depth 1 is complete before +//! depth 2 starts, and both are bounded. Members of the focus that redeclare an +//! ancestor's member by name are marked as overrides — a name match inside a +//! chain the graph already links, not an `overrides` edge. + +use std::collections::{HashMap, HashSet}; + +use codegraph_core::types::{Edge, EdgeKind, Node, NodeKind}; +use codegraph_store::Store; + +/// Node kinds a hierarchy can hang off. +pub fn can_have_hierarchy(node: &Node) -> bool { + matches!( + node.kind, + NodeKind::Class + | NodeKind::Interface + | NodeKind::Struct + | NodeKind::Trait + | NodeKind::Protocol + | NodeKind::Enum + | NodeKind::TypeAlias + | NodeKind::Union + ) +} + +fn is_overridable(kind: NodeKind) -> bool { + matches!( + kind, + NodeKind::Method | NodeKind::Function | NodeKind::Property | NodeKind::Field + ) +} + +pub const MAX_ANCESTOR_DEPTH: usize = 8; +pub const MAX_DESCENDANT_DEPTH: usize = 6; +pub const MAX_DESCENDANTS: usize = 400; +const MAX_OVERRIDE_ANCESTORS: usize = 12; +/// Direct implementers at which a call through the type is "polymorphic". +pub const DISPATCH_MIN_IMPLEMENTERS: usize = 8; + +/// How a subtype is tied to its supertype. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HierarchyRelation { + Extends, + Implements, +} + +impl HierarchyRelation { + pub fn as_str(self) -> &'static str { + match self { + Self::Extends => "extends", + Self::Implements => "implements", + } + } + + fn of(edge: &Edge) -> Self { + if edge.kind == EdgeKind::Implements { + Self::Implements + } else { + Self::Extends + } + } +} + +#[derive(Debug, Clone)] +pub struct HierarchyEntry { + pub node: Node, + /// Steps from the focus; 1 = declared directly on the focus. + pub depth: usize, + /// The entry one step nearer the focus (the focus's own id at depth 1). + pub parent_id: String, + pub relation: HierarchyRelation, + /// The edge, oriented subtype → supertype as the code declares it. + pub edge: Edge, + /// Synthesized rather than parsed (provenance `heuristic`). + pub synthesized: bool, + /// Direct subtypes this entry has that are NOT in the returned set. + pub hidden_subtypes: usize, +} + +#[derive(Debug, Clone)] +pub struct OverrideMatch { + pub member_id: String, + pub base_id: String, + pub base_type_id: String, + pub base_type_name: String, + pub relation: HierarchyRelation, +} + +#[derive(Debug, Clone)] +pub struct TypeHierarchy { + pub focus: Node, + /// Supertypes, nearest first. + pub ancestors: Vec, + /// Subtypes, breadth-first. + pub descendants: Vec, + pub direct_subtypes: usize, + pub direct_implementers: usize, + pub bounded: bool, + pub polymorphic: bool, + /// Members of the focus that redeclare an ancestor's, by member id. + pub overrides: HashMap, +} + +fn locale_cmp(a: &str, b: &str) -> std::cmp::Ordering { + a.to_lowercase() + .cmp(&b.to_lowercase()) + .then_with(|| a.cmp(b)) +} + +fn sort_level(level: &mut [HierarchyEntry]) { + level.sort_by(|a, b| { + let rel = |r: HierarchyRelation| { + if r == HierarchyRelation::Extends { + 0 + } else { + 1 + } + }; + rel(a.relation) + .cmp(&rel(b.relation)) + .then_with(|| locale_cmp(&a.node.name, &b.node.name)) + .then_with(|| locale_cmp(&a.node.file_path, &b.node.file_path)) + .then_with(|| a.node.start_line.cmp(&b.node.start_line)) + }); +} + +fn hierarchy_edges(store: &Store, ids: &[String], up: bool) -> Vec { + let kinds = [EdgeKind::Extends, EdgeKind::Implements]; + let edges = if up { + store.outgoing_edges_from(ids, &kinds) + } else { + store.incoming_edges_to(ids, &kinds) + }; + edges.unwrap_or_default() +} + +fn to_entry(node: Node, depth: usize, parent_id: &str, edge: Edge) -> HierarchyEntry { + HierarchyEntry { + node, + depth, + parent_id: parent_id.to_string(), + relation: HierarchyRelation::of(&edge), + synthesized: edge.provenance.as_deref() == Some("heuristic"), + edge, + hidden_subtypes: 0, + } +} + +fn walk_ancestors(store: &Store, focus: &Node) -> Vec { + let mut out = Vec::new(); + let mut seen: HashSet = HashSet::from([focus.id.clone()]); + let mut frontier = vec![focus.id.clone()]; + for depth in 1..=MAX_ANCESTOR_DEPTH { + if frontier.is_empty() { + break; + } + let edges = hierarchy_edges(store, &frontier, true); + if edges.is_empty() { + break; + } + let targets: Vec = edges.iter().map(|e| e.target.clone()).collect(); + let nodes = store.nodes_by_ids(&targets).unwrap_or_default(); + let mut level = Vec::new(); + for edge in edges { + let Some(node) = nodes.get(&edge.target) else { + continue; + }; + if !seen.insert(node.id.clone()) { + continue; + } + let source = edge.source.clone(); + level.push(to_entry(node.clone(), depth, &source, edge)); + } + sort_level(&mut level); + frontier = level.iter().map(|e| e.node.id.clone()).collect(); + out.extend(level); + } + out +} + +struct Down { + entries: Vec, + direct_total: usize, + direct_implementers: usize, + bounded: bool, +} + +fn walk_descendants(store: &Store, focus: &Node) -> Down { + let mut entries: Vec = Vec::new(); + let mut index: HashMap = HashMap::new(); + let mut seen: HashSet = HashSet::from([focus.id.clone()]); + let mut frontier = vec![focus.id.clone()]; + let mut direct_total = 0; + let mut direct_implementers = 0; + let mut bounded = false; + + for depth in 1..=MAX_DESCENDANT_DEPTH { + if frontier.is_empty() { + break; + } + let edges = hierarchy_edges(store, &frontier, false); + if edges.is_empty() { + break; + } + let sources: Vec = edges.iter().map(|e| e.source.clone()).collect(); + let nodes = store.nodes_by_ids(&sources).unwrap_or_default(); + let mut level: Vec = Vec::new(); + let mut overflow: Vec<(String, usize)> = Vec::new(); + let mut level_seen: HashSet = HashSet::new(); + for edge in edges { + let Some(node) = nodes.get(&edge.source) else { + continue; + }; + if seen.contains(&node.id) { + continue; + } + if level_seen.contains(&node.id) { + // A second edge to an already-seen subtype: `extends` outranks + // `implements` (a class that both extends and implements). + if let Some(existing) = level.iter_mut().find(|e| e.node.id == node.id) + && existing.relation == HierarchyRelation::Implements + && edge.kind == EdgeKind::Extends + { + existing.relation = HierarchyRelation::Extends; + existing.synthesized = edge.provenance.as_deref() == Some("heuristic"); + existing.edge = edge; + } + continue; + } + if depth == 1 { + direct_total += 1; + if edge.kind == EdgeKind::Implements { + direct_implementers += 1; + } + } + if entries.len() + level.len() >= MAX_DESCENDANTS { + bounded = true; + match overflow.iter_mut().find(|(t, _)| *t == edge.target) { + Some((_, n)) => *n += 1, + None => overflow.push((edge.target.clone(), 1)), + } + level_seen.insert(node.id.clone()); + continue; + } + level_seen.insert(node.id.clone()); + let target = edge.target.clone(); + level.push(to_entry(node.clone(), depth, &target, edge)); + } + for entry in &level { + seen.insert(entry.node.id.clone()); + } + sort_level(&mut level); + for entry in level { + index.insert(entry.node.id.clone(), entries.len()); + entries.push(entry); + } + for (parent_id, count) in overflow { + if let Some(&at) = index.get(&parent_id) { + entries[at].hidden_subtypes += count; + } + } + if bounded { + break; + } + frontier = entries + .iter() + .filter(|e| e.depth == depth) + .map(|e| e.node.id.clone()) + .collect(); + if depth == MAX_DESCENDANT_DEPTH && !frontier.is_empty() { + for edge in hierarchy_edges(store, &frontier, false) { + if seen.contains(&edge.source) { + continue; + } + bounded = true; + if let Some(&at) = index.get(&edge.target) { + entries[at].hidden_subtypes += 1; + } + } + } + } + Down { + entries, + direct_total, + direct_implementers, + bounded, + } +} + +fn members_of(store: &Store, container_ids: &[String]) -> Vec<(Node, String)> { + if container_ids.is_empty() { + return Vec::new(); + } + let Ok(edges) = store.outgoing_edges_from(container_ids, &[EdgeKind::Contains]) else { + return Vec::new(); + }; + if edges.is_empty() { + return Vec::new(); + } + let targets: Vec = edges.iter().map(|e| e.target.clone()).collect(); + let nodes = store.nodes_by_ids(&targets).unwrap_or_default(); + let rank: HashMap<&str, usize> = container_ids + .iter() + .enumerate() + .map(|(i, id)| (id.as_str(), i)) + .collect(); + let mut out: Vec<(Node, String)> = edges + .iter() + .filter_map(|e| nodes.get(&e.target).map(|n| (n.clone(), e.source.clone()))) + .collect(); + out.sort_by(|a, b| { + rank.get(a.1.as_str()) + .unwrap_or(&0) + .cmp(rank.get(b.1.as_str()).unwrap_or(&0)) + .then_with(|| a.0.start_line.cmp(&b.0.start_line)) + }); + out +} + +fn match_overrides( + store: &Store, + focus: &Node, + ancestors: &[HierarchyEntry], +) -> HashMap { + let mut result = HashMap::new(); + if ancestors.is_empty() { + return result; + } + let own = members_of(store, std::slice::from_ref(&focus.id)); + if own.is_empty() { + return result; + } + let chain: Vec<&HierarchyEntry> = ancestors.iter().take(MAX_OVERRIDE_ANCESTORS).collect(); + let chain_ids: Vec = chain.iter().map(|a| a.node.id.clone()).collect(); + let base = members_of(store, &chain_ids); + if base.is_empty() { + return result; + } + let by_ancestor: HashMap<&str, &HierarchyEntry> = + chain.iter().map(|a| (a.node.id.as_str(), *a)).collect(); + // The nearest declaration of each name wins (members are ranked by chain order). + let mut by_name: HashMap<&str, (&Node, &str)> = HashMap::new(); + for (member, owner) in &base { + by_name + .entry(member.name.as_str()) + .or_insert((member, owner.as_str())); + } + for (member, _) in &own { + if !is_overridable(member.kind) { + continue; + } + let Some((base_member, owner_id)) = by_name.get(member.name.as_str()) else { + continue; + }; + if base_member.id == member.id { + continue; + } + let Some(owner) = by_ancestor.get(owner_id) else { + continue; + }; + result.insert( + member.id.clone(), + OverrideMatch { + member_id: member.id.clone(), + base_id: base_member.id.clone(), + base_type_id: owner.node.id.clone(), + base_type_name: owner.node.name.clone(), + relation: owner.relation, + }, + ); + } + result +} + +/// The hierarchy of `focus`, or `None` for a node that cannot have one or has +/// none at all. +pub fn build_type_hierarchy(store: &Store, focus: &Node) -> Option { + if !can_have_hierarchy(focus) { + return None; + } + let ancestors = walk_ancestors(store, focus); + let down = walk_descendants(store, focus); + if ancestors.is_empty() && down.entries.is_empty() { + return None; + } + let overrides = match_overrides(store, focus, &ancestors); + Some(TypeHierarchy { + focus: focus.clone(), + polymorphic: down.direct_implementers >= DISPATCH_MIN_IMPLEMENTERS, + ancestors, + descendants: down.entries, + direct_subtypes: down.direct_total, + direct_implementers: down.direct_implementers, + bounded: down.bounded, + overrides, + }) +} diff --git a/crates/codegraph-graph/src/lib.rs b/crates/codegraph-graph/src/lib.rs index c7627602..6248675b 100644 --- a/crates/codegraph-graph/src/lib.rs +++ b/crates/codegraph-graph/src/lib.rs @@ -1,7 +1,13 @@ //! codegraph-graph crate pub mod centrality; +pub mod collation; +pub mod dead_code; pub mod export; +pub mod flow_boundary; pub mod graph; +pub mod hierarchy; +pub mod named_symbol_flow; pub mod query; pub mod segment_match; pub mod segments; +pub mod symbol_lookup; diff --git a/crates/codegraph-graph/src/named_symbol_flow.rs b/crates/codegraph-graph/src/named_symbol_flow.rs new file mode 100644 index 00000000..772072de --- /dev/null +++ b/crates/codegraph-graph/src/named_symbol_flow.rs @@ -0,0 +1,728 @@ +//! The call path among a bag of named symbols — a port of upstream +//! `src/graph/named-symbol-flow.ts` (`resolveNamedSymbolFlow`, `v1.6.1`). +//! +//! Two modes, expressed as options rather than two implementations: +//! +//! - **`Named`** is explore's rule: every resolved symbol is both a possible +//! start and a possible end, at most ONE unnamed symbol may bridge two named +//! ones, and the LONGEST chain wins. The bridge cap is what keeps the search +//! off a god-function's fan-out — the naming is the evidence a hop is on topic. +//! - **`Directed`** is "how does X reach Y": both ends are pinned, so the search +//! bridges freely, walks from both ends at once, and the SHORTEST path wins. +//! +//! Overloads differ for the same reason: a bare ambiguous name in `Named` mode +//! is kept only where the query also names its container; in `Directed` mode +//! every candidate for both ends is tried and the pair that connects is the +//! answer. Explore's own Flow section (family F10) is not wired to this yet; +//! the viewer's `/api/flow` is its first caller. + +use std::collections::{HashMap, HashSet}; + +use codegraph_core::types::{Edge, EdgeKind, Node, NodeKind}; +use codegraph_store::Store; + +use crate::graph::{GraphTraverser, NodeEdge}; +use crate::query::scoring::is_test_file; +use crate::symbol_lookup::find_all_symbol_nodes; + +/// Chain length ceiling, in NODES. Explore's Flow section has always used 7. +pub const DEFAULT_MAX_HOPS: usize = 7; +/// Longer ceiling for a directed question: a real path from an entry point to +/// a storage primitive runs deeper than seven frames. +pub const DIRECTED_MAX_HOPS: usize = 12; +/// At most one consecutive UNNAMED hop may bridge two named symbols. +pub const DEFAULT_MAX_BRIDGE: usize = 1; + +const MAX_SEEDS: usize = 8; +const MAX_CANDIDATES_PER_TOKEN: usize = 6; +/// Candidates a DIRECTED endpoint keeps: `main` can have ten definitions, and +/// the one meant may sort seventh. +const MAX_CANDIDATES_DIRECTED: usize = 12; +const MAX_TOKENS: usize = 16; +const MAX_NAMED: usize = 40; +const MAX_DYN_NAMED: usize = 12; +const MAX_DYN_PER_TOKEN: usize = 4; +const NAMED_VISIT_CAP: usize = 1500; +const DIRECTED_VISIT_CAP: usize = 12_000; + +/// Node kinds that can sit on a call chain (upstream's `constructor` has no +/// Rust kind: constructors are `method` nodes). +pub fn is_flow_callable(kind: NodeKind) -> bool { + matches!( + kind, + NodeKind::Method | NodeKind::Function | NodeKind::Component | NodeKind::Route + ) +} + +/// Edge kinds a flow may ride (`navigates` has no Rust edge kind yet). +fn is_flow_edge(kind: EdgeKind) -> bool { + kind == EdgeKind::Calls +} + +/// Non-callable kinds a SYNTHESIZED edge may end at (an RTK thunk is a constant). +fn is_dyn_kind(kind: NodeKind) -> bool { + matches!( + kind, + NodeKind::Constant | NodeKind::Variable | NodeKind::Field | NodeKind::Property + ) +} + +const FILE_EXTENSIONS: &[&str] = &[ + "java", "kt", "kts", "ts", "tsx", "js", "jsx", "mjs", "cjs", "cs", "py", "go", "rb", "php", + "swift", "rs", "cpp", "cc", "cxx", "c", "h", "hpp", "scala", "lua", "dart", "vue", "svelte", + "astro", "erl", "hrl", +]; + +/// Strip ONE real file extension (case-insensitive) — `Class.method` is kept. +fn strip_file_extension(token: &str) -> &str { + if let Some((stem, ext)) = token.rsplit_once('.') + && FILE_EXTENSIONS.iter().any(|e| e.eq_ignore_ascii_case(ext)) + { + return stem; + } + token +} + +/// The token spelling [`flow_tokens`] would have produced for one word. +pub fn normalize_token(token: &str) -> String { + strip_file_extension(token).trim().to_string() +} + +/// `^[A-Za-z_$][\w$]*(?:(?:::|\.)[\w$]+)*$`, ASCII as in JavaScript. +fn is_symbol_shaped(token: &str) -> bool { + let b = token.as_bytes(); + let word = |c: u8| c.is_ascii_alphanumeric() || c == b'_' || c == b'$'; + let Some(&first) = b.first() else { + return false; + }; + if !(first.is_ascii_alphabetic() || first == b'_' || first == b'$') { + return false; + } + let mut i = 1; + while i < b.len() && word(b[i]) { + i += 1; + } + while i < b.len() { + if b[i] == b'.' { + i += 1; + } else if b[i] == b':' && b.get(i + 1) == Some(&b':') { + i += 2; + } else { + return false; + } + let start = i; + while i < b.len() && word(b[i]) { + i += 1; + } + if i == start { + return false; + } + } + true +} + +/// The symbol-shaped tokens of a query, deduped and capped. +pub fn flow_tokens(query: &str) -> Vec { + let mut out: Vec = Vec::new(); + for raw in query.split(|c: char| c.is_whitespace() || matches!(c, ',' | '(' | ')' | '[' | ']')) + { + let token = strip_file_extension(raw).trim(); + if token.len() >= 3 && is_symbol_shaped(token) && !out.iter().any(|t| t == token) { + out.push(token.to_string()); + } + } + out.truncate(MAX_TOKENS); + out +} + +/// A token is shape-precise when it looks like a symbol reference rather than +/// an English word that happened to exact-match a callable. +fn is_precise_token(token: &str) -> bool { + if token.contains(['.', '_', '$', '/']) || token.contains("::") { + return true; + } + let chars: Vec = token.chars().collect(); + chars + .windows(2) + .any(|w| w[0].is_ascii_lowercase() && w[1].is_ascii_uppercase()) + || chars.first().is_some_and(|c| c.is_ascii_uppercase()) +} + +/// An insertion-ordered map, as JavaScript's `Map` iterates. +#[derive(Debug, Clone)] +pub struct OrderedMap { + keys: Vec, + values: HashMap, +} + +impl Default for OrderedMap { + fn default() -> Self { + Self { + keys: Vec::new(), + values: HashMap::new(), + } + } +} + +impl OrderedMap { + /// `Map.set`: a new key goes last; an existing key keeps its place. + pub fn set(&mut self, key: String, value: V) { + if !self.values.contains_key(&key) { + self.keys.push(key.clone()); + } + self.values.insert(key, value); + } + pub fn get(&self, key: &str) -> Option<&V> { + self.values.get(key) + } + pub fn contains_key(&self, key: &str) -> bool { + self.values.contains_key(key) + } + pub fn len(&self) -> usize { + self.keys.len() + } + pub fn is_empty(&self) -> bool { + self.keys.is_empty() + } + pub fn keys(&self) -> impl Iterator { + self.keys.iter() + } + pub fn values(&self) -> impl Iterator { + self.keys.iter().filter_map(|k| self.values.get(k)) + } + pub fn iter(&self) -> impl Iterator { + self.keys + .iter() + .filter_map(|k| self.values.get(k).map(|v| (k, v))) + } +} + +#[derive(Debug, Clone)] +pub struct FlowStep { + pub node: Node, + /// The edge INTO this node from the previous step; `None` on the first. + pub edge: Option, +} + +#[derive(Debug, Clone)] +pub struct FlowChain { + pub steps: Vec, + /// For each node on the chain, the line where it calls the NEXT one. + pub call_sites: HashMap, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum FlowMode { + #[default] + Named, + Directed, +} + +#[derive(Debug, Clone, Default)] +pub struct NamedSymbolFlowOptions { + pub mode: FlowMode, + /// Required in directed mode: the token the path must start at. + pub from: Option, + /// Required in directed mode: the token the path must end at. + pub to: Option, + pub max_hops: Option, + /// Consecutive unnamed hops allowed; unbounded in directed mode. + pub max_bridge: Option, + /// Distinct chains to return. + pub max_chains: Option, +} + +#[derive(Debug, Clone, Default)] +pub struct NamedSymbolFlow { + /// The query's symbol tokens, in the order they were written. + pub tokens: Vec, + /// Every CALLABLE the tokens resolved to, by node id. + pub named: OrderedMap, + /// Non-callable endpoints of synthesized edges (RTK thunks and friends). + pub dyn_named: OrderedMap, + /// token → the node ids it resolved to. + pub token_nodes: OrderedMap>, + /// token → its whole same-name callable family, before the container filter. + pub token_family: OrderedMap>, + /// Ids whose token was a (near-)unique callable name — at most 3 defs. + pub unique_named_node_ids: HashSet, + /// Ids resolved from a shape-precise token. + pub precise_named_ids: HashSet, + /// Chains found, best first. Empty when nothing connects. + pub chains: Vec, +} + +/// Production code before test and fixture code, otherwise the index's order. +fn rank_for_directed(nodes: Vec) -> Vec { + let mut ranked = nodes; + ranked.sort_by_key(|n| is_test_file(&n.file_path)); + ranked +} + +fn has_heuristic_edge(store: &Store, id: &str) -> rusqlite::Result { + let heuristic = |e: &Edge| e.provenance.as_deref() == Some("heuristic"); + Ok(store.edges_by_target_kind(id, None)?.iter().any(heuristic) + || store.edges_by_source_kind(id, None)?.iter().any(heuristic)) +} + +fn container_segment(qualified_name: &str) -> String { + let lower = qualified_name.to_lowercase(); + let segs: Vec<&str> = lower + .split("::") + .flat_map(|s| s.split('.')) + .filter(|s| !s.is_empty()) + .collect(); + if segs.len() >= 2 { + segs[segs.len() - 2].to_string() + } else { + String::new() + } +} + +/// Resolve a query's tokens to nodes, with the overload rules in the module +/// header. No graph traversal happens here. +pub fn resolve_named_tokens( + store: &Store, + query: &str, + opts: &NamedSymbolFlowOptions, +) -> rusqlite::Result { + let directed = opts.mode == FlowMode::Directed; + let mut out = NamedSymbolFlow { + tokens: flow_tokens(query), + ..NamedSymbolFlow::default() + }; + if out.tokens.len() < 2 { + return Ok(out); + } + // Name SEGMENTS of every token: an ambiguous simple name is kept only + // where its container is itself named. + let mut seg_pool: HashSet = HashSet::new(); + for token in &out.tokens { + for seg in token.to_lowercase().split("::").flat_map(|s| s.split('.')) { + if !seg.is_empty() { + seg_pool.insert(seg.to_string()); + } + } + } + + let tokens = out.tokens.clone(); + for token in &tokens { + let hits = find_all_symbol_nodes(store, token)?; + let cands: Vec = hits + .iter() + .filter(|n| is_flow_callable(n.kind)) + .cloned() + .collect(); + out.token_family.set(token.clone(), cands.clone()); + let specific = cands.len() <= 3; + let pick: Vec = if specific || directed { + cands + } else { + cands + .into_iter() + .filter(|n| { + let container = container_segment(&n.qualified_name); + !container.is_empty() && seg_pool.contains(&container) + }) + .collect() + }; + let kept: Vec = if directed { + rank_for_directed(pick) + .into_iter() + .take(MAX_CANDIDATES_DIRECTED) + .collect() + } else { + pick.into_iter().take(MAX_CANDIDATES_PER_TOKEN).collect() + }; + out.token_nodes + .set(token.clone(), kept.iter().map(|n| n.id.clone()).collect()); + let precise = is_precise_token(token); + for node in kept { + if specific { + out.unique_named_node_ids.insert(node.id.clone()); + } + if precise { + out.precise_named_ids.insert(node.id.clone()); + } + out.named.set(node.id.clone(), node); + } + // Same token, non-callable synthesized endpoints — capped per token and + // gated on an actual heuristic edge so plain constants never qualify. + if out.dyn_named.len() < MAX_DYN_NAMED { + let mut token_dyn = 0; + for node in &hits { + if is_flow_callable(node.kind) + || !is_dyn_kind(node.kind) + || out.dyn_named.contains_key(&node.id) + { + continue; + } + if has_heuristic_edge(store, &node.id)? { + if precise { + out.precise_named_ids.insert(node.id.clone()); + } + out.dyn_named.set(node.id.clone(), node.clone()); + token_dyn += 1; + } + if out.dyn_named.len() >= MAX_DYN_NAMED || token_dyn >= MAX_DYN_PER_TOKEN { + break; + } + } + } + if out.named.len() > MAX_NAMED { + break; + } + } + Ok(out) +} + +fn call_sites_of(steps: &[FlowStep]) -> HashMap { + let mut sites = HashMap::new(); + for pair in steps.windows(2) { + if let Some(line) = pair[1].edge.as_ref().and_then(|e| e.line) + && line > 0 + { + sites.entry(pair[0].node.id.clone()).or_insert(line); + } + } + sites +} + +fn callees(traverser: &GraphTraverser<'_>, id: &str) -> rusqlite::Result> { + traverser.get_callees(id, 1) +} + +fn callers(traverser: &GraphTraverser<'_>, id: &str) -> rusqlite::Result> { + traverser.get_callers(id, 1) +} + +/// One BFS parent entry: where the walk came from, over which edge. +type Parent = HashMap, Option, Node)>; + +/// Breadth-first over `calls` edges — synthesized ones included. Every named +/// symbol is a destination; at most `max_bridge` unnamed symbols between two. +fn walk_calls( + traverser: &GraphTraverser<'_>, + seed: &Node, + named: &HashSet, + max_hops: usize, + max_bridge: usize, +) -> rusqlite::Result<(Parent, Vec)> { + let mut parent: Parent = HashMap::new(); + parent.insert(seed.id.clone(), (None, None, seed.clone())); + let mut queue: Vec<(String, usize, usize)> = vec![(seed.id.clone(), 0, 0)]; + let mut reached: Vec = Vec::new(); + let mut head = 0; + while head < queue.len() && parent.len() < NAMED_VISIT_CAP { + let (id, depth, streak) = queue[head].clone(); + head += 1; + if id != seed.id && named.contains(&id) { + reached.push(id.clone()); + } + if depth + 1 >= max_hops { + continue; + } + for c in callees(traverser, &id)? { + if !is_flow_edge(c.edge.kind) || parent.contains_key(&c.node.id) { + continue; + } + // A route node is a connector: crossing one costs no bridge budget. + let new_streak = if named.contains(&c.node.id) { + 0 + } else if c.node.kind == NodeKind::Route { + streak + } else { + streak + 1 + }; + if new_streak > max_bridge { + continue; + } + let next = c.node.id.clone(); + parent.insert(next.clone(), (Some(id.clone()), Some(c.edge), c.node)); + queue.push((next, depth + 1, new_streak)); + } + } + Ok((parent, reached)) +} + +fn chain_to(parent: &Parent, target: &str) -> Vec { + let mut steps = Vec::new(); + let mut cur = Some(target.to_string()); + while let Some(id) = cur { + let Some((prev, edge, node)) = parent.get(&id) else { + break; + }; + steps.push(FlowStep { + node: node.clone(), + edge: edge.clone(), + }); + cur = prev.clone(); + } + steps.reverse(); + steps +} + +/// A short call path from `seed` to any of `sinks`, searched from BOTH ends a +/// level at a time, always expanding the smaller frontier, stopping the moment +/// the sides share a node. Nothing claims the result is shortest — only that +/// the graph records it. +fn walk_bidirectional( + store: &Store, + traverser: &GraphTraverser<'_>, + seed: &Node, + sinks: &[String], + max_hops: usize, +) -> rusqlite::Result>> { + if sinks.contains(&seed.id) { + return Ok(None); + } + // forward: id → (prev, edge into it, node), in insertion order. + let mut forward_order: Vec = vec![seed.id.clone()]; + let mut forward: Parent = HashMap::new(); + forward.insert(seed.id.clone(), (None, None, seed.clone())); + // backward: id → (next towards the sink, the edge out of it); None AT a sink. + let mut backward: HashMap> = HashMap::new(); + let mut back_nodes: HashMap = HashMap::new(); + let mut front_f: Vec = vec![seed.clone()]; + let mut front_b: Vec = Vec::new(); + for id in sinks { + let Some(node) = store.node_by_id(id)? else { + continue; + }; + if backward.contains_key(id) { + continue; + } + backward.insert(id.clone(), None); + back_nodes.insert(id.clone(), node.clone()); + front_b.push(node); + } + if front_b.is_empty() { + return Ok(None); + } + + let max_edges = max_hops.saturating_sub(1).max(1); + for _ in 0..max_edges { + if front_f.len() <= front_b.len() { + if forward.len() > DIRECTED_VISIT_CAP { + break; + } + let mut next: Vec = Vec::new(); + for node in &front_f { + for c in callees(traverser, &node.id)? { + if !is_flow_edge(c.edge.kind) || forward.contains_key(&c.node.id) { + continue; + } + forward_order.push(c.node.id.clone()); + forward.insert( + c.node.id.clone(), + (Some(node.id.clone()), Some(c.edge), c.node.clone()), + ); + next.push(c.node); + } + } + if next.is_empty() { + break; + } + front_f = next; + } else { + if backward.len() > DIRECTED_VISIT_CAP { + break; + } + let mut next: Vec = Vec::new(); + for node in &front_b { + for c in callers(traverser, &node.id)? { + if !is_flow_edge(c.edge.kind) || backward.contains_key(&c.node.id) { + continue; + } + backward.insert(c.node.id.clone(), Some((node.id.clone(), c.edge))); + back_nodes.insert(c.node.id.clone(), c.node.clone()); + next.push(c.node); + } + } + if next.is_empty() { + break; + } + front_b = next; + } + + let Some(meet) = forward_order + .iter() + .find(|id| backward.contains_key(*id)) + .cloned() + else { + continue; + }; + // Forward half: seed → meet. + let mut steps = chain_to(&forward, &meet); + // Backward half: meet → sink. An entry holds the edge OUT of its node, + // which is the edge INTO the step after it. + let mut link = backward.get(&meet).cloned().flatten(); + while let Some((next_id, edge)) = link { + let Some(node) = back_nodes.get(&next_id) else { + break; + }; + steps.push(FlowStep { + node: node.clone(), + edge: Some(edge), + }); + link = backward.get(&next_id).cloned().flatten(); + } + let reaches_sink = steps + .last() + .is_some_and(|last| sinks.contains(&last.node.id)); + if steps.len() < 2 || !reaches_sink { + return Ok(None); + } + return Ok(if steps.len() <= max_hops { + Some(steps) + } else { + None + }); + } + Ok(None) +} + +/// The call path among a query's named symbols. Any lookup failure answers +/// with an empty flow, as upstream's does. +pub fn resolve_named_symbol_flow( + store: &Store, + query: &str, + opts: &NamedSymbolFlowOptions, +) -> NamedSymbolFlow { + resolve_flow(store, query, opts).unwrap_or_default() +} + +fn resolve_flow( + store: &Store, + query: &str, + opts: &NamedSymbolFlowOptions, +) -> rusqlite::Result { + let directed = opts.mode == FlowMode::Directed; + let mut flow = resolve_named_tokens(store, query, opts)?; + if flow.named.len() < 2 { + return Ok(flow); + } + let max_hops = opts.max_hops.unwrap_or(if directed { + DIRECTED_MAX_HOPS + } else { + DEFAULT_MAX_HOPS + }); + let max_bridge = opts.max_bridge.unwrap_or(if directed { + usize::MAX + } else { + DEFAULT_MAX_BRIDGE + }); + let max_chains = opts.max_chains.unwrap_or(1).max(1); + let named_ids: HashSet = flow.named.keys().cloned().collect(); + let traverser = GraphTraverser::new(store); + + let mut found: Vec> = Vec::new(); + if directed { + let from = normalize_token(opts.from.as_deref().unwrap_or("")); + let to = normalize_token(opts.to.as_deref().unwrap_or("")); + let from_ids = flow.token_nodes.get(&from).cloned().unwrap_or_default(); + let to_ids = flow.token_nodes.get(&to).cloned().unwrap_or_default(); + if from_ids.is_empty() || to_ids.is_empty() { + return Ok(flow); + } + // Every candidate start is searched: the one that connects IS the + // answer to which overload was meant. + for id in &from_ids { + let Some(seed) = flow.named.get(id) else { + continue; + }; + if let Some(steps) = walk_bidirectional(store, &traverser, seed, &to_ids, max_hops)? { + found.push(steps); + } + } + } else { + let seeds: Vec = flow.named.values().take(MAX_SEEDS).cloned().collect(); + for seed in &seeds { + let (parent, reached) = walk_calls(&traverser, seed, &named_ids, max_hops, max_bridge)?; + // Explore's rule: the DEEPEST named sink this seed can reach. + let mut deepest: Option> = None; + for id in &reached { + let steps = chain_to(&parent, id); + if deepest.as_ref().is_none_or(|d| steps.len() > d.len()) { + deepest = Some(steps); + } + } + if let Some(steps) = deepest { + found.push(steps); + } + } + } + if found.is_empty() { + return Ok(flow); + } + if directed { + found.sort_by_key(|steps| steps.len()); + } else { + found.sort_by_key(|steps| std::cmp::Reverse(steps.len())); + } + // A chain that is a shorter run along one already kept is the same answer + // twice; alternatives are for genuinely different routes. + let mut kept: Vec = Vec::new(); + for steps in found { + let key = steps + .iter() + .map(|s| s.node.id.as_str()) + .collect::>() + .join(">"); + if kept + .iter() + .any(|other| *other == key || other.contains(&key)) + { + continue; + } + kept.push(key); + let call_sites = call_sites_of(&steps); + flow.chains.push(FlowChain { steps, call_sites }); + if flow.chains.len() >= max_chains { + break; + } + } + Ok(flow) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn tokens_are_symbol_shaped_deduped_and_extension_free() { + assert_eq!( + flow_tokens("how does cmd_serve reach Store::open_for_read, (main.rs) main.rs ab x.y"), + vec![ + "how", + "does", + "cmd_serve", + "reach", + "Store::open_for_read", + "main", + "x.y" + ] + ); + assert_eq!(flow_tokens("a:b foo/bar 9abc abc.9"), vec!["abc.9"]); + assert_eq!(normalize_token("handler.TS"), "handler"); + // The extension is anchored at the very end, before the trim. + assert_eq!(normalize_token(" a.ts "), "a.ts"); + assert_eq!(normalize_token("Class.method"), "Class.method"); + } + + #[test] + fn precise_tokens_look_like_references() { + assert!(is_precise_token("Store::open")); + assert!(is_precise_token("openForRead")); + assert!(is_precise_token("Store")); + assert!(is_precise_token("open_for_read")); + assert!(!is_precise_token("open")); + } + + #[test] + fn the_container_segment_is_the_one_before_the_name() { + assert_eq!(container_segment("pkg::Store::open"), "store"); + assert_eq!(container_segment("Store.open"), "store"); + assert_eq!(container_segment("open"), ""); + } +} diff --git a/crates/codegraph-graph/src/query/scoring.rs b/crates/codegraph-graph/src/query/scoring.rs index 422b0c45..f88d356f 100644 --- a/crates/codegraph-graph/src/query/scoring.rs +++ b/crates/codegraph-graph/src/query/scoring.rs @@ -481,7 +481,17 @@ fn matches_non_production_dir(lower_path: &str) -> bool { }) } +/// Whether a path looks like test or other non-production code (examples, +/// samples, fixtures, benchmarks, demos) — upstream `isTestFile`, the wide +/// reading ranking uses. pub fn is_test_file(file_path: &str) -> bool { + is_test_path(file_path) || matches_non_production_dir(&file_path.to_lowercase()) +} + +/// Whether a path names a TEST suite — upstream `isTestPath`, the narrow half of +/// [`is_test_file`]: the file-name and directory conventions test suites use, +/// and nothing else (an example or a fixture is not a test). +pub fn is_test_path(file_path: &str) -> bool { let lower = file_path.to_lowercase(); let file_name = basename(file_path); let lower_name = file_name.to_lowercase(); @@ -510,7 +520,7 @@ pub fn is_test_file(file_path: &str) -> bool { return true; } - matches_non_production_dir(&lower) + false } fn matches_separator_test(lower_name: &str) -> bool { @@ -745,6 +755,27 @@ pub fn is_distinctive_identifier(token: &str) -> bool { mod tests { use super::*; + /// Upstream `__tests__/ui-entrypoints-api.test.ts` "keeps the suites and + /// drops the examples": the narrow predicate a "Tests" heading gathers by. + #[test] + fn keeps_the_suites_and_drops_the_examples() { + for suite in [ + "foo_test.go", + "src/foo.test.ts", + "src/__tests__/foo.ts", + "test/foo.rb", + "src/FooTest.java", + "app/src/jvmTest/Bar.kt", + ] { + assert!(is_test_path(suite), "{suite}"); + assert!(is_test_file(suite), "{suite}"); + } + for other in ["examples/demo.ts", "benchmarks/run.ts", "fixtures/a.ts"] { + assert!(is_test_file(other), "{other}"); + assert!(!is_test_path(other), "{other}"); + } + } + fn tokens(strs: &[&str]) -> HashSet { strs.iter().map(|s| (*s).to_string()).collect() } diff --git a/crates/codegraph-graph/src/symbol_lookup.rs b/crates/codegraph-graph/src/symbol_lookup.rs new file mode 100644 index 00000000..3ec40630 --- /dev/null +++ b/crates/codegraph-graph/src/symbol_lookup.rs @@ -0,0 +1,299 @@ +//! How a typed symbol name resolves to nodes — a port of upstream +//! `src/graph/symbol-lookup.ts` (`matchesSymbol`, `lastQualifierPart`) and the +//! node half of `findAllSymbols` (`src/graph/named-symbol-flow.ts`), `v1.6.1`. +//! +//! Bare names are exact-name lookups (#1473: a mistyped name must not silently +//! resolve to the top fuzzy hit). Qualified names go through full-text search +//! and are kept only when [`matches_symbol`] accepts them: against the +//! qualified name under both separator conventions, then — for languages whose +//! hierarchy lives in the path (Rust modules, Python packages) — against the +//! file path. +//! +//! The MCP engine still carries its own, older copy of this matcher; the +//! viewer's flow search uses this one, and explore's Flow section (family F10) +//! is where the two meet. + +use std::collections::HashSet; + +use codegraph_core::file_class::is_generated_file; +use codegraph_core::types::{Language, Node, NodeKind}; +use codegraph_store::Store; + +use crate::query::{SearchOptions, search_nodes}; + +/// Rust path prefixes that name no directory (`crate::x`, `super::y`). +pub const RUST_PATH_PREFIXES: &[&str] = &["crate", "super", "self"]; + +/// Does this query carry any scope qualifier at all? +pub fn is_qualified_symbol(symbol: &str) -> bool { + symbol.contains(['.', '/']) || symbol.contains("::") +} + +/// `fn/3` → `("fn", "3")`: Erlang's written arity, one to three digits. +fn split_arity(symbol: &str) -> Option<(&str, &str)> { + let (base, arity) = symbol.rsplit_once('/')?; + if base.is_empty() + || !(1..=3).contains(&arity.len()) + || !arity.bytes().all(|b| b.is_ascii_digit()) + { + return None; + } + Some((base, arity)) +} + +fn scope_parts(symbol: &str) -> Vec<&str> { + symbol + .split("::") + .flat_map(|part| part.split(['.', '/'])) + .filter(|p| !p.is_empty()) + .collect() +} + +/// The bare identifier at the end of a qualified query (arity spelling stripped). +pub fn last_qualifier_part(symbol: &str) -> &str { + let base = split_arity(symbol).map(|(base, _)| base).unwrap_or(symbol); + scope_parts(base).last().copied().unwrap_or(symbol) +} + +/// Every scope separator rewritten to `.`, so a query and a stored qualified +/// name written in different conventions compare directly. +fn canonical_scope(text: &str) -> String { + text.replace("::", ".").replace('/', ".") +} + +/// `name.replace(/\.[^.]+$/, '')`: the last extension, when one follows the dot. +fn strip_extension(name: &str) -> &str { + match name.rfind('.') { + Some(dot) if dot + 1 < name.len() => &name[..dot], + _ => name, + } +} + +/// Does `node` satisfy the user's symbol query? +pub fn matches_symbol(node: &Node, symbol: &str) -> bool { + let mut symbol = symbol; + // Erlang arity spelling: when the node's qualified name carries an arity + // (#1610) the written arity must match, and the rest runs arity-less. + if let Some((base, arity)) = split_arity(symbol) + && let Some((_, node_arity)) = split_arity(&node.qualified_name) + { + if node_arity != arity { + return false; + } + symbol = base; + } + + if node.name == symbol { + return true; + } + // File basename match ("product-card" matches "product-card.liquid"). + if node.kind == NodeKind::File && strip_extension(&node.name) == symbol { + return true; + } + if !is_qualified_symbol(symbol) { + return false; + } + let parts = scope_parts(symbol); + if parts.len() < 2 { + return false; + } + let last = parts[parts.len() - 1]; + if node.name != last { + return false; + } + // Stage 1: qualified-name containment under the extractor's `::` convention. + if node.qualified_name.contains(&parts.join("::")) { + return true; + } + // Stage 1b: boundary-aligned suffix under a canonical separator, for + // languages whose module names are themselves dotted. + let canonical_query = canonical_scope(symbol); + let canonical_node = canonical_scope(&node.qualified_name); + if canonical_node == canonical_query || canonical_node.ends_with(&format!(".{canonical_query}")) + { + return true; + } + // Stage 2: file-path containment (Rust modules, Python packages). + let hints: Vec<&str> = parts[..parts.len() - 1] + .iter() + .copied() + .filter(|p| !RUST_PATH_PREFIXES.contains(p)) + .collect(); + if hints.is_empty() { + return false; + } + let segments: Vec<&str> = node + .file_path + .split('/') + .filter(|s| !s.is_empty()) + .collect(); + hints.iter().all(|hint| { + segments + .iter() + .any(|seg| seg == hint || strip_extension(seg) == *hint) + }) +} + +/// A Nix option path: `xdg.configFile`, `launchd.user.agents`. +fn is_nix_option_path(symbol: &str) -> bool { + // JavaScript's `\w` without the `u` flag: ASCII word characters only. + let word_char = |c: char| c.is_ascii_alphanumeric() || c == '_' || c == '\'' || c == '-'; + let word = |s: &str| !s.is_empty() && s.chars().all(word_char); + let mut parts = symbol.split('.'); + let Some(head) = parts.next() else { + return false; + }; + let mut head_chars = head.chars(); + let head_ok = + head_chars.next().is_some_and(|c| c.is_ascii_lowercase()) && head_chars.all(word_char); + let rest: Vec<&str> = parts.collect(); + head_ok && !rest.is_empty() && rest.iter().all(|p| word(p)) +} + +fn by_position(nodes: &mut [Node]) { + nodes.sort_by(|a, b| { + a.file_path + .cmp(&b.file_path) + .then_with(|| a.start_line.cmp(&b.start_line)) + .then_with(|| a.id.cmp(&b.id)) + }); +} + +/// Every node a symbol name names (`findAllSymbols`' node list): exact names +/// for a bare name, matcher-filtered search hits for a qualified one, and with +/// several matches the hand-written ones before generated scaffolds. +pub fn find_all_symbol_nodes(store: &Store, symbol: &str) -> rusqlite::Result> { + // Nix option paths: the declaration is `options.` and config writes + // carry longer tails, so resolve the convention directly. + if is_nix_option_path(symbol) { + let mut hits = store.nodes_by_name(&format!("options.{symbol}"))?; + by_position(&mut hits); + let mut exact = store.nodes_by_name(symbol)?; + by_position(&mut exact); + hits.extend(exact); + hits.extend(store.nodes_by_name_prefix(&format!("{symbol}."), 12)?); + let mut seen = HashSet::new(); + let nix: Vec = hits + .into_iter() + .filter(|n| n.language == Language::Nix && seen.insert(n.id.clone())) + .take(10) + .collect(); + if !nix.is_empty() { + return Ok(nix); + } + } + + let no_tokens = HashSet::new(); + let exact: Vec = if !is_qualified_symbol(symbol) { + let mut nodes = store.nodes_by_name(symbol)?; + by_position(&mut nodes); + nodes + } else { + let options = SearchOptions { + limit: Some(50), + ..SearchOptions::default() + }; + let mut results = search_nodes(store, symbol, &options, &no_tokens)?; + if results.is_empty() { + let tail = last_qualifier_part(symbol); + if !tail.is_empty() && tail != symbol { + results = search_nodes(store, tail, &options, &no_tokens)?; + } + } + results + .into_iter() + .map(|r| r.node) + .filter(|n| matches_symbol(n, symbol)) + .collect() + }; + if exact.len() <= 1 { + return Ok(exact); + } + // Same generated-file down-rank as the single-symbol lookup: the + // hand-written implementations before the protobuf scaffold. + let paths: Vec = exact.iter().map(|n| n.file_path.clone()).collect(); + let flagged = store.generated_paths_among(&paths)?; + let mut ranked = exact; + ranked.sort_by_key(|n| flagged.contains(&n.file_path) || is_generated_file(&n.file_path)); + Ok(ranked) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn node(name: &str, qualified: &str, file: &str, kind: NodeKind) -> Node { + Node { + id: format!("{}:{name}", kind.as_str()), + kind, + name: name.to_string(), + qualified_name: qualified.to_string(), + file_path: file.to_string(), + language: Language::Rust, + start_line: 1, + end_line: 2, + start_column: 0, + end_column: 0, + docstring: None, + signature: None, + visibility: None, + is_exported: false, + is_async: false, + is_static: false, + is_abstract: false, + decorators: Vec::new(), + type_parameters: Vec::new(), + return_type: None, + updated_at: 0, + } + } + + #[test] + fn qualified_names_match_through_name_scope_or_path() { + let run = node( + "run", + "stage_apply::run", + "crates/x/src/stage_apply.rs", + NodeKind::Function, + ); + assert!(matches_symbol(&run, "run")); + assert!(matches_symbol(&run, "stage_apply::run")); + assert!(matches_symbol(&run, "crate::stage_apply::run")); + assert!(!matches_symbol(&run, "other::run")); + let elixir = node( + "group", + "AppWeb.Format::group", + "lib/format.ex", + NodeKind::Function, + ); + assert!(matches_symbol(&elixir, "AppWeb.Format.group")); + assert!(matches_symbol(&elixir, "Format.group")); + assert!(!matches_symbol(&elixir, "Web.Format.group")); + let file = node( + "product-card.liquid", + "", + "snippets/product-card.liquid", + NodeKind::File, + ); + assert!(matches_symbol(&file, "product-card")); + } + + #[test] + fn erlang_arity_must_match_when_the_node_has_one() { + let f = node("f", "mod::f/2", "src/mod.erl", NodeKind::Function); + assert!(matches_symbol(&f, "f/2")); + assert!(!matches_symbol(&f, "f/1")); + assert_eq!(last_qualifier_part("mod::fn/2"), "fn"); + assert_eq!(last_qualifier_part("a.b"), "b"); + assert_eq!(last_qualifier_part("plain"), "plain"); + } + + #[test] + fn nix_option_paths_are_recognised() { + assert!(is_nix_option_path("xdg.configFile")); + assert!(is_nix_option_path("launchd.user.agents")); + assert!(!is_nix_option_path("Foo.bar")); + assert!(!is_nix_option_path("plain")); + assert!(!is_nix_option_path("a..b")); + } +} diff --git a/crates/codegraph-graph/tests/hierarchy_bounds.rs b/crates/codegraph-graph/tests/hierarchy_bounds.rs new file mode 100644 index 00000000..f2ac9577 --- /dev/null +++ b/crates/codegraph-graph/tests/hierarchy_bounds.rs @@ -0,0 +1,79 @@ +//! The type hierarchy's descendant bound, against a synthetic store — upstream +//! `__tests__/type-hierarchy.test.ts` "the descendant bound" (`v1.6.1`). A fan +//! wide enough to hit the cap would be hundreds of files to index for one +//! assertion; the property pinned is arithmetic, not extraction. + +use codegraph_core::types::{Edge, EdgeKind, Language, Node, NodeKind}; +use codegraph_graph::hierarchy::{MAX_DESCENDANTS, build_type_hierarchy}; +use codegraph_store::Store; + +fn class(id: &str, name: &str) -> Node { + Node { + id: id.to_string(), + kind: NodeKind::Class, + name: name.to_string(), + qualified_name: name.to_string(), + file_path: format!("src/{name}.ts"), + language: Language::TypeScript, + start_line: 1, + end_line: 2, + start_column: 0, + end_column: 0, + docstring: None, + signature: None, + visibility: None, + is_exported: true, + is_async: false, + is_static: false, + is_abstract: false, + decorators: Vec::new(), + type_parameters: Vec::new(), + return_type: None, + updated_at: 1, + } +} + +/// A root class and `children` subclasses of it. +fn fan(children: usize) -> (tempfile::TempDir, Store, Node) { + let dir = tempfile::tempdir().unwrap(); + let mut store = Store::open(&dir.path().join("graph.db")).expect("open store"); + let root = class("class:root", "Root"); + let mut nodes = vec![root.clone()]; + let mut edges = Vec::new(); + for i in 0..children { + let child = class(&format!("class:c{i:04}"), &format!("Child{i:04}")); + edges.push(Edge { + id: None, + source: child.id.clone(), + target: root.id.clone(), + kind: EdgeKind::Extends, + metadata: None, + line: Some(1), + col: Some(0), + provenance: None, + }); + nodes.push(child); + } + store.upsert_nodes(&nodes).unwrap(); + store.insert_edges(&edges).unwrap(); + (dir, store, root) +} + +#[test] +fn stays_unbounded_under_the_cap() { + let (_dir, store, root) = fan(10); + let hierarchy = build_type_hierarchy(&store, &root).unwrap(); + assert_eq!(hierarchy.descendants.len(), 10); + assert_eq!(hierarchy.direct_subtypes, 10); + assert!(!hierarchy.bounded); +} + +#[test] +fn stops_materialising_rows_past_the_cap_but_keeps_the_direct_count_true() { + let (_dir, store, root) = fan(MAX_DESCENDANTS + 37); + let hierarchy = build_type_hierarchy(&store, &root).unwrap(); + assert_eq!(hierarchy.descendants.len(), MAX_DESCENDANTS); + // The number of subtypes is not the number of rows, and says so. + assert_eq!(hierarchy.direct_subtypes, MAX_DESCENDANTS + 37); + assert!(hierarchy.bounded); +} diff --git a/crates/codegraph-store/src/lib.rs b/crates/codegraph-store/src/lib.rs index 255dfcdd..0f3acc6a 100644 --- a/crates/codegraph-store/src/lib.rs +++ b/crates/codegraph-store/src/lib.rs @@ -10,6 +10,7 @@ pub mod schema; #[cfg(feature = "test-hooks")] pub mod test_support; pub mod uninit; +pub mod viewer; pub mod wsl_shared_index; pub use connection::{ @@ -34,4 +35,5 @@ pub use rebuild::{ resume_full_rebuild, }; pub use uninit::{UninitError, UninitOutcome, uninit_index, uninit_index_with_drain}; +pub use viewer::{FilePairEdgeRow, RoutingRow, TopCallingFile}; pub use wsl_shared_index::wsl_shared_index_guidance; diff --git a/crates/codegraph-store/src/queries.rs b/crates/codegraph-store/src/queries.rs index 52b73663..539d78ed 100644 --- a/crates/codegraph-store/src/queries.rs +++ b/crates/codegraph-store/src/queries.rs @@ -10,7 +10,7 @@ use serde_json::Value; use crate::connection::Store; -const SQLITE_PARAM_CHUNK_SIZE: usize = 500; +pub(crate) const SQLITE_PARAM_CHUNK_SIZE: usize = 500; /// Env var name (#1231): set to `1` to opt out of bulk-index WAL-checkpoint /// deferral and keep SQLite's default `wal_autocheckpoint` interval. @@ -1683,7 +1683,7 @@ where rows.collect() } -fn row_to_node(row: &Row<'_>) -> rusqlite::Result { +pub(crate) fn row_to_node(row: &Row<'_>) -> rusqlite::Result { Ok(Node { id: row.get("id")?, kind: parse_node_kind(row.get::<_, String>("kind")?)?, @@ -1709,7 +1709,7 @@ fn row_to_node(row: &Row<'_>) -> rusqlite::Result { }) } -fn row_to_edge(row: &Row<'_>) -> rusqlite::Result { +pub(crate) fn row_to_edge(row: &Row<'_>) -> rusqlite::Result { Ok(Edge { id: row.get("id")?, source: row.get("source")?, @@ -1722,7 +1722,7 @@ fn row_to_edge(row: &Row<'_>) -> rusqlite::Result { }) } -fn row_to_file(row: &Row<'_>) -> rusqlite::Result { +pub(crate) fn row_to_file(row: &Row<'_>) -> rusqlite::Result { Ok(FileRecord { path: row.get("path")?, content_hash: row.get("content_hash")?, @@ -1756,7 +1756,7 @@ fn millis_column(row: &Row<'_>, name: &str) -> rusqlite::Result { } } -fn row_to_unresolved_ref(row: &Row<'_>) -> rusqlite::Result { +pub(crate) fn row_to_unresolved_ref(row: &Row<'_>) -> rusqlite::Result { let raw_kind = row.get::<_, String>("reference_kind")?; let is_function_ref = raw_kind == "function_ref"; let reference_kind = if is_function_ref { diff --git a/crates/codegraph-store/src/viewer.rs b/crates/codegraph-store/src/viewer.rs new file mode 100644 index 00000000..490bb483 --- /dev/null +++ b/crates/codegraph-store/src/viewer.rs @@ -0,0 +1,729 @@ +//! Read-only aggregate queries the browser viewer (`codegraph-ui`) answers from. +//! +//! Each is a port of the upstream query the viewer's server calls +//! (`src/db/queries.ts`, `v1.6.1`), named after it. All of them are SELECTs: the +//! viewer never writes the index. + +use std::collections::{HashMap, HashSet}; + +use codegraph_core::types::{Edge, EdgeKind, Node, NodeKind, UnresolvedRef}; +use rusqlite::{ToSql, params}; + +use crate::connection::Store; +use crate::queries::{SQLITE_PARAM_CHUNK_SIZE, row_to_edge, row_to_node, row_to_unresolved_ref}; + +/// One executable root, as `getTopCallingFiles` ranks it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TopCallingFile { + pub node_id: String, + pub file_path: String, + pub calls: i64, + pub reaches: i64, + pub score: i64, +} + +/// One route → handler row, as `getRoutingManifest` selects it (before the +/// test/generated filter, which needs file classification the caller owns). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RoutingRow { + pub url: String, + pub route_id: String, + pub route_file: String, + pub route_line: i64, + pub handler: String, + pub handler_file: String, + pub handler_line: i64, + pub handler_kind: String, +} + +/// Cross-file edge counts by file pair, kind and symbol names — the input the +/// module map folds into module links (upstream `aggregateModuleGraph`'s single +/// pass, grouped at file rather than module level so no temp table is needed). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FilePairEdgeRow { + pub source_file: String, + pub target_file: String, + pub kind: EdgeKind, + pub from_name: String, + pub to_name: String, + /// Edges at or above the confidence floor. + pub count: i64, + /// Of those, the ones resolved by declaration (import / qualified name / + /// extends / implements / confident instance-method). + pub declared: i64, + /// Edges below the confidence floor. + pub uncertain: i64, +} + +fn placeholders(n: usize) -> String { + vec!["?"; n].join(",") +} + +fn unique(ids: &[String]) -> Vec { + let mut seen = std::collections::HashSet::new(); + ids.iter() + .filter(|id| seen.insert(id.as_str())) + .cloned() + .collect() +} + +/// The last segment of a (possibly qualified) reference name — what a plain +/// node name could match: `util.greet` → `greet`, `mod::fn` → `fn`, and for an +/// Erlang ref with a written arity (`mod::fn/2`) the arity-less name. +pub fn reference_name_tail(reference_name: &str) -> &str { + let base = match reference_name.rsplit_once('/') { + Some((head, arity)) + if !head.is_empty() + && (1..=3).contains(&arity.len()) + && arity.bytes().all(|b| b.is_ascii_digit()) => + { + head + } + _ => reference_name, + }; + match base.rfind(['.', ':']) { + Some(at) => &base[at + 1..], + None => base, + } +} + +impl Store { + /// `getStats`' edge histogram: `SELECT kind, COUNT(*) FROM edges GROUP BY kind`. + pub fn edge_counts_by_kind(&self) -> rusqlite::Result> { + let mut stmt = self + .connection() + .prepare("SELECT kind, COUNT(*) FROM edges GROUP BY kind ORDER BY kind")?; + let rows = stmt.query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?; + rows.collect() + } + + /// `getLastIndexedAt`: the most recent `indexed_at` across files. + pub fn last_indexed_at(&self) -> rusqlite::Result> { + self.connection() + .query_row("SELECT MAX(indexed_at) FROM files", [], |row| { + row.get::<_, Option>(0) + }) + .map(|v| v.map(|f| f as i64)) + } + + /// `getIndexRevision`: `(MAX(indexed_at), COUNT(*))` over files — moves on a + /// sync that only deletes, which `MAX` alone would miss. + pub fn index_revision(&self) -> rusqlite::Result<(Option, i64)> { + self.connection() + .query_row("SELECT MAX(indexed_at), COUNT(*) FROM files", [], |row| { + Ok((row.get::<_, Option>(0)?.map(|f| f as i64), row.get(1)?)) + }) + } + + /// `getFilesIndexedSince`: files re-indexed strictly after `since`, newest + /// first, capped at `limit`, with the true total. + pub fn files_indexed_since( + &self, + since: i64, + limit: i64, + ) -> rusqlite::Result<(Vec, i64)> { + let total: i64 = self.connection().query_row( + "SELECT COUNT(*) FROM files WHERE indexed_at > ?", + [since], + |row| row.get(0), + )?; + let mut stmt = self.connection().prepare( + "SELECT path FROM files WHERE indexed_at > ? ORDER BY indexed_at DESC, path LIMIT ?", + )?; + let rows = stmt.query_map(params![since, limit.max(0)], |row| row.get::<_, String>(0))?; + Ok((rows.collect::>>()?, total)) + } + + /// `countGeneratedFiles`: files flagged tool-generated. + pub fn generated_file_count(&self) -> rusqlite::Result { + self.connection().query_row( + "SELECT COUNT(*) FROM files WHERE generated = 1", + [], + |row| row.get(0), + ) + } + + /// `PRAGMA journal_mode`. + pub fn journal_mode(&self) -> rusqlite::Result { + self.connection() + .query_row("PRAGMA journal_mode", [], |row| row.get(0)) + } + + fn batched_edges( + &self, + column: &str, + ids: &[String], + kinds: &[EdgeKind], + ) -> rusqlite::Result> { + let ids = unique(ids); + let mut out = Vec::new(); + for chunk in ids.chunks(SQLITE_PARAM_CHUNK_SIZE) { + let mut sql = format!( + "SELECT * FROM edges WHERE {column} IN ({})", + placeholders(chunk.len()) + ); + let mut values: Vec<&dyn ToSql> = chunk.iter().map(|id| id as &dyn ToSql).collect(); + let kind_strs: Vec<&'static str> = kinds.iter().map(|k| k.as_str()).collect(); + if !kind_strs.is_empty() { + sql.push_str(&format!(" AND kind IN ({})", placeholders(kind_strs.len()))); + values.extend(kind_strs.iter().map(|k| k as &dyn ToSql)); + } + let mut stmt = self.connection().prepare(&sql)?; + let rows = stmt.query_map(values.as_slice(), row_to_edge)?; + for row in rows { + out.push(row?); + } + } + Ok(out) + } + + /// `getOutgoingEdgesFrom`: every edge out of any of `ids`, optionally only + /// of `kinds`, one batched query per chunk. + pub fn outgoing_edges_from( + &self, + ids: &[String], + kinds: &[EdgeKind], + ) -> rusqlite::Result> { + self.batched_edges("source", ids, kinds) + } + + /// `getIncomingEdgesTo`: the mirror of [`Self::outgoing_edges_from`]. + pub fn incoming_edges_to( + &self, + ids: &[String], + kinds: &[EdgeKind], + ) -> rusqlite::Result> { + self.batched_edges("target", ids, kinds) + } + + fn batched_counts( + &self, + column: &str, + ids: &[String], + ) -> rusqlite::Result> { + let ids = unique(ids); + let mut out = HashMap::new(); + for chunk in ids.chunks(SQLITE_PARAM_CHUNK_SIZE) { + let sql = format!( + "SELECT {column}, COUNT(*) FROM edges WHERE {column} IN ({}) GROUP BY {column}", + placeholders(chunk.len()) + ); + let values: Vec<&dyn ToSql> = chunk.iter().map(|id| id as &dyn ToSql).collect(); + let mut stmt = self.connection().prepare(&sql)?; + let rows = stmt.query_map(values.as_slice(), |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)) + })?; + for row in rows { + let (id, count) = row?; + out.insert(id, count); + } + } + Ok(out) + } + + /// `countIncomingEdges` (`getFanIn`): incoming edge count per id. + pub fn count_incoming_edges(&self, ids: &[String]) -> rusqlite::Result> { + self.batched_counts("target", ids) + } + + /// `countOutgoingEdges` (`getFanOut`): outgoing edge count per id. + pub fn count_outgoing_edges(&self, ids: &[String]) -> rusqlite::Result> { + self.batched_counts("source", ids) + } + + /// `getTopDependedOn`: the nodes with the most distinct dependents. + pub fn top_depended_on(&self, limit: i64) -> rusqlite::Result> { + if limit <= 0 { + return Ok(Vec::new()); + } + let mut stmt = self.connection().prepare( + "SELECT target, COUNT(DISTINCT source) AS dependents + FROM edges + WHERE kind != 'contains' AND source != target + GROUP BY target + ORDER BY dependents DESC, target + LIMIT ?", + )?; + let rows = stmt.query_map([limit], |row| Ok((row.get(0)?, row.get(1)?)))?; + rows.collect() + } + + /// `getTopCallingFiles`: files that run something at module level, ranked by + /// calls × (1 + distinct other files their symbols reach). + pub fn top_calling_files(&self, limit: i64) -> rusqlite::Result> { + if limit <= 0 { + return Ok(Vec::new()); + } + let mut stmt = self.connection().prepare( + "WITH tops AS ( + SELECT n.id AS file_id, n.id AS src + FROM nodes n + WHERE n.kind = 'file' + UNION ALL + SELECT c.source AS file_id, c.target AS src + FROM edges c + JOIN nodes f ON f.id = c.source + JOIN nodes v ON v.id = c.target + WHERE c.kind = 'contains' + AND f.kind = 'file' + AND v.kind IN ('variable', 'constant') + ), + runs AS ( + SELECT t.file_id AS id, COUNT(*) AS calls + FROM tops t + JOIN edges e ON e.source = t.src + WHERE e.kind IN ('calls', 'instantiates') + GROUP BY t.file_id + ), + cand AS ( + SELECT r.id AS id, n.file_path AS fp, r.calls AS calls + FROM runs r JOIN nodes n ON n.id = r.id + ), + wires AS ( + SELECT sn.file_path AS fp, COUNT(DISTINCT tn.file_path) AS reaches + FROM edges e + JOIN nodes sn ON sn.id = e.source + JOIN nodes tn ON tn.id = e.target + WHERE e.kind != 'contains' + AND sn.file_path <> tn.file_path + AND sn.file_path IN (SELECT fp FROM cand) + GROUP BY sn.file_path + ) + SELECT c.id, c.fp, c.calls, COALESCE(w.reaches, 0), + c.calls * (1 + COALESCE(w.reaches, 0)) AS score + FROM cand c LEFT JOIN wires w ON w.fp = c.fp + ORDER BY score DESC, c.calls DESC, c.fp + LIMIT ?", + )?; + let rows = stmt.query_map([limit], |row| { + Ok(TopCallingFile { + node_id: row.get(0)?, + file_path: row.get(1)?, + calls: row.get(2)?, + reaches: row.get(3)?, + score: row.get(4)?, + }) + })?; + rows.collect() + } + + /// `getFileDependentCounts`: distinct other files depending on each file. + pub fn file_dependent_counts(&self, paths: &[String]) -> rusqlite::Result> { + if paths.is_empty() { + return Ok(Vec::new()); + } + let json = serde_json::to_string(paths).unwrap_or_else(|_| "[]".to_string()); + let mut stmt = self.connection().prepare( + "SELECT tn.file_path, COUNT(DISTINCT sn.file_path) + FROM edges e + JOIN nodes tn ON tn.id = e.target + JOIN nodes sn ON sn.id = e.source + WHERE e.kind != 'contains' + AND tn.file_path IN (SELECT value FROM json_each(?)) + AND sn.file_path <> tn.file_path + GROUP BY tn.file_path + ORDER BY tn.file_path", + )?; + let rows = stmt.query_map([json], |row| Ok((row.get(0)?, row.get(1)?)))?; + rows.collect() + } + + /// `getFileReachCounts`: per file, the distinct other files its symbols reach + /// and the number of such references. + pub fn file_reach_counts(&self, paths: &[String]) -> rusqlite::Result> { + if paths.is_empty() { + return Ok(Vec::new()); + } + let json = serde_json::to_string(paths).unwrap_or_else(|_| "[]".to_string()); + let mut stmt = self.connection().prepare( + "SELECT sn.file_path, COUNT(DISTINCT tn.file_path), COUNT(*) + FROM nodes sn + JOIN edges e ON e.source = sn.id + JOIN nodes tn ON tn.id = e.target + WHERE sn.file_path IN (SELECT value FROM json_each(?)) + AND e.kind != 'contains' + AND tn.file_path <> sn.file_path + GROUP BY sn.file_path + ORDER BY sn.file_path", + )?; + let rows = stmt.query_map([json], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))?; + rows.collect() + } + + /// `getFileNodes`: the `file` nodes of `paths`. + pub fn file_nodes(&self, paths: &[String]) -> rusqlite::Result> { + if paths.is_empty() { + return Ok(Vec::new()); + } + let json = serde_json::to_string(paths).unwrap_or_else(|_| "[]".to_string()); + let mut stmt = self.connection().prepare( + "SELECT * FROM nodes WHERE kind = 'file' AND file_path IN (SELECT value FROM json_each(?)) ORDER BY file_path", + )?; + let rows = stmt.query_map([json], row_to_node)?; + rows.collect() + } + + /// `getNodesByFiles`: every node in any of `paths`, one chunked `IN` query + /// per chunk rather than one query per file (#1975). Ordered by file, line + /// and id, so a caller that keys these by location resolves ties the same + /// way on every run. + pub fn nodes_in_files(&self, paths: &[String]) -> rusqlite::Result> { + let paths = unique(paths); + let mut out = Vec::new(); + for chunk in paths.chunks(SQLITE_PARAM_CHUNK_SIZE) { + let sql = format!( + "SELECT * FROM nodes WHERE file_path IN ({}) ORDER BY file_path, start_line, id", + placeholders(chunk.len()) + ); + let values: Vec<&dyn ToSql> = chunk.iter().map(|p| p as &dyn ToSql).collect(); + let mut stmt = self.connection().prepare(&sql)?; + let rows = stmt.query_map(values.as_slice(), row_to_node)?; + for row in rows { + out.push(row?); + } + } + Ok(out) + } + + /// `getStats().nodesByKind[kind]`: how many nodes of one kind the index holds. + pub fn count_nodes_of_kind(&self, kind: NodeKind) -> rusqlite::Result { + self.connection().query_row( + "SELECT COUNT(*) FROM nodes WHERE kind = ?", + [kind.as_str()], + |row| row.get(0), + ) + } + + /// `getNodesByNamePrefix`: nodes whose name starts with `prefix`, by index + /// range scan (a LIKE would skip `idx_nodes_name`). + pub fn nodes_by_name_prefix(&self, prefix: &str, limit: i64) -> rusqlite::Result> { + let upper = format!("{prefix}\u{ffff}"); + let mut stmt = self.connection().prepare( + "SELECT * FROM nodes WHERE name >= ? AND name < ? ORDER BY name, id LIMIT ?", + )?; + let rows = stmt.query_map(params![prefix, upper, limit], row_to_node)?; + rows.collect() + } + + /// `getNodesByKind`, in upstream's canonical order. + pub fn nodes_of_kind_ordered(&self, kind: NodeKind) -> rusqlite::Result> { + let mut stmt = self + .connection() + .prepare("SELECT * FROM nodes WHERE kind = ? ORDER BY file_path, start_line, id")?; + let rows = stmt.query_map([kind.as_str()], row_to_node)?; + rows.collect() + } + + /// `getRoutingManifest`'s SELECT: route → handler rows, routes in file and + /// line order, at most `limit`. + pub fn routing_manifest_rows(&self, limit: i64) -> rusqlite::Result> { + let mut stmt = self.connection().prepare( + "SELECT r.name, r.id, r.file_path, r.start_line, + h.name, h.file_path, h.start_line, h.kind + FROM nodes r + JOIN edges e ON e.source = r.id + JOIN nodes h ON e.target = h.id + WHERE r.kind = 'route' + AND e.kind IN ('references', 'calls') + AND h.kind IN ('function', 'method', 'class', 'constant', 'variable') + ORDER BY r.file_path, r.start_line, r.id, h.id + LIMIT ?", + )?; + let rows = stmt.query_map([limit], |row| { + Ok(RoutingRow { + url: row.get(0)?, + route_id: row.get(1)?, + route_file: row.get(2)?, + route_line: row.get(3)?, + handler: row.get(4)?, + handler_file: row.get(5)?, + handler_line: row.get(6)?, + handler_kind: row.get(7)?, + }) + })?; + rows.collect() + } + + /// The module map's single pass, grouped by FILE pair, kind and the two + /// symbol names (upstream groups by module pair; the caller maps files to + /// modules and folds, which is the same sum without a temp table). + pub fn cross_file_edge_rows( + &self, + kinds: &[EdgeKind], + min_confidence: f64, + ) -> rusqlite::Result> { + if kinds.is_empty() { + return Ok(Vec::new()); + } + let confidence = "COALESCE(json_extract(e.metadata, '$.confidence'), 1)"; + let declared = format!( + "(json_extract(e.metadata, '$.resolvedBy') IN ('import', 'qualified-name') \ + OR e.kind IN ('extends', 'implements') \ + OR (json_extract(e.metadata, '$.resolvedBy') = 'instance-method' AND {confidence} >= 0.9))" + ); + let sql = format!( + "SELECT sn.file_path, tn.file_path, e.kind, sn.name, tn.name, + SUM(CASE WHEN {confidence} >= ?1 THEN 1 ELSE 0 END), + SUM(CASE WHEN {confidence} >= ?1 AND {declared} THEN 1 ELSE 0 END), + SUM(CASE WHEN {confidence} < ?1 THEN 1 ELSE 0 END) + FROM edges e + JOIN nodes sn ON sn.id = e.source + JOIN nodes tn ON tn.id = e.target + WHERE e.kind IN (SELECT value FROM json_each(?2)) + AND sn.file_path <> tn.file_path + GROUP BY sn.file_path, tn.file_path, e.kind, sn.name, tn.name + ORDER BY sn.file_path, tn.file_path, e.kind, sn.name, tn.name" + ); + let kinds_json = + serde_json::to_string(&kinds.iter().map(|k| k.as_str()).collect::>()) + .unwrap_or_else(|_| "[]".to_string()); + let mut stmt = self.connection().prepare(&sql)?; + let rows = stmt.query_map(params![min_confidence, kinds_json], |row| { + let kind: String = row.get(2)?; + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + kind, + row.get::<_, String>(3)?, + row.get::<_, String>(4)?, + row.get::<_, i64>(5)?, + row.get::<_, i64>(6)?, + row.get::<_, i64>(7)?, + )) + })?; + let mut out = Vec::new(); + for row in rows { + let (source_file, target_file, kind, from_name, to_name, count, declared, uncertain) = + row?; + let Some(kind) = EdgeKind::ALL.into_iter().find(|k| k.as_str() == kind) else { + continue; + }; + out.push(FilePairEdgeRow { + source_file, + target_file, + kind, + from_name, + to_name, + count, + declared, + uncertain, + }); + } + Ok(out) + } + + /// `getCrossFileDependencyPairs`: every ordered pair of distinct files where + /// one reaches into the other at or above `min_confidence`. + pub fn cross_file_dependency_pairs( + &self, + min_confidence: f64, + ) -> rusqlite::Result> { + let mut stmt = self.connection().prepare( + "SELECT DISTINCT sn.file_path, tn.file_path + FROM edges e + JOIN nodes sn ON sn.id = e.source + JOIN nodes tn ON tn.id = e.target + WHERE e.kind <> 'contains' + AND sn.file_path <> tn.file_path + AND COALESCE(json_extract(e.metadata, '$.confidence'), 1) >= ? + ORDER BY sn.file_path, tn.file_path", + )?; + let rows = stmt.query_map([min_confidence], |row| Ok((row.get(0)?, row.get(1)?)))?; + rows.collect() + } + + /// `getUnreferencedNodes`: symbols of `kinds` that no non-`contains` edge + /// points at, with their file's generated flag — the dead-code candidates. + pub fn unreferenced_nodes( + &self, + kinds: &[NodeKind], + limit: i64, + ) -> rusqlite::Result> { + if kinds.is_empty() || limit <= 0 { + return Ok(Vec::new()); + } + let kinds_json = + serde_json::to_string(&kinds.iter().map(|k| k.as_str()).collect::>()) + .unwrap_or_else(|_| "[]".to_string()); + let mut stmt = self.connection().prepare( + "SELECT n.*, COALESCE(f.generated, 0) AS file_generated + FROM nodes n + LEFT JOIN files f ON f.path = n.file_path + WHERE n.kind IN (SELECT value FROM json_each(?1)) + AND NOT EXISTS ( + SELECT 1 FROM edges e + WHERE e.target = n.id AND e.kind != 'contains' AND e.source != n.id + ) + ORDER BY n.file_path, n.start_line, n.name, n.id + LIMIT ?2", + )?; + let rows = stmt.query_map(params![kinds_json, limit], |row| { + Ok((row_to_node(row)?, row.get::<_, i64>("file_generated")? != 0)) + })?; + rows.collect() + } + + /// `getUnresolvedNamesAmong`: which of `names` the index holds an + /// unresolved reference to, matched on the reference name AND on its tail + /// (`util.greet` → `greet`, `mod::fn` → `fn`, Erlang `f/1` → `f`), because + /// the question is "could this be the name we failed to follow", and a maybe + /// counts as a yes. One pass over the distinct reference names (the schema + /// carries no tail column, and the name index covers the scan). + pub fn unresolved_names_among(&self, names: &[String]) -> rusqlite::Result> { + let wanted: HashSet<&str> = names + .iter() + .map(String::as_str) + .filter(|n| !n.is_empty()) + .collect(); + let mut found = HashSet::new(); + if wanted.is_empty() { + return Ok(found); + } + let mut stmt = self + .connection() + .prepare("SELECT DISTINCT reference_name FROM unresolved_refs")?; + let rows = stmt.query_map([], |row| row.get::<_, String>(0))?; + for row in rows { + let name = row?; + if wanted.contains(name.as_str()) { + found.insert(name.clone()); + } + let tail = reference_name_tail(&name); + if wanted.contains(tail) { + found.insert(tail.to_string()); + } + } + Ok(found) + } + + /// `getUnresolvedSupertypeSourcesAmong`: which of `ids` extend or implement + /// a type the resolver could not follow — the one record that an ancestor + /// outside the index exists (#1973). + pub fn unresolved_supertype_sources_among( + &self, + ids: &[String], + ) -> rusqlite::Result> { + let ids = unique(ids); + let mut found = HashSet::new(); + for chunk in ids.chunks(SQLITE_PARAM_CHUNK_SIZE) { + let sql = format!( + "SELECT DISTINCT from_node_id FROM unresolved_refs + WHERE from_node_id IN ({}) + AND reference_kind IN ('extends', 'implements')", + placeholders(chunk.len()) + ); + let values: Vec<&dyn ToSql> = chunk.iter().map(|id| id as &dyn ToSql).collect(); + let mut stmt = self.connection().prepare(&sql)?; + let rows = stmt.query_map(values.as_slice(), |row| row.get::<_, String>(0))?; + for row in rows { + found.insert(row?); + } + } + Ok(found) + } + + /// `getAmbiguousReferencedNames`: which of `names` are carried by MORE THAN + /// ONE symbol, at least one of which something points at (self-edges + /// counted — a self-edge is the fingerprint of a same-file mis-resolution). + pub fn ambiguous_referenced_names( + &self, + names: &[String], + ) -> rusqlite::Result> { + let names: Vec = unique(names) + .into_iter() + .filter(|n| !n.is_empty()) + .collect(); + let mut found = HashSet::new(); + for chunk in names.chunks(SQLITE_PARAM_CHUNK_SIZE) { + let sql = format!( + "SELECT name FROM ( + SELECT n.name AS name, + EXISTS (SELECT 1 FROM edges e WHERE e.target = n.id AND e.kind != 'contains') AS referenced + FROM nodes n + WHERE n.name IN ({}) + ) + GROUP BY name + HAVING COUNT(*) > 1 AND SUM(referenced) > 0", + placeholders(chunk.len()) + ); + let values: Vec<&dyn ToSql> = chunk.iter().map(|n| n as &dyn ToSql).collect(); + let mut stmt = self.connection().prepare(&sql)?; + let rows = stmt.query_map(values.as_slice(), |row| row.get::<_, String>(0))?; + for row in rows { + found.insert(row?); + } + } + Ok(found) + } + + /// `getLanguagesWithExports`: which of `languages` the index records an + /// export marker for. A language with none (Rust `pub` is not recorded; + /// Python and C have no such concept) is one the exported filter cannot run + /// on, and the caller must say so rather than claim outside-unreachability. + pub fn languages_with_exports( + &self, + languages: &[String], + ) -> rusqlite::Result> { + let languages: Vec = unique(languages) + .into_iter() + .filter(|l| !l.is_empty()) + .collect(); + let mut found = HashSet::new(); + for chunk in languages.chunks(SQLITE_PARAM_CHUNK_SIZE) { + let sql = format!( + "SELECT language, MAX(is_exported) FROM nodes WHERE language IN ({}) GROUP BY language", + placeholders(chunk.len()) + ); + let values: Vec<&dyn ToSql> = chunk.iter().map(|l| l as &dyn ToSql).collect(); + let mut stmt = self.connection().prepare(&sql)?; + let rows = stmt.query_map(values.as_slice(), |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, Option>(1)?)) + })?; + for row in rows { + let (language, any) = row?; + if any == Some(1) { + found.insert(language); + } + } + } + Ok(found) + } + + /// `getUnresolvedReferencesFrom`: references from one symbol that never + /// resolved to an indexed node, in line order. + pub fn unresolved_refs_from(&self, from_node_id: &str) -> rusqlite::Result> { + let mut stmt = self.connection().prepare( + "SELECT * FROM unresolved_refs WHERE from_node_id = ? ORDER BY line, col, id", + )?; + let rows = stmt.query_map([from_node_id], row_to_unresolved_ref)?; + rows.collect() + } + + /// `getUnresolvedReferencesInFile`: every unresolved reference in a file, in + /// line order, capped. + pub fn unresolved_refs_in_file( + &self, + file_path: &str, + limit: i64, + ) -> rusqlite::Result> { + let mut stmt = self.connection().prepare( + "SELECT * FROM unresolved_refs WHERE file_path = ? ORDER BY line, col, id LIMIT ?", + )?; + let rows = stmt.query_map(params![file_path, limit], row_to_unresolved_ref)?; + rows.collect() + } + + /// `findNodesByNameSubstring`: nodes whose name contains `substring` (SQLite + /// LIKE, ASCII-case-insensitive), shortest name first. + pub fn nodes_by_name_substring( + &self, + substring: &str, + limit: i64, + ) -> rusqlite::Result> { + let mut stmt = self.connection().prepare( + "SELECT * FROM nodes WHERE name LIKE ? ORDER BY length(name) ASC, name, file_path, start_line, id LIMIT ?", + )?; + let rows = stmt.query_map(params![format!("%{substring}%"), limit], row_to_node)?; + rows.collect() + } +} diff --git a/crates/codegraph-ui/Cargo.toml b/crates/codegraph-ui/Cargo.toml new file mode 100644 index 00000000..609b4b6f --- /dev/null +++ b/crates/codegraph-ui/Cargo.toml @@ -0,0 +1,45 @@ +[package] +name = "codegraph-ui" +version.workspace = true +publish.workspace = true +edition.workspace = true +license.workspace = true +authors.workspace = true +repository.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +rust-version.workspace = true +description = "Local browser viewer for CodeGraph-rs: the loopback HTTP API and the embedded web app." +readme = "../../README.md" +build = "build.rs" + +[dependencies] +serde = { workspace = true } +serde_json = { workspace = true } +anyhow = { workspace = true } +tracing = { workspace = true } +sha2 = { workspace = true } +notify = { workspace = true } +rusqlite = { workspace = true } +codegraph-core = { path = "../codegraph-core" } +codegraph-store = { path = "../codegraph-store" } +codegraph-graph = { path = "../codegraph-graph" } +codegraph-extract = { path = "../codegraph-extract" } +codegraph-resolve = { path = "../codegraph-resolve" } +codegraph-watch = { path = "../codegraph-watch" } +# The dynamic-dispatch scanner `codegraph_explore` already uses, for the Flow end cap. +codegraph-mcp = { path = "../codegraph-mcp" } +# The same HTTP stack the MCP streamable-HTTP server already links. +axum = { version = "0.8", default-features = false, features = ["http1", "tokio"] } +tokio = { version = "1", features = ["rt-multi-thread", "macros", "net", "sync", "time", "signal", "fs"] } +tokio-stream = { version = "0.1", features = ["sync"] } +futures-util = { version = "0.3", default-features = false } +percent-encoding = "2.3" +form_urlencoded = "1.2" +httpdate = "1" +http-body-util = "0.1" + +[dev-dependencies] +tower = { version = "0.5", features = ["util"] } +tempfile = "3" diff --git a/crates/codegraph-ui/build.rs b/crates/codegraph-ui/build.rs new file mode 100644 index 00000000..b09ba1b8 --- /dev/null +++ b/crates/codegraph-ui/build.rs @@ -0,0 +1,59 @@ +//! Embeds the committed viewer bundle (`viewer/`, written by `npm run build` in +//! `ui/`) into the library as a sorted `(path, bytes)` table. +//! +//! The bundle is committed so that `cargo build` and `cargo install --git` never +//! need Node; CI rebuilds it and fails when the committed copy differs. The table +//! is sorted by path so the generated source, and therefore the binary, does not +//! depend on directory iteration order. + +use std::fs; +use std::io::Write; +use std::path::{Path, PathBuf}; + +fn collect(root: &Path, dir: &Path, out: &mut Vec<(String, PathBuf)>) { + let Ok(entries) = fs::read_dir(dir) else { + return; + }; + for entry in entries.flatten() { + let path = entry.path(); + let Ok(kind) = entry.file_type() else { + continue; + }; + if kind.is_dir() { + collect(root, &path, out); + } else if kind.is_file() { + let rel = path + .strip_prefix(root) + .expect("walked path is under the viewer root") + .components() + .map(|c| c.as_os_str().to_string_lossy().into_owned()) + .collect::>() + .join("/"); + out.push((rel, path)); + } + } +} + +fn main() { + let manifest = PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR")); + let viewer = manifest.join("viewer"); + println!("cargo:rerun-if-changed=viewer"); + println!("cargo:rerun-if-changed=build.rs"); + + let mut files = Vec::new(); + collect(&viewer, &viewer, &mut files); + files.sort_by(|a, b| a.0.cmp(&b.0)); + + let out = PathBuf::from(std::env::var("OUT_DIR").expect("OUT_DIR")).join("viewer_assets.rs"); + let mut src = String::from("/// Every file of the embedded viewer bundle, sorted by path.\n"); + src.push_str("pub static VIEWER_FILES: &[(&str, &[u8])] = &[\n"); + for (rel, path) in &files { + src.push_str(&format!( + " ({rel:?}, include_bytes!({:?})),\n", + path.display().to_string() + )); + } + src.push_str("];\n"); + let mut f = fs::File::create(&out).expect("create viewer_assets.rs"); + f.write_all(src.as_bytes()).expect("write viewer_assets.rs"); +} diff --git a/crates/codegraph-ui/src/api/boundary.rs b/crates/codegraph-ui/src/api/boundary.rs new file mode 100644 index 00000000..313120f1 --- /dev/null +++ b/crates/codegraph-ui/src/api/boundary.rs @@ -0,0 +1,324 @@ +//! Where the static path ends at runtime dispatch — the structured half of +//! upstream `src/graph/dynamic-boundary-report.ts` (`findDynamicBoundaries`, +//! `shortlistBoundaryCandidates`, `v1.6.1`) that the Flow strip's end cap +//! renders. +//! +//! The scan is `codegraph_explore`'s own dispatch scanner +//! (`codegraph_mcp::dynamic_boundaries`), so the form, the key and the candidate +//! targets are the ones explore would print. Bodies are read through the +//! viewer's chokepoint, and a file that drifted since the index is skipped: its +//! recorded line ranges no longer name the body. + +use std::collections::HashSet; + +use codegraph_core::types::{EdgeKind, Node, NodeKind}; +use codegraph_graph::named_symbol_flow::OrderedMap; +use codegraph_graph::query::{SearchOptions, search_nodes}; +use codegraph_mcp::dynamic_boundaries::{BoundaryMatch, scan_dynamic_dispatch}; + +use super::Ctx; +use super::source::{ + find_indexed_file, has_drifted_on_disk, max_file_size, read_indexed_file_text, +}; + +/// Bodies scanned per question, and the characters read across them. +const MAX_SCAN: usize = 8; +const MAX_TOTAL_CHARS: usize = 200_000; +/// Candidates per site, and the search hits the shortlist draws from. +const MAX_CANDIDATES: usize = 4; +const CANDIDATE_SEARCH_LIMIT: i64 = 12; +/// The biggest file a body is read from. +const MAX_BODY_FILE_BYTES: u64 = 8 * 1024 * 1024; + +#[derive(Debug, Clone)] +pub struct BoundaryCandidate { + pub node: Node, + /// Usually the qualified name; for a typed-bus key, `Class.handlerMethod`. + pub display: String, + /// The reader already named this symbol. + pub named: bool, +} + +#[derive(Debug, Clone)] +pub struct BoundarySite { + pub matched: BoundaryMatch, + pub candidates: Vec, + /// Why there is no shortlist, when a key was visible but too generic. + pub candidate_note: Option, +} + +#[derive(Debug, Clone)] +pub struct NodeBoundary { + pub node: Node, + pub sites: Vec, +} + +fn is_candidate_kind(kind: NodeKind) -> bool { + matches!( + kind, + NodeKind::Method | NodeKind::Function | NodeKind::Component | NodeKind::Class + ) +} + +/// `/^(handle|handleAsync|execute|executeAsync|consume|consumeAsync|run|__invoke)$/i`. +fn is_handler_method(name: &str) -> bool { + matches!( + name.to_ascii_lowercase().as_str(), + "handle" + | "handleasync" + | "execute" + | "executeasync" + | "consume" + | "consumeasync" + | "run" + | "__invoke" + ) +} + +/// Lower case, ASCII letters and digits only. +fn normalize_name(text: &str) -> String { + text.to_lowercase() + .chars() + .filter(|c| c.is_ascii_lowercase() || c.is_ascii_digit()) + .collect() +} + +/// The text of `node`'s body, when its file is indexed, readable and current. +fn body_of(ctx: &Ctx<'_>, node: &Node) -> Option { + let (record, stored) = find_indexed_file(ctx, &node.file_path).ok()??; + if has_drifted_on_disk(ctx, &stored, &record, max_file_size(ctx)) { + return None; + } + let content = read_indexed_file_text(ctx, &stored, MAX_BODY_FILE_BYTES)?; + let lines: Vec<&str> = content.split('\n').collect(); + let start = usize::try_from(node.start_line - 1).ok()?.min(lines.len()); + let end = usize::try_from(node.end_line).ok()?.min(lines.len()); + Some(lines[start..end.max(start)].join("\n")) +} + +/// Scan `scan_list`'s bodies, in order, for dispatch sites — at most +/// `max_sites` across all of them. +pub fn find_dynamic_boundaries( + ctx: &Ctx<'_>, + scan_list: &[Node], + named: &OrderedMap, + max_sites: usize, +) -> Vec { + let mut out: Vec = Vec::new(); + let mut seen_node: HashSet = HashSet::new(); + let mut seen_site: HashSet = HashSet::new(); + let mut sites = 0; + let mut scanned = 0; + let mut chars_scanned = 0; + for node in scan_list { + if sites >= max_sites || scanned >= MAX_SCAN || chars_scanned > MAX_TOTAL_CHARS { + break; + } + if seen_node.contains(&node.id) || node.start_line <= 0 || node.end_line <= 0 { + continue; + } + seen_node.insert(node.id.clone()); + let Some(body) = body_of(ctx, node) else { + continue; + }; + scanned += 1; + chars_scanned += body.encode_utf16().count(); + let mut found: Vec = Vec::new(); + for matched in scan_dynamic_dispatch(&body, node.language.as_str(), node.start_line) { + if sites >= max_sites { + break; + } + if !seen_site.insert(format!( + "{}:{}:{}", + node.file_path, matched.line, matched.form + )) { + continue; + } + let (candidates, candidate_note) = match &matched.key { + Some(key) => { + shortlist_boundary_candidates(ctx, key, matched.key_is_type, named, &node.id) + } + None => (Vec::new(), None), + }; + found.push(BoundarySite { + matched, + candidates, + candidate_note, + }); + sites += 1; + } + if !found.is_empty() { + out.push(NodeBoundary { + node: node.clone(), + sites: found, + }); + } + } + out +} + +/// Runtime targets for a dispatch key: the conventional exact names first +/// (`save` → `onSave` / `handleSave`; `CreateCmd` → `CreateCmdHandler`), then +/// full-text search, kept where one normalised name contains the other. +pub fn shortlist_boundary_candidates( + ctx: &Ctx<'_>, + key: &str, + key_is_type: bool, + named: &OrderedMap, + self_id: &str, +) -> (Vec, Option) { + let key_norm = normalize_name(key); + if key_norm.len() < 3 { + return (Vec::new(), None); + } + let mut cands: Vec = Vec::new(); + let mut consider = |n: &Node| { + if n.id == self_id || !is_candidate_kind(n.kind) || cands.iter().any(|c| c.id == n.id) { + return; + } + let name_norm = normalize_name(&n.name); + if name_norm.len() < 3 || (!name_norm.contains(&key_norm) && !key_norm.contains(&name_norm)) + { + return; + } + cands.push(n.clone()); + }; + + let mut chars = key.chars(); + let cap = match chars.next() { + Some(first) => format!("{}{}", first.to_uppercase(), chars.as_str()), + None => String::new(), + }; + let probes: Vec = if key_is_type { + vec![format!("{key}Handler"), key.to_string()] + } else { + vec![ + key.to_string(), + format!("on{cap}"), + format!("handle{cap}"), + format!("{key}Handler"), + format!("handle_{key}"), + ] + }; + for probe in &probes { + if let Ok(mut nodes) = ctx.store.nodes_by_name(probe) { + nodes.sort_by(|a, b| { + a.file_path + .cmp(&b.file_path) + .then_with(|| a.start_line.cmp(&b.start_line)) + .then_with(|| a.id.cmp(&b.id)) + }); + for n in &nodes { + consider(n); + } + } + } + let mut raw = 0; + let options = SearchOptions { + limit: Some(CANDIDATE_SEARCH_LIMIT), + ..SearchOptions::default() + }; + if let Ok(results) = search_nodes(ctx.store, key, &options, &HashSet::new()) { + raw = results.len(); + for r in &results { + consider(&r.node); + } + } + + if cands.is_empty() { + let generic = raw as i64 >= CANDIDATE_SEARCH_LIMIT && key.encode_utf16().count() < 5; + let note = + generic.then(|| format!("key `{key}` is too generic to shortlist ({raw}+ matches)")); + return (Vec::new(), note); + } + + // A constructor candidate duplicates its class (constructors are METHOD + // nodes named like the class) — keep the class. + let class_key: HashSet = cands + .iter() + .filter(|n| n.kind == NodeKind::Class) + .map(|n| format!("{}|{}", n.name, n.file_path)) + .collect(); + // The flow's named set holds callables only, so a class whose METHOD the + // reader named still counts as named — transfer the mark by name. + let named_names: HashSet<&str> = named.values().map(|n| n.name.as_str()).collect(); + let is_named = |n: &Node| named.contains_key(&n.id) || named_names.contains(n.name.as_str()); + + let mut list: Vec = cands + .into_iter() + .filter(|n| { + !(n.kind != NodeKind::Class + && class_key.contains(&format!("{}|{}", n.name, n.file_path))) + }) + .collect(); + list.sort_by_key(|n| !is_named(n)); + list.truncate(MAX_CANDIDATES); + let candidates = list + .into_iter() + .map(|n| { + let named = is_named(&n); + // Typed-bus convention: the runtime target is the class's + // Handle/Execute/Consume method — name that exact node. + if key_is_type + && n.kind == NodeKind::Class + && let Some(method) = handler_method_of(ctx, &n) + { + return BoundaryCandidate { + display: format!("{}.{}", n.name, method.name), + node: method, + named, + }; + } + BoundaryCandidate { + display: if n.qualified_name.is_empty() { + n.name.clone() + } else { + n.qualified_name.clone() + }, + node: n, + named, + } + }) + .collect(); + (candidates, None) +} + +fn handler_method_of(ctx: &Ctx<'_>, class: &Node) -> Option { + let edges = ctx + .store + .edges_by_source_kind(&class.id, Some(EdgeKind::Contains)) + .ok()?; + let ids: Vec = edges.iter().map(|e| e.target.clone()).collect(); + let nodes = ctx.store.nodes_by_ids(&ids).ok()?; + edges + .iter() + .filter_map(|e| nodes.get(&e.target)) + .find(|c| c.kind == NodeKind::Method && is_handler_method(&c.name)) + .cloned() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn names_normalise_to_ascii_letters_and_digits() { + assert_eq!(normalize_name("handle_Save-2"), "handlesave2"); + assert_eq!(normalize_name("été"), "t"); + } + + #[test] + fn handler_methods_follow_the_typed_bus_convention() { + for name in [ + "Handle", + "handleAsync", + "EXECUTE", + "consume", + "run", + "__invoke", + ] { + assert!(is_handler_method(name), "{name}"); + } + assert!(!is_handler_method("handler")); + } +} diff --git a/crates/codegraph-ui/src/api/deadcode.rs b/crates/codegraph-ui/src/api/deadcode.rs new file mode 100644 index 00000000..1f5da9b2 --- /dev/null +++ b/crates/codegraph-ui/src/api/deadcode.rs @@ -0,0 +1,226 @@ +//! `GET /api/deadcode?limit=&kinds=&exported=&tests=&generated=` — symbols +//! nothing in this repository reaches, grouped by the file they live in. +//! Upstream `src/ui-server/api/deadcode.ts`. +//! +//! The derivation is `codegraph_graph::dead_code`, shared so a second surface +//! asking the same question cannot get a different answer. This module hands it +//! a source reader that goes through the viewer's read chokepoint, and carries +//! the exclusion counts onto the wire so the screen can say what the list could +//! not see. Rows come back ranked (largest first) and are grouped by file for +//! display; the group order follows the best row in it. + +use std::collections::HashMap; +use std::time::Instant; + +use codegraph_core::file_class::is_generated_file; +use codegraph_core::types::NodeKind; +use codegraph_graph::dead_code::{ + DeadCodeQuery, MAX_CORROBORATION_BYTES, build_dead_code_report, is_allowed_dead_code_kind, +}; +use serde::Serialize; +use serde_json::{Value, json}; + +use super::Ctx; +use super::source::read_indexed_file_text; +use super::wire::{WireList, WireNodeRef, locale_compare, to_node_ref, wire_list}; +use crate::respond::{ApiResult, Query}; + +/// Rows carried on the payload. The screen shows every one it is given. +pub const MAX_DEAD_CODE_ROWS: i64 = 300; +/// Members folded under one row before the row just counts them. +pub const MAX_DEAD_CODE_MEMBERS: usize = 12; + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +struct WireDeadCodeRow { + #[serde(flatten)] + node: WireNodeRef, + /// Source lines it spans — the rank, and what deleting it would remove. + lines: i64, + members: WireList, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +struct WireDeadCodeGroup { + file: String, + generated: bool, + test: bool, + lines: i64, + rows: Vec, +} + +/// The sentence each exclusion prints under the list, as "N