diff --git a/plugins/semctx/skills/codebase-retrieval/SKILL.md b/plugins/semctx/skills/codebase-retrieval/SKILL.md index c05b05a..f6c58e8 100644 --- a/plugins/semctx/skills/codebase-retrieval/SKILL.md +++ b/plugins/semctx/skills/codebase-retrieval/SKILL.md @@ -99,6 +99,13 @@ re-syncs do not block use of the last complete index. `semctx index` while a known local checkout is active. If only a codebase id is known, pass its indexed local path as the codebase selector to activate watching. +- An expanded `search_codebase` hit marked `[not expanded: — …]` is + a chunk, not the whole symbol. Do not reason from it as a complete body. + Follow the text after the reason: search again later for `GraphLoading`, or + use `expand_chunk` / `file_outline` for the other reasons. +- A `search_codebase` or `grep` result that starts with `note: the server cut + this result to its content budget` is not complete. Follow its `next:` line + before you conclude that something is absent. - A tool reports that semctl is not logged in → answer the immediate question with host tools, and end your reply with one line telling the user semctl is logged out and `semctl auth login` restores it (tools recover on diff --git a/src/client/api.rs b/src/client/api.rs index b92ac6f..e885301 100644 --- a/src/client/api.rs +++ b/src/client/api.rs @@ -108,6 +108,35 @@ pub struct SearchHit { #[serde(default)] pub kind: String, pub snippet: String, + /// Symbol-granularity searches only: `true` when `snippet` is the body of + /// the enclosing symbol, `false` when the hit keeps its chunk snippet. + /// Absent on every other hit. + #[serde(default)] + pub expanded: Option, + /// Why a Symbol-granularity hit keeps its chunk snippet. Set only when + /// `expanded` is `false`. + #[serde(default)] + pub expansion_reason: Option, +} + +/// Why a hit of a Symbol-granularity search keeps its chunk snippet instead of +/// the body of its enclosing symbol. The server sends the variant name. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +pub enum ExpansionReason { + /// The hit is a function or type chunk. Its snippet is that declaration + /// chunk. + SymbolChunk, + /// No named declaration contains the hit. + NoEnclosingSymbol, + /// The graph of the snapshot is not complete or not reachable. + GraphLoading, + /// The expansion deadline of the server passed. + Deadline, + /// The body does not fit in the content budget of the response. + ContentBudget, + /// The search spans more than one codebase. Only a search of one codebase + /// expands hits. + MultipleProjects, } /// `GET /v1/codebases/{id}/graph/outline` response — a file's table of diff --git a/src/client/mod.rs b/src/client/mod.rs index 3bc1cd7..b9807ff 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -29,6 +29,7 @@ mod retry; #[cfg(test)] pub(crate) mod stub; pub(crate) mod transport; +mod truncation; use failure::gateway_error; use retry::Idempotency; @@ -37,6 +38,7 @@ pub(crate) use failure::ApiFailure; pub(crate) use graph_state::{PartialAnswer, PartialAnswers}; pub(crate) use retry::DEADLINE_MARGIN; pub(crate) use transport::HttpTransport; +pub(crate) use truncation::ResultTruncation; const TENANT_HEADER: &str = "X-Tenant-Id"; /// The checkout a request is made from. The server prefers that copy of a @@ -437,6 +439,19 @@ impl Client { unwrap_envelope(resp, "GET", &url).await } + /// Like [`Self::get`], for a search or grep result that the content budget + /// of the server can shorten. Also returns the truncation that the answer + /// reports, if any. + pub async fn get_budgeted( + &self, + path: &str, + ) -> Result<(T, Option)> { + let (resp, url) = self + .send(reqwest::Method::GET, path, None, Idempotency::Idempotent) + .await?; + unwrap_budgeted(resp, "GET", &url).await + } + /// Like [`Self::get`], but returns `Ok(None)` on a 404 instead of erroring — /// for "does this still exist?" probes (e.g. validating a cached codebase id /// before trusting it against the current server). @@ -493,6 +508,26 @@ impl Client { self.post_with(path, body, Idempotency::Idempotent).await } + /// Like [`Self::post_read`], for a search result that the content budget + /// of the server can shorten. Also returns the truncation that the answer + /// reports, if any. + pub async fn post_read_budgeted( + &self, + path: &str, + body: &B, + ) -> Result<(T, Option)> { + let body = serde_json::to_value(body).context("serialize POST body")?; + let (resp, url) = self + .send( + reqwest::Method::POST, + path, + Some(body), + Idempotency::Idempotent, + ) + .await?; + unwrap_budgeted(resp, "POST", &url).await + } + async fn post_with( &self, path: &str, @@ -582,6 +617,16 @@ async fn unwrap_envelope( .ok_or_else(|| anyhow!("{method} {url} -> {status}: success but no data")) } +/// [`unwrap_envelope`], with the truncation that the headers report. +async fn unwrap_budgeted( + resp: reqwest::Response, + method: &str, + url: &str, +) -> Result<(T, Option)> { + let truncation = ResultTruncation::from_headers(resp.headers()); + Ok((unwrap_envelope(resp, method, url).await?, truncation)) +} + /// Read one envelope answer. A failure by status or by the envelope's own flag /// becomes an [`ApiFailure`] that carries the typed error code and the /// server's retry delay. diff --git a/src/client/truncation.rs b/src/client/truncation.rs new file mode 100644 index 0000000..7c3aba4 --- /dev/null +++ b/src/client/truncation.rs @@ -0,0 +1,74 @@ +//! The header of a search or grep result that a response budget of the server +//! shortened. +//! +//! semctx sends `X-Semctx-Result-Truncated: content-budget` on +//! `POST /v1/search` and `GET /v1/codebases/{id}/grep` when its content budget +//! cut a snippet or line, left a Symbol hit unexpanded, or dropped later +//! results. The status stays 200 and the body keeps its shape, so only the +//! header tells the client that the result is not complete. + +use reqwest::header::HeaderMap; + +const TRUNCATED_HEADER: &str = "X-Semctx-Result-Truncated"; +const CONTENT_BUDGET: &str = "content-budget"; + +/// Why the server shortened a result. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum ResultTruncation { + /// The content budget of the server (`content-budget`). + ContentBudget, + /// A value that this client does not know. The header still says that the + /// result is shortened, so the client reports the value as it is. + Other(String), +} + +impl ResultTruncation { + /// Read the headers of a success answer. `None` when the result is + /// complete, that is, when the header is absent. + pub(crate) fn from_headers(headers: &HeaderMap) -> Option { + let value = headers.get(TRUNCATED_HEADER)?; + let value = String::from_utf8_lossy(value.as_bytes()).trim().to_owned(); + Some(if value.eq_ignore_ascii_case(CONTENT_BUDGET) { + Self::ContentBudget + } else { + Self::Other(value) + }) + } + + /// The header and its value, as the server sent them. + pub(crate) fn header(&self) -> String { + let value = match self { + Self::ContentBudget => CONTENT_BUDGET, + Self::Other(value) => value, + }; + format!("{TRUNCATED_HEADER}: {value}") + } +} + +#[cfg(test)] +mod tests { + use reqwest::header::{HeaderMap, HeaderValue}; + + use super::ResultTruncation; + + fn headers(value: Option<&'static str>) -> HeaderMap { + let mut map = HeaderMap::new(); + if let Some(value) = value { + map.insert("X-Semctx-Result-Truncated", HeaderValue::from_static(value)); + } + map + } + + #[test] + fn only_a_present_header_marks_a_result_as_shortened() { + assert_eq!(ResultTruncation::from_headers(&headers(None)), None); + assert_eq!( + ResultTruncation::from_headers(&headers(Some(" Content-Budget "))), + Some(ResultTruncation::ContentBudget) + ); + let other = ResultTruncation::from_headers(&headers(Some("line-budget"))) + .expect("an unknown value still marks the result as shortened"); + assert_eq!(other, ResultTruncation::Other("line-budget".into())); + assert_eq!(other.header(), "X-Semctx-Result-Truncated: line-budget"); + } +} diff --git a/src/commands/hook/mod.rs b/src/commands/hook/mod.rs index 505d5fe..dc19ecc 100644 --- a/src/commands/hook/mod.rs +++ b/src/commands/hook/mod.rs @@ -191,10 +191,12 @@ fn resets_segment(event: &str, source: &str) -> bool { /// prompts. Direct graph/tool requests and prompts that can rely on supplied /// context remain model-routed, avoiding a generic search before a precise call. async fn user_prompt_context(cli: &Cli, input: &HookInput) -> Option { - let top_k = env_parse::("SEMCTX_HOOK_TOP_K").unwrap_or(5); - if top_k == 0 { + let Some(top_k) = prompt_top_k(env_parse::("SEMCTX_HOOK_TOP_K")) else { + debug(format_args!( + "prompt search is off or SEMCTX_HOOK_TOP_K is above 1000" + )); return None; - } + }; let prompt = input.prompt.trim(); // A global UserPromptSubmit hook fires on every message. Keep this path // high-precision: the skill and MCP metadata route exact operations, while @@ -206,7 +208,7 @@ async fn user_prompt_context(cli: &Cli, input: &HookInput) -> Option { let (client, codebase) = connect(cli, &input.cwd).await?; let body = api::SearchRequestBody { query: prompt, - top_k: u32::try_from(top_k).unwrap_or(u32::MAX), + top_k, codebase_id: Some(&codebase), codebase_ids: None, scope: None, @@ -243,7 +245,7 @@ async fn user_prompt_context(cli: &Cli, input: &HookInput) -> Option { let hits: Vec<&api::SearchHit> = hits .iter() .filter(|h| min_score.is_none_or(|m| h.score >= f64::from(m))) - .take(top_k) + .take(top_k as usize) .collect(); if hits.is_empty() { return None; @@ -283,6 +285,15 @@ async fn user_prompt_context(cli: &Cli, input: &HookInput) -> Option { Some(out) } +/// The `topK` of the prompt search: `SEMCTX_HOOK_TOP_K`, else 5. `None` when +/// the value is 0, which turns the search off, or when the value is above the +/// limit of the server, which answers 400 to it. +fn prompt_top_k(configured: Option) -> Option { + u32::try_from(configured.unwrap_or(5)) + .ok() + .filter(|top_k| crate::query::RESULT_LIMIT.contains(top_k)) +} + const HOOK_SEARCH_TIMEOUT_SECS: u64 = 6; #[derive(Debug, Clone, Copy, PartialEq, Eq)] diff --git a/src/commands/hook/tests.rs b/src/commands/hook/tests.rs index 5c59191..c211eda 100644 --- a/src/commands/hook/tests.rs +++ b/src/commands/hook/tests.rs @@ -14,6 +14,20 @@ fn parses_real_snake_case_payload() { assert_eq!(input.cwd, "/repo"); } +#[test] +fn the_prompt_search_never_asks_for_a_top_k_outside_1_to_1000() { + assert_eq!(prompt_top_k(None), Some(5), "the default"); + assert_eq!(prompt_top_k(Some(0)), None, "0 turns the search off"); + assert_eq!(prompt_top_k(Some(1)), Some(1)); + assert_eq!(prompt_top_k(Some(1000)), Some(1000)); + assert_eq!(prompt_top_k(Some(1001)), None, "the server refuses it"); + assert_eq!( + prompt_top_k(Some(usize::MAX)), + None, + "the server refuses it" + ); +} + #[test] fn unindexed_startup_notice_requires_opt_in_and_names_the_tool() { let message = unindexed_notice(std::path::Path::new("/repo")); diff --git a/src/commands/search.rs b/src/commands/search.rs index 3d54447..43ddc01 100644 --- a/src/commands/search.rs +++ b/src/commands/search.rs @@ -13,7 +13,7 @@ pub struct SearchArgs { /// Natural-language query. pub query: String, - /// Max hits to return. Server clamps at 1000. + /// Max hits to return, 1 to 1000. Another value fails before the search. #[arg(long, default_value_t = 20)] pub top_k: u32, @@ -32,6 +32,8 @@ pub struct SearchArgs { pub kinds: Vec, /// Render the full enclosing-symbol body per hit instead of a short snippet. + /// Each hit is marked `[expanded]`, or `[not expanded: ]` when its + /// snippet is only a chunk. #[arg(long)] pub expand: bool, diff --git a/src/mcp/docs/tools/grep.md b/src/mcp/docs/tools/grep.md index 56554c5..251e887 100644 --- a/src/mcp/docs/tools/grep.md +++ b/src/mcp/docs/tools/grep.md @@ -8,7 +8,7 @@ Use this for exact strings and patterns: an error message, a config key, a TODO, - `literal` — match `pattern` as an exact substring: every character (including `. * ( ) [ ] \`) is taken verbatim, so nothing needs escaping and there are no accidental regex matches. Default false. Use it for exact code like `.unwrap()`, `foo(bar)`, or `Vec`. - `ignore_case` — case-insensitive matching. Default false. - `path` — optional path substring to narrow which files are scanned (e.g. `server/`). -- `max` — max matches to return, 1–1000. Default 100. +- `max` — max matches to return, 1–1000. Default 100. Another value fails before the search. ## Regex syntax @@ -30,4 +30,4 @@ Example: find function definitions → `{ "pattern": "fn \\w+\\(" }`. To search ## Notes - Searches the indexed snapshot, so brand-new local edits aren't reflected until the codebase re-syncs (see `sync_status`). -- Results stop at `max`. Narrow with `path` before raising it. +- Results stop at `max`, or earlier at the byte budget of the server: by default 32 KiB of path and line text in one response. The server cuts each line at 512 bytes by default. When the server cuts a line or drops matches, the result starts with `note: the server cut this result to its content budget …` and a `next:` step. Narrow with `path` before raising `max`. diff --git a/src/mcp/docs/tools/search_codebase.md b/src/mcp/docs/tools/search_codebase.md index 68192a5..a0d9539 100644 --- a/src/mcp/docs/tools/search_codebase.md +++ b/src/mcp/docs/tools/search_codebase.md @@ -15,11 +15,28 @@ Hybrid (dense + lexical) retrieval over the indexed codebase. Returns ranked hit ## Parameters - `query` (required) — the natural-language search. -- `top_k` — max hits to return (default 8). +- `top_k` — max hits to return, 1–1000 (default 8). Another value fails before the search. The content budget of the server can return fewer hits. - `domains` — restrict to specific registered domain ids. Omit to fan out across every registered domain (run `list_domains` to see what's available). - `prefer` — ranking bias: `"code"` demotes documentation (markdown) so the implementation leads; `"docs"` does the reverse. Omit for the unbiased hybrid ranking. Use `"code"` for "how is X implemented", leave off for "explain X conceptually". - `kinds` — restrict to chunk kinds: `function`, `container` (type/class/module), or `block` (free text, incl. prose). Omit for every kind. -- `expand` — return the **full enclosing-symbol body** per hit instead of a 4-line snippet. Block hits snap to their enclosing function/type. Default false. Bodies stop at 30,000 characters in total; later hits stay snippets. Expand a small `top_k`, and do not reread files whose bodies were returned. +- `expand` — return the **full enclosing-symbol body** per hit instead of a 4-line snippet. Block hits snap to their enclosing function/type. Default false. Expand a small `top_k`, and do not reread files whose bodies were returned. + +## Result budget and expanded hits + +The server limits the snippet text of one search to 32 KiB by default (`Search:ContentBudgetBytes`). It cuts the snippet that reaches the limit and drops the hits after it. Ask for a small `top_k`, mainly with `expand`. + +When the server shortens a result, the result starts with `note: the server cut this result to its content budget …` and a `next:` step. The result is then not complete: narrow the query, lower `top_k`, or search without `expand`. + +With `expand`, each hit tells whether its snippet is the whole symbol: + +- `[expanded]` — the snippet is the body of the enclosing symbol. +- `[not expanded: — …]` — the snippet is the chunk only. Do not treat it as the whole symbol. The reasons: + - `SymbolChunk` — the hit is a function or type chunk, and the snippet is that chunk. + - `NoEnclosingSymbol` — no named declaration contains the hit. + - `GraphLoading` — the graph is not ready. Search again later for the body. + - `Deadline` — the expansion deadline of the server passed. Use `expand_chunk`. + - `ContentBudget` — the body does not fit in the content budget. Use `expand_chunk`. + - `MultipleProjects` — the search spans more than one codebase. Only a search of one codebase expands hits. ## Result freshness diff --git a/src/mcp/tool_types.rs b/src/mcp/tool_types.rs index c2c1a31..6840b76 100644 --- a/src/mcp/tool_types.rs +++ b/src/mcp/tool_types.rs @@ -23,7 +23,8 @@ pub struct SearchArgs { pub copy: Option, /// Natural-language query. pub query: String, - /// Max hits to return. Defaults to 8. + /// Max hits to return, 1–1000. Defaults to 8. Another value fails before + /// the search. pub top_k: Option, /// Restrict to these registered domain ids. Empty / omitted = all. pub domains: Option>, @@ -34,8 +35,10 @@ pub struct SearchArgs { /// Omit for every kind. pub kinds: Option>, /// Return full enclosing-symbol bodies instead of 4-line snippets. Defaults - /// false. Bodies stop at 30,000 characters in total; later hits stay - /// snippets. Do not reread files whose bodies were returned. + /// false. The server limits the snippet text of one search to 32 KiB by + /// default. Each hit is marked `[expanded]`, or `[not expanded: ]` + /// when its snippet is only a chunk. Do not reread files whose bodies were + /// returned. pub expand: Option, /// Server scope lens: `local`, `personal`, `organization`, or `global`. /// Mutually exclusive with `codebase_ids`. @@ -150,7 +153,8 @@ pub struct GrepArgs { pub ignore_case: Option, /// Optional path substring to narrow which files are searched. pub path: Option, - /// Max matches to return, 1–1000. Defaults to 100. + /// Max matches to return, 1–1000. Defaults to 100. Another value fails + /// before the search. pub max: Option, } diff --git a/src/query/mod.rs b/src/query/mod.rs index 95de13d..114ca00 100644 --- a/src/query/mod.rs +++ b/src/query/mod.rs @@ -33,8 +33,8 @@ pub use advanced::*; pub use render::render_projects; use render::{ - HitContext, hit_location, local_path, render_boundaries, render_compact, render_hits, - render_hits_inner, + BudgetedResult, HitContext, budget_note, hit_location, local_path, render_boundaries, + render_compact, render_hits, render_hits_inner, }; /// Map the user's `--prefer` value to the server's `SearchPreference` enum name @@ -92,6 +92,29 @@ impl SearchOpts { } } +/// The values of `top_k` and of the grep `max` that the server accepts. The +/// server answers 400 to a value outside this range, so the client refuses +/// the value before it sends a request. +pub(crate) const RESULT_LIMIT: std::ops::RangeInclusive = 1..=1000; + +/// `value` when it is in [`RESULT_LIMIT`], else the invalid-argument failure +/// of `op` that names the argument `name`. +pub(crate) fn result_limit(op: &'static str, name: &str, value: u32) -> Result { + if RESULT_LIMIT.contains(&value) { + Ok(value) + } else { + Err(ToolError::new( + op, + FailureKind::InvalidArgument, + format!( + "{name} must be between {} and {}, not {value}", + RESULT_LIMIT.start(), + RESULT_LIMIT.end() + ), + )) + } +} + /// Cross-domain search, scoped to the launched codebase when one is set. The /// server applies the kind filter, ranking bias, and symbol granularity; the /// client sends the options, annotates local staleness, and renders. @@ -102,6 +125,7 @@ pub async fn search( domains: &[String], opts: &SearchOpts, ) -> Result { + let top_k = result_limit("search_codebase", "top_k", top_k)?; let scope = opts.normalized_scope()?; let body = api::SearchRequestBody { query, @@ -125,12 +149,17 @@ pub async fn search( .map(str::to_string), granularity: opts.expand.then(|| "Symbol".to_string()), }; - let hits = client - .post_read::<_, Vec>("/v1/search", &body) + let (hits, truncation) = client + .post_read_budgeted::<_, Vec>("/v1/search", &body) .await .map_err(|e| ToolError::from_client("search_codebase", &e))?; + // The note leads, so an agent reads it before it reasons from the hits. + let mut out = truncation + .map(|truncation| budget_note(BudgetedResult::Search, &truncation)) + .unwrap_or_default(); if hits.is_empty() { - return Ok("no results".to_string()); + out.push_str("no results"); + return Ok(out); } // Staleness is the one shaping the client owns — only it has the local bytes. @@ -140,14 +169,15 @@ pub async fn search( opts.scope.is_none() && opts.codebase_ids.is_empty() && client.codebase_raw().is_some(), ); let stale = stale_paths_in_context(client, &hits, context).await; - Ok(render_hits_inner( + out.push_str(&render_hits_inner( &hits, "no results", context, true, &stale, opts.expand, - )) + )); + Ok(out) } fn normalize_scope(scope: &str) -> Option<&'static str> { @@ -463,6 +493,7 @@ pub async fn grep( path: Option<&str>, max: u32, ) -> Result { + let max = result_limit("grep", "max", max)?; let cb = bound_codebase(client, "grep")?; let mut url = format!( "/v1/codebases/{cb}/grep?pattern={}®ex={regex}&ignoreCase={ignore_case}&max={max}", @@ -471,15 +502,18 @@ pub async fn grep( if let Some(p) = path.filter(|p| !p.is_empty()) { write!(url, "&path={}", urlencode(p)).expect("writing to a String cannot fail"); } - let matches = client - .get::>(&url) + let (matches, truncation) = client + .get_budgeted::>(&url) .await .map_err(|e| ToolError::from_client("grep", &e))?; + let mut out = truncation + .map(|truncation| budget_note(BudgetedResult::Grep, &truncation)) + .unwrap_or_default(); if matches.is_empty() { - return Ok(format!("no matches for `{pattern}`")); + write!(out, "no matches for `{pattern}`").expect("writing to a String cannot fail"); + return Ok(out); } let root = client.local_root(); - let mut out = String::new(); if let Some(file) = single_file_target(path, &matches) && root.is_some_and(|root| root.join(file).is_file()) { @@ -667,9 +701,10 @@ fn urlencode(s: &str) -> String { #[cfg(test)] mod tests { use super::{ - SearchOpts, bound_codebase, human_bytes, is_stale, local_blake3, normalize_prefer, - single_file_target, + SearchOpts, bound_codebase, grep, human_bytes, is_stale, local_blake3, normalize_prefer, + search, single_file_target, }; + use crate::client::stub::{Reply, Stub}; use crate::client::{Client, api}; fn at(path: &str) -> api::GrepMatch { @@ -731,6 +766,160 @@ mod tests { assert!(message.ends_with("next: use local Read/Grep for this request.")); } + /// A stub that answers every request with an empty list, and a client + /// bound to codebase `A` on it. + async fn empty_list_stub() -> (Stub, Client) { + let stub = Stub::serve(vec![Reply::ok("[]")]).await; + let client = Client::for_test_server(&stub.url).with_codebase("A".to_string()); + (stub, client) + } + + #[tokio::test] + async fn a_top_k_outside_1_to_1000_is_an_invalid_argument_and_sends_nothing() { + let (stub, client) = empty_list_stub().await; + for top_k in [0, 1001] { + let message = search(&client, "q", top_k, &[], &SearchOpts::default()) + .await + .unwrap_err() + .to_string(); + assert!( + message.starts_with(&format!( + "search_codebase failed: top_k must be between 1 and 1000, not {top_k}" + )), + "{message}" + ); + assert!(message.ends_with(CORRECT_THE_ARGUMENT), "{message}"); + } + assert_eq!(stub.requests(), 0); + + for top_k in [1, 1000] { + search(&client, "q", top_k, &[], &SearchOpts::default()) + .await + .expect("a top_k from 1 to 1000 reaches the server"); + } + assert_eq!(stub.requests(), 2); + } + + #[tokio::test] + async fn a_grep_max_outside_1_to_1000_is_an_invalid_argument_and_sends_nothing() { + let (stub, client) = empty_list_stub().await; + for max in [0, 1001] { + let message = grep(&client, "needle", false, false, None, max) + .await + .unwrap_err() + .to_string(); + assert!( + message.starts_with(&format!( + "grep failed: max must be between 1 and 1000, not {max}" + )), + "{message}" + ); + assert!(message.ends_with(CORRECT_THE_ARGUMENT), "{message}"); + } + assert_eq!(stub.requests(), 0); + + for max in [1, 1000] { + grep(&client, "needle", false, false, None, max) + .await + .expect("a max from 1 to 1000 reaches the server"); + } + let lines = stub.request_lines(); + assert_eq!(lines.len(), 2); + assert!(lines[0].contains("&max=1 "), "{}", lines[0]); + assert!(lines[1].contains("&max=1000 "), "{}", lines[1]); + } + + const ONE_HIT: &str = r#"[{"domainId":"code","id":"x","score":1.0,"path":"a.rs", + "lineStart":1,"lineEnd":2,"kind":"block","snippet":"s"}]"#; + const ONE_MATCH: &str = r#"[{"path":"a.rs","lineNumber":1,"line":"needle"}]"#; + + /// A stub whose every answer is `data` with `X-Semctx-Result-Truncated: + /// value`, or without the header when `value` is `None`. + async fn truncated_stub(value: Option<&'static str>, data: &str) -> (Stub, Client) { + let stub = Stub::serve(vec![Reply::Answer { + status: "200 OK", + headers: value + .map(|value| vec![("X-Semctx-Result-Truncated", value)]) + .unwrap_or_default(), + body: format!(r#"{{"success":true,"data":{data}}}"#), + }]) + .await; + let client = Client::for_test_server(&stub.url).with_codebase("A".to_string()); + (stub, client) + } + + #[tokio::test] + async fn a_search_that_the_server_cut_names_the_budget_and_the_next_step() { + let next = "next: narrow the query, lower top_k, or search without expand."; + let (_stub, client) = truncated_stub(Some("content-budget"), ONE_HIT).await; + let out = search(&client, "q", 8, &[], &SearchOpts::default()) + .await + .expect("a cut result is a success"); + assert!( + out.starts_with(&format!( + "note: the server cut this result to its content budget (32 KiB of snippet \ + text by default). A snippet is cut, a hit is not expanded, or later hits \ + are dropped. {next}\n" + )), + "{out}" + ); + assert!(out.contains("a.rs:1-2"), "the hits stay: {out}"); + + let (_stub, client) = truncated_stub(Some("other-budget"), ONE_HIT).await; + let out = search(&client, "q", 8, &[], &SearchOpts::default()) + .await + .expect("a cut result is a success"); + assert!( + out.starts_with(&format!( + "note: the server shortened this result \ + (X-Semctx-Result-Truncated: other-budget). {next}\n" + )), + "{out}" + ); + + let (_stub, client) = truncated_stub(None, ONE_HIT).await; + let out = search(&client, "q", 8, &[], &SearchOpts::default()) + .await + .expect("a complete result is a success"); + assert!(!out.contains("note:"), "{out}"); + } + + #[tokio::test] + async fn a_grep_that_the_server_cut_names_the_budget_and_the_next_step() { + let next = "next: narrow path or the pattern, or lower max."; + let (_stub, client) = truncated_stub(Some("content-budget"), ONE_MATCH).await; + let out = grep(&client, "needle", false, false, None, 100) + .await + .expect("a cut result is a success"); + assert!( + out.starts_with(&format!( + "note: the server cut this result to its content budget (32 KiB of path and \ + line text, and 512 bytes for each line, by default). A line is cut, or later \ + matches are dropped. {next}\n" + )), + "{out}" + ); + assert!(out.contains("a.rs:1: needle"), "the matches stay: {out}"); + + let (_stub, client) = truncated_stub(Some("other-budget"), ONE_MATCH).await; + let out = grep(&client, "needle", false, false, None, 100) + .await + .expect("a cut result is a success"); + assert!( + out.starts_with(&format!( + "note: the server shortened this result \ + (X-Semctx-Result-Truncated: other-budget). {next}\n" + )), + "{out}" + ); + + let (_stub, client) = truncated_stub(None, ONE_MATCH).await; + let out = grep(&client, "needle", false, false, None, 100) + .await + .expect("a complete result is a success"); + assert!(!out.contains("note:"), "{out}"); + } + #[test] fn total_source_bytes_are_human_readable_without_losing_exactness() { assert_eq!(human_bytes(0), "0 B"); diff --git a/src/query/render.rs b/src/query/render.rs index a98e01e..727c246 100644 --- a/src/query/render.rs +++ b/src/query/render.rs @@ -7,7 +7,7 @@ use std::collections::HashSet; use std::fmt::Write; use std::path::Path; -use crate::client::api; +use crate::client::{ResultTruncation, api}; use super::inspection::GRAPH_LIST_CAP; @@ -247,8 +247,10 @@ pub(super) fn render_hits( /// As [`render_hits`], but annotates any hit whose codebase-relative path is in /// `stale` (its local file no longer matches the index), and — when `full_body` /// — prints the whole snippet instead of the first four lines (for `--expand`, -/// where the server already returned the full enclosing-symbol body). The public -/// [`render_hits`] passes an empty set and `false`; only `search` varies them. +/// where the server returns the enclosing-symbol body, or the chunk snippet and +/// the reason). The server bounds the snippet text of the response by its +/// content budget. The public [`render_hits`] passes an empty set and `false`; +/// only `search` varies them. pub(super) fn render_hits_inner( hits: &[api::SearchHit], empty_msg: &str, @@ -269,8 +271,6 @@ pub(super) fn render_hits_inner( let mut out = String::new(); let mut separated = false; - let mut expanded_chars = 0_usize; - let mut snipped = 0_usize; for h in hits { let root = context.root_for(h); if let Some(floor) = weak_below @@ -319,46 +319,106 @@ pub(super) fn render_hits_inner( }; writeln!( out, - "[{}{lang}]{codebase} {} {sym}{kind}{}{score}{stale_mark}", + "[{}{lang}]{codebase} {} {sym}{kind}{}{score}{stale_mark}{}", h.domain_id, hit_location(h, root), - write_marker(h) + write_marker(h), + expansion_marker(h) ) .expect("writing to a String cannot fail"); // Skip leading blank lines so the declaration/signature leads the snippet // rather than whitespace. `--expand` shows the whole body; otherwise the // first few lines, enough to judge relevance without flooding context. let lines = h.snippet.lines().skip_while(|l| l.trim().is_empty()); - let expand = full_body - && (expanded_chars == 0 || expanded_chars + h.snippet.len() <= EXPANDED_BODY_CHARS); - if expand { - expanded_chars += h.snippet.len(); - } else if full_body { - snipped += 1; - } - for line in lines.take(if expand { usize::MAX } else { 4 }) { + for line in lines.take(if full_body { usize::MAX } else { 4 }) { out.push_str(" "); out.push_str(line); out.push('\n'); } out.push('\n'); } - if snipped > 0 { - writeln!( - out, - "{snipped} hit(s) shown as snippets: expanded bodies stop at \ - {EXPANDED_BODY_CHARS} characters. Use expand_chunk for one of them." - ) - .expect("writing to a String cannot fail"); - } out } -/// The total length of expanded bodies in one search answer. The first hit is -/// always expanded; a later hit that does not fit is shown as a snippet. The -/// server has no size limit of its own, and one expanded search with a broad -/// `top_k` can otherwise return tens of thousands of characters. -const EXPANDED_BODY_CHARS: usize = 30_000; +/// A result that the content budget of the server can shorten. The budget and +/// the next step differ between the two. +#[derive(Clone, Copy)] +pub(super) enum BudgetedResult { + Search, + Grep, +} + +/// The first line of a result that the server shortened: what the budget is +/// and what the agent can do. The budget values are the server defaults +/// (`Search:ContentBudgetBytes`, `Search:GrepMaxBytes`, +/// `Search:GrepMaxLineBytes`). +pub(super) fn budget_note(result: BudgetedResult, truncation: &ResultTruncation) -> String { + let next = match result { + BudgetedResult::Search => "next: narrow the query, lower top_k, or search without expand.", + BudgetedResult::Grep => "next: narrow path or the pattern, or lower max.", + }; + match (truncation, result) { + (ResultTruncation::ContentBudget, BudgetedResult::Search) => format!( + "note: the server cut this result to its content budget (32 KiB of snippet text \ + by default). A snippet is cut, a hit is not expanded, or later hits are dropped. \ + {next}\n" + ), + (ResultTruncation::ContentBudget, BudgetedResult::Grep) => format!( + "note: the server cut this result to its content budget (32 KiB of path and line \ + text, and 512 bytes for each line, by default). A line is cut, or later matches \ + are dropped. {next}\n" + ), + (ResultTruncation::Other(_), _) => format!( + "note: the server shortened this result ({}). {next}\n", + truncation.header() + ), + } +} + +/// The expansion state of a hit of a Symbol-granularity search, for the end of +/// its header line. An agent must know when a snippet is a chunk and not the +/// whole symbol. Empty for a hit without the state: a Chunk-granularity search +/// or a graph lookup. +fn expansion_marker(h: &api::SearchHit) -> String { + match (h.expanded, h.expansion_reason) { + (Some(true), _) => " [expanded]".to_string(), + (Some(false), Some(reason)) => { + let (name, why) = expansion_reason_text(reason); + format!(" [not expanded: {name} — {why}]") + } + (Some(false), None) => " [not expanded]".to_string(), + (None, _) => String::new(), + } +} + +/// The wire name of `reason` and what it means for the agent. +fn expansion_reason_text(reason: api::ExpansionReason) -> (&'static str, &'static str) { + match reason { + api::ExpansionReason::SymbolChunk => ( + "SymbolChunk", + "the snippet is the function or type chunk itself", + ), + api::ExpansionReason::NoEnclosingSymbol => { + ("NoEnclosingSymbol", "no named declaration contains the hit") + } + api::ExpansionReason::GraphLoading => ( + "GraphLoading", + "the graph is not ready; search again later for the body", + ), + api::ExpansionReason::Deadline => ( + "Deadline", + "the expansion deadline passed; use expand_chunk for more context", + ), + api::ExpansionReason::ContentBudget => ( + "ContentBudget", + "the body does not fit in the content budget of the server; use expand_chunk", + ), + api::ExpansionReason::MultipleProjects => ( + "MultipleProjects", + "a search of more than one codebase does not expand hits", + ), + } +} /// A hit scoring below this fraction of the top hit's score is sorted under the /// "weaker matches" separator. Relative (not absolute) because raw hybrid @@ -421,9 +481,7 @@ pub(super) fn hit_location(h: &api::SearchHit, root: Option<&Path>) -> String { mod tests { use std::collections::HashSet; - use super::{ - EXPANDED_BODY_CHARS, HitContext, api, render_hits_inner, render_job, strip_verbatim_prefix, - }; + use super::{HitContext, api, render_hits_inner, render_job, strip_verbatim_prefix}; /// A `SearchHit` with a chosen chunk id + path — for the render test that /// keys on those. @@ -454,6 +512,8 @@ mod tests { codebase_id: None, kind: "function".into(), snippet: String::new(), + expanded: None, + expansion_reason: None, } } @@ -495,32 +555,83 @@ mod tests { ) } + /// The hits of a Symbol search as the server sends them: one expanded hit, + /// one hit for each reason, and one hit without the fields. + fn symbol_search_hits() -> Vec { + let hit = |id: &str, fields: &str| { + format!( + r#"{{"domainId":"code","id":"{id}","score":1.0,"path":"{id}.rs","lineStart":1, + "lineEnd":2,"kind":"block","snippet":"{id}-snippet\n"{fields}}}"# + ) + }; + let mut hits = vec![hit("body", r#","expanded":true"#)]; + for reason in [ + "SymbolChunk", + "NoEnclosingSymbol", + "GraphLoading", + "Deadline", + "ContentBudget", + "MultipleProjects", + ] { + hits.push(hit( + reason, + &format!(r#","expanded":false,"expansionReason":"{reason}""#), + )); + } + hits.push(hit("plain", "")); + serde_json::from_str(&format!("[{}]", hits.join(","))) + .expect("the hits of a Symbol search parse") + } + #[test] - fn expanded_bodies_stop_at_the_limit_and_later_hits_that_fit_still_expand() { - // 2,000 + 1,500 + 500 lines of 10 characters: the second body would pass - // 30,000 characters, the third still fits after it is skipped. - let out = expanded(&[body("a", 2_000), body("b", 1_500), body("c", 500)]); + fn an_expanded_search_tells_for_each_hit_whether_it_is_the_symbol_body_and_why_not() { + let out = expanded(&symbol_search_hits()); + let header = |id: &str| { + out.lines() + .find(|line| line.contains(&format!(" {id}.rs:1-2"))) + .unwrap_or_else(|| panic!("no header for {id}: {out}")) + }; - assert_eq!(out.matches("a-body-ln").count(), 2_000); - assert_eq!(out.matches("b-body-ln").count(), 4, "shown as a snippet"); - assert_eq!(out.matches("c-body-ln").count(), 500); - assert!( - out.ends_with(&format!( - "1 hit(s) shown as snippets: expanded bodies stop at {EXPANDED_BODY_CHARS} \ - characters. Use expand_chunk for one of them.\n" - )), - "{out}" - ); + assert!(header("body").ends_with(" [expanded]"), "{out}"); + for (reason, why) in [ + ( + "SymbolChunk", + "the snippet is the function or type chunk itself", + ), + ("NoEnclosingSymbol", "no named declaration contains the hit"), + ( + "GraphLoading", + "the graph is not ready; search again later for the body", + ), + ( + "Deadline", + "the expansion deadline passed; use expand_chunk for more context", + ), + ( + "ContentBudget", + "the body does not fit in the content budget of the server; use expand_chunk", + ), + ( + "MultipleProjects", + "a search of more than one codebase does not expand hits", + ), + ] { + assert!( + header(reason).ends_with(&format!(" [not expanded: {reason} — {why}]")), + "{out}" + ); + } + assert!(!header("plain").contains("expanded"), "{out}"); } #[test] - fn the_first_expanded_body_is_never_cut_and_small_answers_have_no_note() { - let oversized = expanded(&[body("a", 4_000)]); - assert_eq!(oversized.matches("a-body-ln").count(), 4_000); - assert!(!oversized.contains("shown as snippets"), "{oversized}"); + fn expanded_bodies_render_in_full_because_the_server_owns_the_budget() { + let out = expanded(&[body("a", 2_000), body("b", 1_500), body("c", 500)]); - let small = expanded(&[body("a", 10), body("b", 10)]); - assert!(!small.contains("shown as snippets"), "{small}"); + assert_eq!(out.matches("a-body-ln").count(), 2_000); + assert_eq!(out.matches("b-body-ln").count(), 1_500); + assert_eq!(out.matches("c-body-ln").count(), 500); + assert!(!out.contains("shown as snippets"), "{out}"); } #[test]