Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions plugins/semctx/skills/codebase-retrieval/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,13 @@ re-syncs do not block use of the last complete index.
`semctx index` while a known local checkout is active. If only a codebase id
is known, pass its indexed local path as the codebase selector to activate
watching.
- An expanded `search_codebase` hit marked `[not expanded: <reason> — …]` is
a chunk, not the whole symbol. Do not reason from it as a complete body.
Follow the text after the reason: search again later for `GraphLoading`, or
use `expand_chunk` / `file_outline` for the other reasons.
- A `search_codebase` or `grep` result that starts with `note: the server cut
this result to its content budget` is not complete. Follow its `next:` line
before you conclude that something is absent.
- A tool reports that semctl is not logged in → answer the immediate
question with host tools, and end your reply with one line telling the user
semctl is logged out and `semctl auth login` restores it (tools recover on
Expand Down
29 changes: 29 additions & 0 deletions src/client/api.rs
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,35 @@ pub struct SearchHit {
#[serde(default)]
pub kind: String,
pub snippet: String,
/// Symbol-granularity searches only: `true` when `snippet` is the body of
/// the enclosing symbol, `false` when the hit keeps its chunk snippet.
/// Absent on every other hit.
#[serde(default)]
pub expanded: Option<bool>,
/// Why a Symbol-granularity hit keeps its chunk snippet. Set only when
/// `expanded` is `false`.
#[serde(default)]
pub expansion_reason: Option<ExpansionReason>,
}

/// Why a hit of a Symbol-granularity search keeps its chunk snippet instead of
/// the body of its enclosing symbol. The server sends the variant name.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
pub enum ExpansionReason {
/// The hit is a function or type chunk. Its snippet is that declaration
/// chunk.
SymbolChunk,
/// No named declaration contains the hit.
NoEnclosingSymbol,
/// The graph of the snapshot is not complete or not reachable.
GraphLoading,
/// The expansion deadline of the server passed.
Deadline,
/// The body does not fit in the content budget of the response.
ContentBudget,
/// The search spans more than one codebase. Only a search of one codebase
/// expands hits.
MultipleProjects,
}

/// `GET /v1/codebases/{id}/graph/outline` response — a file's table of
Expand Down
45 changes: 45 additions & 0 deletions src/client/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ mod retry;
#[cfg(test)]
pub(crate) mod stub;
pub(crate) mod transport;
mod truncation;

use failure::gateway_error;
use retry::Idempotency;
Expand All @@ -37,6 +38,7 @@ pub(crate) use failure::ApiFailure;
pub(crate) use graph_state::{PartialAnswer, PartialAnswers};
pub(crate) use retry::DEADLINE_MARGIN;
pub(crate) use transport::HttpTransport;
pub(crate) use truncation::ResultTruncation;

const TENANT_HEADER: &str = "X-Tenant-Id";
/// The checkout a request is made from. The server prefers that copy of a
Expand Down Expand Up @@ -437,6 +439,19 @@ impl Client {
unwrap_envelope(resp, "GET", &url).await
}

/// Like [`Self::get`], for a search or grep result that the content budget
/// of the server can shorten. Also returns the truncation that the answer
/// reports, if any.
pub async fn get_budgeted<T: DeserializeOwned>(
&self,
path: &str,
) -> Result<(T, Option<ResultTruncation>)> {
let (resp, url) = self
.send(reqwest::Method::GET, path, None, Idempotency::Idempotent)
.await?;
unwrap_budgeted(resp, "GET", &url).await
}

/// Like [`Self::get`], but returns `Ok(None)` on a 404 instead of erroring —
/// for "does this still exist?" probes (e.g. validating a cached codebase id
/// before trusting it against the current server).
Expand Down Expand Up @@ -493,6 +508,26 @@ impl Client {
self.post_with(path, body, Idempotency::Idempotent).await
}

/// Like [`Self::post_read`], for a search result that the content budget
/// of the server can shorten. Also returns the truncation that the answer
/// reports, if any.
pub async fn post_read_budgeted<B: Serialize, T: DeserializeOwned>(
&self,
path: &str,
body: &B,
) -> Result<(T, Option<ResultTruncation>)> {
let body = serde_json::to_value(body).context("serialize POST body")?;
let (resp, url) = self
.send(
reqwest::Method::POST,
path,
Some(body),
Idempotency::Idempotent,
)
.await?;
unwrap_budgeted(resp, "POST", &url).await
}

async fn post_with<B: Serialize, T: DeserializeOwned>(
&self,
path: &str,
Expand Down Expand Up @@ -582,6 +617,16 @@ async fn unwrap_envelope<T: DeserializeOwned>(
.ok_or_else(|| anyhow!("{method} {url} -> {status}: success but no data"))
}

/// [`unwrap_envelope`], with the truncation that the headers report.
async fn unwrap_budgeted<T: DeserializeOwned>(
resp: reqwest::Response,
method: &str,
url: &str,
) -> Result<(T, Option<ResultTruncation>)> {
let truncation = ResultTruncation::from_headers(resp.headers());
Ok((unwrap_envelope(resp, method, url).await?, truncation))
}

/// Read one envelope answer. A failure by status or by the envelope's own flag
/// becomes an [`ApiFailure`] that carries the typed error code and the
/// server's retry delay.
Expand Down
74 changes: 74 additions & 0 deletions src/client/truncation.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
//! The header of a search or grep result that a response budget of the server
//! shortened.
//!
//! semctx sends `X-Semctx-Result-Truncated: content-budget` on
//! `POST /v1/search` and `GET /v1/codebases/{id}/grep` when its content budget
//! cut a snippet or line, left a Symbol hit unexpanded, or dropped later
//! results. The status stays 200 and the body keeps its shape, so only the
//! header tells the client that the result is not complete.

use reqwest::header::HeaderMap;

const TRUNCATED_HEADER: &str = "X-Semctx-Result-Truncated";
const CONTENT_BUDGET: &str = "content-budget";

/// Why the server shortened a result.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum ResultTruncation {
/// The content budget of the server (`content-budget`).
ContentBudget,
/// A value that this client does not know. The header still says that the
/// result is shortened, so the client reports the value as it is.
Other(String),
}

impl ResultTruncation {
/// Read the headers of a success answer. `None` when the result is
/// complete, that is, when the header is absent.
pub(crate) fn from_headers(headers: &HeaderMap) -> Option<Self> {
let value = headers.get(TRUNCATED_HEADER)?;
let value = String::from_utf8_lossy(value.as_bytes()).trim().to_owned();
Some(if value.eq_ignore_ascii_case(CONTENT_BUDGET) {
Self::ContentBudget
} else {
Self::Other(value)
})
}

/// The header and its value, as the server sent them.
pub(crate) fn header(&self) -> String {
let value = match self {
Self::ContentBudget => CONTENT_BUDGET,
Self::Other(value) => value,
};
format!("{TRUNCATED_HEADER}: {value}")
}
}

#[cfg(test)]
mod tests {
use reqwest::header::{HeaderMap, HeaderValue};

use super::ResultTruncation;

fn headers(value: Option<&'static str>) -> HeaderMap {
let mut map = HeaderMap::new();
if let Some(value) = value {
map.insert("X-Semctx-Result-Truncated", HeaderValue::from_static(value));
}
map
}

#[test]
fn only_a_present_header_marks_a_result_as_shortened() {
assert_eq!(ResultTruncation::from_headers(&headers(None)), None);
assert_eq!(
ResultTruncation::from_headers(&headers(Some(" Content-Budget "))),
Some(ResultTruncation::ContentBudget)
);
let other = ResultTruncation::from_headers(&headers(Some("line-budget")))
.expect("an unknown value still marks the result as shortened");
assert_eq!(other, ResultTruncation::Other("line-budget".into()));
assert_eq!(other.header(), "X-Semctx-Result-Truncated: line-budget");
}
}
21 changes: 16 additions & 5 deletions src/commands/hook/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -191,10 +191,12 @@ fn resets_segment(event: &str, source: &str) -> bool {
/// prompts. Direct graph/tool requests and prompts that can rely on supplied
/// context remain model-routed, avoiding a generic search before a precise call.
async fn user_prompt_context(cli: &Cli, input: &HookInput) -> Option<String> {
let top_k = env_parse::<usize>("SEMCTX_HOOK_TOP_K").unwrap_or(5);
if top_k == 0 {
let Some(top_k) = prompt_top_k(env_parse::<usize>("SEMCTX_HOOK_TOP_K")) else {
debug(format_args!(
"prompt search is off or SEMCTX_HOOK_TOP_K is above 1000"
));
return None;
}
};
let prompt = input.prompt.trim();
// A global UserPromptSubmit hook fires on every message. Keep this path
// high-precision: the skill and MCP metadata route exact operations, while
Expand All @@ -206,7 +208,7 @@ async fn user_prompt_context(cli: &Cli, input: &HookInput) -> Option<String> {
let (client, codebase) = connect(cli, &input.cwd).await?;
let body = api::SearchRequestBody {
query: prompt,
top_k: u32::try_from(top_k).unwrap_or(u32::MAX),
top_k,
codebase_id: Some(&codebase),
codebase_ids: None,
scope: None,
Expand Down Expand Up @@ -243,7 +245,7 @@ async fn user_prompt_context(cli: &Cli, input: &HookInput) -> Option<String> {
let hits: Vec<&api::SearchHit> = hits
.iter()
.filter(|h| min_score.is_none_or(|m| h.score >= f64::from(m)))
.take(top_k)
.take(top_k as usize)
.collect();
if hits.is_empty() {
return None;
Expand Down Expand Up @@ -283,6 +285,15 @@ async fn user_prompt_context(cli: &Cli, input: &HookInput) -> Option<String> {
Some(out)
}

/// The `topK` of the prompt search: `SEMCTX_HOOK_TOP_K`, else 5. `None` when
/// the value is 0, which turns the search off, or when the value is above the
/// limit of the server, which answers 400 to it.
fn prompt_top_k(configured: Option<usize>) -> Option<u32> {
u32::try_from(configured.unwrap_or(5))
.ok()
.filter(|top_k| crate::query::RESULT_LIMIT.contains(top_k))
}

const HOOK_SEARCH_TIMEOUT_SECS: u64 = 6;

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
Expand Down
14 changes: 14 additions & 0 deletions src/commands/hook/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,20 @@ fn parses_real_snake_case_payload() {
assert_eq!(input.cwd, "/repo");
}

#[test]
fn the_prompt_search_never_asks_for_a_top_k_outside_1_to_1000() {
assert_eq!(prompt_top_k(None), Some(5), "the default");
assert_eq!(prompt_top_k(Some(0)), None, "0 turns the search off");
assert_eq!(prompt_top_k(Some(1)), Some(1));
assert_eq!(prompt_top_k(Some(1000)), Some(1000));
assert_eq!(prompt_top_k(Some(1001)), None, "the server refuses it");
assert_eq!(
prompt_top_k(Some(usize::MAX)),
None,
"the server refuses it"
);
}

#[test]
fn unindexed_startup_notice_requires_opt_in_and_names_the_tool() {
let message = unindexed_notice(std::path::Path::new("/repo"));
Expand Down
4 changes: 3 additions & 1 deletion src/commands/search.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ pub struct SearchArgs {
/// Natural-language query.
pub query: String,

/// Max hits to return. Server clamps at 1000.
/// Max hits to return, 1 to 1000. Another value fails before the search.
#[arg(long, default_value_t = 20)]
pub top_k: u32,

Expand All @@ -32,6 +32,8 @@ pub struct SearchArgs {
pub kinds: Vec<String>,

/// Render the full enclosing-symbol body per hit instead of a short snippet.
/// Each hit is marked `[expanded]`, or `[not expanded: <reason>]` when its
/// snippet is only a chunk.
#[arg(long)]
pub expand: bool,

Expand Down
4 changes: 2 additions & 2 deletions src/mcp/docs/tools/grep.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ Use this for exact strings and patterns: an error message, a config key, a TODO,
- `literal` — match `pattern` as an exact substring: every character (including `. * ( ) [ ] \`) is taken verbatim, so nothing needs escaping and there are no accidental regex matches. Default false. Use it for exact code like `.unwrap()`, `foo(bar)`, or `Vec<T>`.
- `ignore_case` — case-insensitive matching. Default false.
- `path` — optional path substring to narrow which files are scanned (e.g. `server/`).
- `max` — max matches to return, 1–1000. Default 100.
- `max` — max matches to return, 1–1000. Default 100. Another value fails before the search.

## Regex syntax

Expand All @@ -30,4 +30,4 @@ Example: find function definitions → `{ "pattern": "fn \\w+\\(" }`. To search
## Notes

- Searches the indexed snapshot, so brand-new local edits aren't reflected until the codebase re-syncs (see `sync_status`).
- Results stop at `max`. Narrow with `path` before raising it.
- Results stop at `max`, or earlier at the byte budget of the server: by default 32 KiB of path and line text in one response. The server cuts each line at 512 bytes by default. When the server cuts a line or drops matches, the result starts with `note: the server cut this result to its content budget …` and a `next:` step. Narrow with `path` before raising `max`.
21 changes: 19 additions & 2 deletions src/mcp/docs/tools/search_codebase.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,28 @@ Hybrid (dense + lexical) retrieval over the indexed codebase. Returns ranked hit
## Parameters

- `query` (required) — the natural-language search.
- `top_k` — max hits to return (default 8).
- `top_k` — max hits to return, 1–1000 (default 8). Another value fails before the search. The content budget of the server can return fewer hits.
- `domains` — restrict to specific registered domain ids. Omit to fan out across every registered domain (run `list_domains` to see what's available).
- `prefer` — ranking bias: `"code"` demotes documentation (markdown) so the implementation leads; `"docs"` does the reverse. Omit for the unbiased hybrid ranking. Use `"code"` for "how is X implemented", leave off for "explain X conceptually".
- `kinds` — restrict to chunk kinds: `function`, `container` (type/class/module), or `block` (free text, incl. prose). Omit for every kind.
- `expand` — return the **full enclosing-symbol body** per hit instead of a 4-line snippet. Block hits snap to their enclosing function/type. Default false. Bodies stop at 30,000 characters in total; later hits stay snippets. Expand a small `top_k`, and do not reread files whose bodies were returned.
- `expand` — return the **full enclosing-symbol body** per hit instead of a 4-line snippet. Block hits snap to their enclosing function/type. Default false. Expand a small `top_k`, and do not reread files whose bodies were returned.

## Result budget and expanded hits

The server limits the snippet text of one search to 32 KiB by default (`Search:ContentBudgetBytes`). It cuts the snippet that reaches the limit and drops the hits after it. Ask for a small `top_k`, mainly with `expand`.

When the server shortens a result, the result starts with `note: the server cut this result to its content budget …` and a `next:` step. The result is then not complete: narrow the query, lower `top_k`, or search without `expand`.

With `expand`, each hit tells whether its snippet is the whole symbol:

- `[expanded]` — the snippet is the body of the enclosing symbol.
- `[not expanded: <reason> — …]` — the snippet is the chunk only. Do not treat it as the whole symbol. The reasons:
- `SymbolChunk` — the hit is a function or type chunk, and the snippet is that chunk.
- `NoEnclosingSymbol` — no named declaration contains the hit.
- `GraphLoading` — the graph is not ready. Search again later for the body.
- `Deadline` — the expansion deadline of the server passed. Use `expand_chunk`.
- `ContentBudget` — the body does not fit in the content budget. Use `expand_chunk`.
- `MultipleProjects` — the search spans more than one codebase. Only a search of one codebase expands hits.

## Result freshness

Expand Down
12 changes: 8 additions & 4 deletions src/mcp/tool_types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,8 @@ pub struct SearchArgs {
pub copy: Option<String>,
/// Natural-language query.
pub query: String,
/// Max hits to return. Defaults to 8.
/// Max hits to return, 1–1000. Defaults to 8. Another value fails before
/// the search.
pub top_k: Option<u32>,
/// Restrict to these registered domain ids. Empty / omitted = all.
pub domains: Option<Vec<String>>,
Expand All @@ -34,8 +35,10 @@ pub struct SearchArgs {
/// Omit for every kind.
pub kinds: Option<Vec<String>>,
/// Return full enclosing-symbol bodies instead of 4-line snippets. Defaults
/// false. Bodies stop at 30,000 characters in total; later hits stay
/// snippets. Do not reread files whose bodies were returned.
/// false. The server limits the snippet text of one search to 32 KiB by
/// default. Each hit is marked `[expanded]`, or `[not expanded: <reason>]`
/// when its snippet is only a chunk. Do not reread files whose bodies were
/// returned.
pub expand: Option<bool>,
/// Server scope lens: `local`, `personal`, `organization`, or `global`.
/// Mutually exclusive with `codebase_ids`.
Expand Down Expand Up @@ -150,7 +153,8 @@ pub struct GrepArgs {
pub ignore_case: Option<bool>,
/// Optional path substring to narrow which files are searched.
pub path: Option<String>,
/// Max matches to return, 1–1000. Defaults to 100.
/// Max matches to return, 1–1000. Defaults to 100. Another value fails
/// before the search.
pub max: Option<u32>,
}

Expand Down
Loading
Loading