Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions crates/k-wiki/src/search/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -328,8 +328,15 @@ fn highlight_match(value: &str, normalized_query: &str, tokens: &[String]) -> Op
}

let (position, length) = matches.into_iter().min_by_key(|(position, _)| *position)?;
let start = position.saturating_sub(SNIPPET_RADIUS);
let end = value.len().min(position + length + SNIPPET_RADIUS);
let mut start = position.saturating_sub(SNIPPET_RADIUS);
let mut end = value.len().min(position + length + SNIPPET_RADIUS);
// Byte-based context windows can land inside a multibyte character.
while !value.is_char_boundary(start) {
start -= 1;
}
while !value.is_char_boundary(end) {
end += 1;
}

let prefix = escape_html(&value[start..position]);
let matched = escape_html(&value[position..position + length]);
Expand Down
96 changes: 96 additions & 0 deletions crates/k-wiki/tests/api_integration.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1647,6 +1647,102 @@ fn mcp_stdio_binary_advertises_the_packaged_knowledge_wiki_schema() {
}));
}

#[test]
fn mcp_stdio_memory_recall_with_unicode_keeps_the_transport_open() {
let temp = TestDir::new("k-wiki-mcp-unicode-memory");
let bundle = temp.path().join("docs");
fs::create_dir_all(&bundle).expect("create bundle");
fs::write(
bundle.join("index.md"),
"---\nokf_version: '0.1'\ntitle: Docs\n---\n# Docs\n",
)
.expect("write bundle index");
let body = format!("🦀{}needle{}é", "x".repeat(47), "x".repeat(47));
let messages = [
json!({
"jsonrpc": "2.0", "id": 1, "method": "initialize",
"params": {"protocolVersion": "2025-11-25"}
}),
json!({"jsonrpc": "2.0", "method": "notifications/initialized"}),
json!({
"jsonrpc": "2.0", "id": 2, "method": "tools/call",
"params": {"name": "wiki_memory_record", "arguments": {
"bundle_id": "docs", "memory_id": "unicode-note", "kind": "episodic",
"title": "Unicode note", "body_markdown": body,
"owner": "repository-agent", "created_at": "2026-09-30T00:00:00Z",
"sources": [{"kind": "test", "reference": "unicode fixture"}]
}}
}),
json!({
"jsonrpc": "2.0", "id": 3, "method": "tools/call",
"params": {"name": "wiki_memory_transition", "arguments": {
"bundle_id": "docs", "memory_id": "unicode-note", "to_status": "active",
"actor": "reviewer", "transitioned_at": "2026-09-30T01:00:00Z",
"reason": "verified fixture"
}}
}),
json!({
"jsonrpc": "2.0", "id": 4, "method": "tools/call",
"params": {"name": "wiki_memory_recall", "arguments": {
"bundle_id": "docs", "text": "needle"
}}
}),
json!({
"jsonrpc": "2.0", "id": 5, "method": "tools/call",
"params": {"name": "wiki_memory_recall", "arguments": {
"bundle_id": "docs", "text": "needle", "include_structured_content": true
}}
}),
json!({"jsonrpc": "2.0", "id": 6, "method": "ping"}),
];

let mut child = Command::new(env!("CARGO_BIN_EXE_k-wiki"))
.arg("mcp")
.arg(&bundle)
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.expect("start MCP binary");
{
let mut stdin = child.stdin.take().expect("MCP stdin");
for message in messages {
writeln!(stdin, "{message}").expect("write MCP request");
}
}
let output = child.wait_with_output().expect("wait for MCP binary");
assert!(
output.status.success(),
"{}",
String::from_utf8_lossy(&output.stderr)
);
let responses = String::from_utf8(output.stdout)
.expect("UTF-8 MCP output")
.lines()
.map(|line| serde_json::from_str::<Value>(line).expect("JSON-RPC response"))
.collect::<Vec<_>>();
assert_eq!(responses.len(), 6);
for (index, response) in responses.iter().enumerate() {
assert_eq!(response["id"], index + 1);
assert!(response.get("error").is_none(), "{response}");
}
let recalled: Value = serde_json::from_str(
responses[3]["result"]["content"][0]["text"]
.as_str()
.expect("text recall result"),
)
.expect("recall payload");
assert_eq!(recalled["kind"], "memory_recalled");
assert_eq!(recalled["result"].as_array().unwrap().len(), 1);
assert_eq!(recalled["result"][0]["memory_id"], "unicode-note");
assert_eq!(
recalled["result"][0]["snippet"],
format!("🦀{}<mark>needle</mark>{}é", "x".repeat(47), "x".repeat(47))
);
assert_eq!(responses[4]["result"]["structuredContent"], recalled);
assert_eq!(responses[5]["result"], json!({}));
}

#[tokio::test]
async fn preview_http_dispatches_health_and_serves_static_content_with_security_headers() {
let temp = TestDir::new("k-wiki-http");
Expand Down
35 changes: 35 additions & 0 deletions crates/k-wiki/tests/search.rs
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,41 @@ fn snippets_are_html_escaped_bounded_and_deterministic() {
assert!(snippet.len() <= 160);
}

#[test]
fn snippet_windows_preserve_utf8_boundaries() {
for character in ['é', '中', '🦀'] {
for offset in 1..character.len_utf8() {
let padding = "x".repeat(48 - offset);
for term in ["needle", "mémoire", "知识"] {
for (body, expected) in [
(
format!("{character}{padding}{term}"),
format!("{character}{padding}<mark>{term}</mark>"),
),
(
format!("{term}{padding}{character}"),
format!("<mark>{term}</mark>{padding}{character}"),
),
] {
let mut projection = sample_projection();
let concept = &mut projection.bundles[0].concepts[0];
concept.title = Some("Unicode note".into());
concept.description = None;
concept.headings.clear();
concept.citations.clear();
concept.extensions.clear();
concept.body_markdown = body;

let index = SearchIndex::build(&projection);
let results = index.search(&SearchQuery::new(term));
assert_eq!(results.len(), 1);
assert_eq!(results[0].snippet.as_deref(), Some(expected.as_str()));
}
}
}
}
}

#[test]
fn serialized_index_round_trips_deterministically() {
let index = SearchIndex::build(&sample_projection());
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
---
agent_memory:
version: 1
kind: episodic
scope: repository
status: active
owner: repository-maintainers
created_at: 2026-09-30T21:48:51Z
last_verified_at: 2026-09-30T21:48:51Z
verified_by: codex
review_after: null
supersedes: []
superseded_by: null
sources:
- kind: source
reference: crates/k-wiki/src/search/mod.rs::highlight_match
content_hash: null
- kind: test
reference: crates/k-wiki/tests/search.rs::snippet_windows_preserve_utf8_boundaries
content_hash: null
- kind: test
reference: crates/k-wiki/tests/api_integration.rs::mcp_stdio_memory_recall_with_unicode_keeps_the_transport_open
content_hash: null
- kind: test-run
reference: '2026-10-01 cargo nextest run -p k-wiki: 98 passed; cargo fmt --all --check; cargo clippy -p k-wiki --all-targets -- -D warnings'
content_hash: null
history:
- from: candidate
to: active
actor: codex
at: 2026-09-30T21:48:51Z
reason: Reviewed the panic reproduction, UTF-8 boundary repair, direct-search matrix, and real MCP recall/ping regression against source; all 98 k-wiki tests, formatting, and strict Clippy passed.
description: Memory recall can close stdio through a shared search-snippet panic when a byte-based excerpt boundary splits a UTF-8 character.
tags:
- k-wiki
- mcp
- memory
- search
- unicode
timestamp: 2026-09-30T21:48:51Z
title: Unicode snippet boundaries can terminate the wiki MCP transport
type: agent-memory
---
A `wiki_memory_recall` transport closure can originate in shared search snippet generation. A disposable MCP process with an activated Unicode memory reproduced a main-thread panic when the snippet context boundary split a multibyte character. The process exited during recall before answering the request, which leaves clients with a closed transport rather than a typed wiki error.

The repair moves the context window endpoints outward to valid UTF-8 character boundaries before slicing. Regression coverage exercises every interior byte boundary of two-, three-, and four-byte characters on both sides of ASCII and Unicode matches. A real stdio process records and activates a Unicode memory, recalls it with both text and structured output, and successfully answers a subsequent ping. The full k-wiki suite passed 98 tests; formatting and strict Clippy also passed.

When diagnosing a closed stdio transport during text retrieval, reproduce the call in an isolated MCP process and inspect its exit status and stderr before assuming an idle timeout or connection lifecycle defect. Preserve Unicode in the reproduction because an ASCII-only fixture hides this failure.
Loading