feat: session history searchable by agents — live+archived, retention (#491)

docs/AGENT_GUIDE.md

@@ -164,6 +164,29 @@ to its subject with `vogt_work_item` (a ref like `WI-42`) or `vogt_project` (a
 slug) so the run's findings file as observations against that subject with the
 same freshness and trust every other kind of evidence carries.
 
+### Search session history — live and archived
+
+Vogt exposes the engine's session history as three reads, so you can find what
+any session has printed without leaving the tool surface (they need only a
+running engine; no capability beyond the ordinary session token):
+
+- `session_search_output` — full-text search over session output. It covers
+  **running** sessions too, not just the archive: each hit carries `live`, so a
+  match in a session that is still going is distinguishable from one in a
+  finished run. Pass `include_live=false` for archive-only.
+- `session_log_tail` — the tail of one session's output log, readable
+  (`strip_ansi` defaults on). Works for a live session as well as an archived
+  one.
+- `session_history_list` — the archived-session listing, newest first.
+
+```console
+$ uv run vogt session search --q "connection refused"
+$ uv run vogt session log --id <session-id>
+```
+
+Each returns an `engine` field that is set (with the reason) when the engine
+could not be asked; an outage reads as an empty result, never as "no history".
+
 ### Name the branch so Vogt can see it
 
 Vogt recognises which work item a git branch belongs to by its **name**, using

docs/ENGINE.md

@@ -1109,13 +1109,21 @@ exactly those.
 
 - `GET /api/history/sessions?limit=&offset=` -> `SessionMetadata[]`; `limit`
   defaults to 50.
-- `GET /api/history/search?q=&limit=` -> `SearchResult[]` — full-text over
-  archived output, ranked; `limit` defaults to 20.
+- `GET /api/history/search?q=&limit=&include_live=` -> `SearchResult[]` —
+  full-text over archived output, ranked; `limit` defaults to 20. Each result
+  carries `live` (default false). `include_live` defaults to **true**: on top
+  of the archived FTS hits, each running session's scrollback is scanned
+  on-demand (the last `history_live_scan_bytes`, ANSI-stripped, same
+  AND-of-terms match) and matches are appended with `live: true`, so output
+  that has not been archived yet is still found (#491). The combined list is
+  held to `limit`. Pass `include_live=false` for archive-only results.
 - `GET /api/history/:id` -> `SessionMetadata`
-- `GET /api/history/:id/log?tail_bytes=` -> `SessionLogPreview`
+- `GET /api/history/:id/log?tail_bytes=&strip_ansi=` -> `SessionLogPreview`
   `{session_id, text, bytes, total_bytes, truncated}` — the *tail*, 64 KiB by
   default. `truncated` is how a client knows it is not looking at the whole
-  run.
+  run. `strip_ansi` (default false) removes the escape sequences a terminal
+  consumes without printing, so `text` is readable plain text; the byte
+  counters still describe the raw tail window that was read.
 - `GET /api/history/:id/download` -> the whole log, streamed, as an attachment
   named for the session.
 - `DELETE /api/history/:id` -> `{"ok": true}`, `404` if it was already gone
@@ -1326,6 +1334,8 @@ Every setting, with the TOML key for a `--config` file and its default
 | `assistant_profiles` | `ENGINE_ASSISTANT_PROFILES_JSON` | `[]` | additional named providers, a JSON array of profile objects (below) |
 | `assistant_default_profile` | `ENGINE_ASSISTANT_DEFAULT_PROFILE` | the implicit `default` | which profile a request that names none runs on |
 | `assistant_log_retention_days` | `ENGINE_ASSISTANT_LOG_RETENTION_DAYS` | `30` | horizon of the durable interaction log, enforced by a daily sweep |
+| `history_retention_days` | `ENGINE_HISTORY_RETENTION_DAYS` | `30` | horizon for archived session history (FTS index + raw logs), enforced by a daily sweep; `0` keeps forever |
+| `history_live_scan_bytes` | `ENGINE_HISTORY_LIVE_SCAN_BYTES` | `262144` | trailing scrollback bytes scanned per live session when a history search sets `include_live` |
 | `assistant_stt_base_urls` | `ENGINE_ASSISTANT_STT_BASE_URLS` (comma-separated) | empty (server STT off) | ordered list of OpenAI-compatible `/audio/transcriptions` bases |
 | `assistant_stt_model` | `ENGINE_ASSISTANT_STT_MODEL` | `whisper-1` | transcription model |
 | `assistant_stt_api_key` | `ENGINE_ASSISTANT_STT_API_KEY` | unset | key for whichever STT entry needs one; a local server needs none |

engine/server/src/agent_tasks.rs

@@ -5192,6 +5192,8 @@ mod tests {
             assistant_profiles: vec![],
             assistant_default_profile: None,
             assistant_log_retention_days: 30,
+            history_retention_days: 30,
+            history_live_scan_bytes: 256 * 1024,
             assistant_stt_base_urls: vec![],
             assistant_stt_api_key: None,
             assistant_stt_model: "whisper-1".into(),

engine/server/src/app.rs

@@ -228,6 +228,10 @@ pub async fn router(cfg: Config) -> (Router, Arc<AppState>) {
     // whatever the last caller passed — the failure mode r18 named in the
     // session log.
     spawn_assistant_log_retention_sweeper(Arc::clone(&state));
+    // Background task: enforce the session-history retention horizon on the same
+    // daily schedule (#491), so `history.db` and the raw `session-logs/` cannot
+    // grow without bound. A no-op when history is disabled or the horizon is 0.
+    spawn_history_retention_sweeper(Arc::clone(&state));
 
     // Public: /healthz, /api/config, /api/push/public-key. None reveal secrets.
     let public = Router::new()
@@ -442,6 +446,33 @@ fn spawn_assistant_log_retention_sweeper(state: Arc<AppState>) {
     });
 }
 
+/// Enforce the session-history retention horizon on a daily schedule (#491),
+/// mirroring the assistant-log sweeper: one sweep at startup and then every 24
+/// hours. A no-op when history is disabled, and when the horizon is `0` — the
+/// documented "keep forever" — so the sweep never deletes everything by
+/// treating 0 days as "older than now".
+fn spawn_history_retention_sweeper(state: Arc<AppState>) {
+    let Some(history) = state.history.clone() else {
+        return;
+    };
+    let retention_days = state.config.history_retention_days;
+    if retention_days == 0 {
+        return;
+    }
+    tokio::spawn(async move {
+        loop {
+            match history.cleanup_old_sessions(retention_days).await {
+                Ok(removed) if removed > 0 => {
+                    tracing::info!(removed, retention_days, "session history retention sweep")
+                }
+                Ok(_) => {}
+                Err(e) => tracing::warn!("session history retention sweep failed: {e}"),
+            }
+            tokio::time::sleep(std::time::Duration::from_secs(24 * 60 * 60)).await;
+        }
+    });
+}
+
 /// The engine's ordinary error body, at 404, for a path under `/api` that no
 /// route claimed. Deliberately the same shape every other failure here uses —
 /// a client that already parses `{"error": ...}` needs nothing new to read it.

engine/server/src/assistant.rs

@@ -1909,6 +1909,8 @@ mod tests {
             assistant_profiles: vec![],
             assistant_default_profile: None,
             assistant_log_retention_days: 30,
+            history_retention_days: 30,
+            history_live_scan_bytes: 256 * 1024,
             assistant_stt_base_urls: vec![],
             assistant_stt_api_key: None,
             assistant_stt_model: "whisper-1".into(),

engine/server/src/auth.rs

@@ -510,6 +510,8 @@ mod tests {
             assistant_profiles: vec![],
             assistant_default_profile: None,
             assistant_log_retention_days: 30,
+            history_retention_days: 30,
+            history_live_scan_bytes: 256 * 1024,
             assistant_stt_base_urls: vec![],
             assistant_stt_api_key: None,
             assistant_stt_model: "whisper-1".into(),

engine/server/src/config.rs

@@ -272,6 +272,16 @@ pub struct Config {
     /// days. Enforced on a schedule so the horizon is a configured maximum
     /// rather than whatever the last caller passed. Defaults to 30.
     pub assistant_log_retention_days: u32,
+    /// Retention horizon for archived session history (the FTS index and the
+    /// raw scrollback logs), in days. Enforced by a daily sweep, mirroring the
+    /// assistant-log horizon, so `history.db` and `session-logs/` cannot grow
+    /// without bound. Defaults to 30; `0` disables the sweep (keep forever).
+    pub history_retention_days: u32,
+    /// How many trailing bytes of each *live* session's scrollback a history
+    /// search scans when `include_live` is set. Bounds the per-search cost of
+    /// live coverage (no DB writes; the scan is read-only). Defaults to 256
+    /// KiB.
+    pub history_live_scan_bytes: u64,
     /// Server-side speech (FR-T12), configured **independently of the chat
     /// profile**: the whole point of the requirement is that a deployment
     /// whose chat runs through OpenRouter (which does not front audio
@@ -387,6 +397,8 @@ struct FileConfig {
     assistant_profiles: Option<Vec<AssistantProfile>>,
     assistant_default_profile: Option<String>,
     assistant_log_retention_days: Option<u32>,
+    history_retention_days: Option<u32>,
+    history_live_scan_bytes: Option<u64>,
     assistant_stt_base_urls: Option<Vec<String>>,
     assistant_stt_api_key: Option<String>,
     assistant_stt_model: Option<String>,
@@ -685,6 +697,12 @@ pub fn load(
         assistant_log_retention_days: parse_u32_env("ENGINE_ASSISTANT_LOG_RETENTION_DAYS")?
             .or(from_file.assistant_log_retention_days)
             .unwrap_or(30),
+        history_retention_days: parse_u32_env("ENGINE_HISTORY_RETENTION_DAYS")?
+            .or(from_file.history_retention_days)
+            .unwrap_or(30),
+        history_live_scan_bytes: parse_u64_env("ENGINE_HISTORY_LIVE_SCAN_BYTES")?
+            .or(from_file.history_live_scan_bytes)
+            .unwrap_or(256 * 1024),
         assistant_stt_base_urls,
         assistant_stt_api_key,
         assistant_stt_model: from_file

engine/server/src/history.rs

@@ -49,6 +49,11 @@ pub struct SearchResult {
     pub created_at: String,
     pub match_snippet: String,
     pub rank: f64,
+    /// True when the hit came from a *live* session's scrollback (an on-demand
+    /// bounded scan), rather than the archived FTS index. Archived rows default
+    /// this to false, including when an older engine omits the field.
+    #[serde(default)]
+    pub live: bool,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -299,6 +304,7 @@ impl SessionHistory {
                 created_at: row.get("created_at"),
                 match_snippet: row.get("match_snippet"),
                 rank: row.get("rank"),
+                live: false,
             })
             .collect();
 
@@ -467,7 +473,18 @@ impl SessionHistory {
     }
 
     /// Read the tail of an archived raw session log for replay-oriented views.
-    pub async fn read_log_preview(&self, id: Uuid, tail_bytes: u64) -> Result<SessionLogPreview> {
+    ///
+    /// With `strip_ansi`, the escape sequences a terminal consumes without
+    /// printing are removed from `text` (the same stripper the archive path
+    /// uses), so an agent reading over MCP gets plain text rather than a wall
+    /// of `\x1b[` noise. The byte counters still describe the raw tail window
+    /// that was read from disk — `text` is the rendered view of it.
+    pub async fn read_log_preview(
+        &self,
+        id: Uuid,
+        tail_bytes: u64,
+        strip_ansi: bool,
+    ) -> Result<SessionLogPreview> {
         let path = self.log_path(id);
         let mut file = tokio::fs::File::open(&path)
             .await
@@ -496,9 +513,15 @@ impl SessionHistory {
                 .map_err(|e| ApiError::Internal(format!("failed to read session log: {e}")))?;
         }
 
+        let text = if strip_ansi {
+            String::from_utf8_lossy(&crate::activity::strip_ansi(&buf)).into_owned()
+        } else {
+            String::from_utf8_lossy(&buf).into_owned()
+        };
+
         Ok(SessionLogPreview {
             session_id: id.to_string(),
-            text: String::from_utf8_lossy(&buf).into_owned(),
+            text,
             bytes,
             total_bytes,
             truncated: total_bytes > bytes,
@@ -584,6 +607,88 @@ fn user_query_to_fts(query: &str) -> Option<String> {
     }
 }
 
+/// Lowercased alphanumeric/underscore tokens (max 16), the same tokenisation
+/// the FTS query builder uses, for substring-matching live-session scrollback.
+/// Empty when the query carries no usable term.
+pub fn query_tokens(query: &str) -> Vec<String> {
+    query
+        .split(|c: char| !c.is_alphanumeric() && c != '_')
+        .filter_map(|part| {
+            let trimmed = part.trim();
+            if trimmed.is_empty() {
+                None
+            } else {
+                Some(trimmed.to_lowercase())
+            }
+        })
+        .take(16)
+        .collect()
+}
+
+/// Build a live-search hit from a live session's raw scrollback tail, when
+/// every query token appears (case-insensitively) in the ANSI-stripped text.
+/// Mirrors the archived path's AND-of-terms semantics and its plain-text
+/// snippet contract — terminal output is untrusted, so the snippet is never
+/// marked up; the PWA owns highlighting at its text sink.
+pub fn live_match(
+    session_id: &str,
+    session_name: &str,
+    created_at: &str,
+    raw_tail: &[u8],
+    tokens: &[String],
+) -> Option<SearchResult> {
+    if tokens.is_empty() {
+        return None;
+    }
+    let stripped = crate::activity::strip_ansi(raw_tail);
+    let text = String::from_utf8_lossy(&stripped);
+    let haystack = text.to_lowercase();
+    if !tokens.iter().all(|t| haystack.contains(t.as_str())) {
+        return None;
+    }
+    Some(SearchResult {
+        session_id: session_id.to_string(),
+        session_name: session_name.to_string(),
+        created_at: created_at.to_string(),
+        match_snippet: live_snippet(&text, &tokens[0]),
+        // Archived hits carry FTS `rank` (ascending = best); live hits have no
+        // FTS score. 0.0 keeps them ahead of nothing in particular; the `live`
+        // flag, not the rank, is what a consumer keys on.
+        rank: 0.0,
+        live: true,
+    })
+}
+
+/// A readable one-line snippet for a live hit: the first scrollback line that
+/// contains the leading token (case-insensitively), trimmed and length-capped;
+/// failing that, the first non-empty line. Line-oriented rather than a byte
+/// window, so it stays on character boundaries and reads cleanly.
+fn live_snippet(text: &str, first_token: &str) -> String {
+    const MAX_CHARS: usize = 160;
+    for line in text.lines() {
+        let trimmed = line.trim();
+        if !trimmed.is_empty() && trimmed.to_lowercase().contains(first_token) {
+            return truncate_chars(trimmed, MAX_CHARS);
+        }
+    }
+    text.lines()
+        .map(str::trim)
+        .find(|line| !line.is_empty())
+        .map(|line| truncate_chars(line, MAX_CHARS))
+        .unwrap_or_default()
+}
+
+/// Truncate to at most `max` characters (not bytes), appending an ellipsis when
+/// anything was dropped.
+fn truncate_chars(text: &str, max: usize) -> String {
+    if text.chars().count() <= max {
+        return text.to_string();
+    }
+    let mut out: String = text.chars().take(max).collect();
+    out.push_str("...");
+    out
+}
+
 fn remove_log_file(path: PathBuf) -> Result<()> {
     match std::fs::remove_file(&path) {
         Ok(()) => Ok(()),
@@ -617,3 +722,66 @@ fn summarize_regular_files(path: &Path) -> Result<(usize, u64)> {
         ))),
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn query_tokens_lowercases_splits_and_keeps_underscores() {
+        assert_eq!(
+            query_tokens("Hello, World_1!"),
+            vec!["hello".to_string(), "world_1".to_string()]
+        );
+    }
+
+    #[test]
+    fn query_tokens_are_empty_for_a_query_with_no_usable_term() {
+        assert!(query_tokens("   ").is_empty());
+        assert!(query_tokens("!!! ---").is_empty());
+    }
+
+    #[test]
+    fn query_tokens_cap_at_sixteen() {
+        let many = (0..30)
+            .map(|i| format!("w{i}"))
+            .collect::<Vec<_>>()
+            .join(" ");
+        assert_eq!(query_tokens(&many).len(), 16);
+    }
+
+    #[test]
+    fn live_match_requires_every_token_and_strips_ansi() {
+        // Two tokens, both present across the ANSI-coloured tail.
+        let tail = b"\x1b[2K\x1b[1Ghello there\n\x1b[31mworld needle\x1b[0m\n";
+        let tokens = query_tokens("needle world");
+        let hit = live_match("sid", "sname", "2026-01-01T00:00:00Z", tail, &tokens)
+            .expect("all tokens present -> a hit");
+        assert!(hit.live);
+        assert_eq!(hit.session_id, "sid");
+        assert_eq!(hit.session_name, "sname");
+        assert_eq!(hit.rank, 0.0);
+        // Snippet is the matching line, ANSI removed, plain text (untrusted
+        // output preserved as data — never marked up).
+        assert_eq!(hit.match_snippet, "world needle");
+        assert!(!hit.match_snippet.contains('\u{1b}'));
+    }
+
+    #[test]
+    fn live_match_returns_none_when_a_token_is_absent() {
+        let tail = b"only has the word alpha\n";
+        let tokens = query_tokens("alpha beta");
+        assert!(live_match("s", "n", "t", tail, &tokens).is_none());
+    }
+
+    #[test]
+    fn live_match_returns_none_for_empty_tokens() {
+        assert!(live_match("s", "n", "t", b"anything", &[]).is_none());
+    }
+
+    #[test]
+    fn truncate_chars_appends_ellipsis_only_when_dropping() {
+        assert_eq!(truncate_chars("short", 10), "short");
+        assert_eq!(truncate_chars("abcdef", 3), "abc...");
+    }
+}