From c0cc1fb42dee89939b01fe9709f015078569a32a Mon Sep 17 00:00:00 2001 From: adhaile Date: Tue, 29 Sep 2026 10:35:56 -0700 Subject: [PATCH] feat(vgr): derive fail-closed verification capabilities Signed-off-by: adhaile --- crates/libsy/src/algorithms.rs | 1 + crates/libsy/src/algorithms/vgr.rs | 107 +++++++++++ crates/libsy/src/algorithms/vgr/render.rs | 134 ++++++++++++++ crates/libsy/src/algorithms/vgr/tests.rs | 134 ++++++++++++++ crates/libsy/src/algorithms/vgr/text.rs | 206 ++++++++++++++++++++++ 5 files changed, 582 insertions(+) create mode 100644 crates/libsy/src/algorithms/vgr.rs create mode 100644 crates/libsy/src/algorithms/vgr/render.rs create mode 100644 crates/libsy/src/algorithms/vgr/tests.rs create mode 100644 crates/libsy/src/algorithms/vgr/text.rs diff --git a/crates/libsy/src/algorithms.rs b/crates/libsy/src/algorithms.rs index cd2d36b50..4a1401db1 100644 --- a/crates/libsy/src/algorithms.rs +++ b/crates/libsy/src/algorithms.rs @@ -17,6 +17,7 @@ pub mod plan_execute; pub mod rand; pub mod stage; pub mod subagent; +pub(crate) mod vgr; pub mod util; diff --git a/crates/libsy/src/algorithms/vgr.rs b/crates/libsy/src/algorithms/vgr.rs new file mode 100644 index 000000000..b3adcff7a --- /dev/null +++ b/crates/libsy/src/algorithms/vgr.rs @@ -0,0 +1,107 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Verification-gated routing: fail-closed capability derivation. +//! +//! Capabilities are never client-declared. They come from the router's own +//! typing of the request, the attempt it produced locally, and its own reading +//! of the tool results in the conversation. No client-reachable input selects +//! a weaker regime than the default one; unsupported content and missing +//! evidence select [`Branch::Unknown`], which never commits. + +#![allow(dead_code)] + +use switchyard_protocol::Request; + +use self::text::ToolRecord; + +mod render; +mod text; + +#[cfg(test)] +mod tests; + +/// The verification regime a request's capabilities license. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +enum Branch { + /// Code or test activity, with no sandboxed checker to appeal to. + Coding, + /// A single-turn request typed as answer-seeking. + Answer, + /// Conversational traffic. + Chat, + /// A tool-using session verified from its trajectory. + Agentic, + /// Anything else with an attempt to verify. + DefaultVerified, + /// Nothing to verify. + #[default] + Unknown, +} + +/// A task type produced by the router's own typing call; `None` abstains. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum TaskType { + Coding, + Agentic, + Answer, + Chat, +} + +/// The complete input the decision core sees. +#[derive(Clone, Debug, Default, PartialEq)] +struct Capabilities { + branch: Branch, + /// The judged view of the request and attempt. + transcript: Option, + /// The router's own tool-result record; unused by the answer regime. + tools: Option, +} + +/// Derives capabilities from the router's own evidence, failing closed. +fn derive_capabilities( + request: &Request, + attempt: &str, + task_type: Option, +) -> Capabilities { + let (turns, unsupported) = text::turns(request); + if unsupported || text::user_task_text(&turns).trim().is_empty() || attempt.trim().is_empty() { + return Capabilities::default(); + } + let caps = |branch, transcript| Capabilities { + branch, + transcript, + tools: Some(ToolRecord::from_request(request)), + }; + + if task_type == Some(TaskType::Answer) && !text::has_assistant_turn(&turns) { + return Capabilities { + tools: None, + ..caps( + Branch::Answer, + Some(render::render_session(&turns, attempt)), + ) + }; + } + // A tool trajectory is judged as agentic work unless typed conversational. + if task_type == Some(TaskType::Agentic) + || (text::has_tool_trajectory(request) + && !matches!(task_type, Some(TaskType::Answer | TaskType::Chat))) + { + return caps( + Branch::Agentic, + Some(render::render_agentic_view(request, &turns, attempt)), + ); + } + if text::observed_hardening(attempt) || task_type == Some(TaskType::Coding) { + return caps(Branch::Coding, None); + } + let transcript = Some(render::render_session(&turns, attempt)); + if text::observed_tool_activity(attempt) { + caps(Branch::Agentic, transcript) + } else if matches!(task_type, Some(TaskType::Chat | TaskType::Answer)) { + caps(Branch::Chat, transcript) + } else { + caps(Branch::DefaultVerified, transcript) + } +} diff --git a/crates/libsy/src/algorithms/vgr/render.rs b/crates/libsy/src/algorithms/vgr/render.rs new file mode 100644 index 000000000..e7d6570fc --- /dev/null +++ b/crates/libsy/src/algorithms/vgr/render.rs @@ -0,0 +1,134 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! The judged views a verifier reads. +//! +//! Each section is redacted before its budget applies, and user requirements +//! render in full while trajectory and attempt evidence clip. + +use switchyard_protocol::{ContentBlock, Request, Role}; + +use super::text::{Turn, clip_mid, content_text, redact}; + +const ASSISTANT_TURN_BUDGET: usize = 1200; +pub(super) const ATTEMPT_BUDGET: usize = 2200; +const AGENTIC_TRAJECTORY_BUDGET: usize = 12_000; +const AGENTIC_EVENT_BUDGET: usize = 1400; + +/// Redacted non-empty turns, and the index of the latest user turn among them. +fn rendered(turns: &[Turn]) -> (Vec<(Role, String)>, Option) { + let rendered: Vec<(Role, String)> = turns + .iter() + .filter(|turn| !turn.text.trim().is_empty()) + .map(|turn| (turn.role, redact(&turn.text))) + .collect(); + let latest_user = rendered.iter().rposition(|(role, _)| *role == Role::User); + (rendered, latest_user) +} + +fn role_label(role: Role) -> &'static str { + match role { + Role::System => "system", + Role::Developer => "developer", + Role::User => "user", + Role::Assistant => "assistant", + Role::Tool => "tool", + } +} + +fn attempt_section(attempt: &str) -> String { + clip_mid(&redact(attempt), ATTEMPT_BUDGET, 1.0 / 3.0) +} + +/// Earlier turns first, then the latest user instruction and the attempt. +pub(super) fn render_session(turns: &[Turn], attempt: &str) -> String { + let (turns, latest_user) = rendered(turns); + let mut lines: Vec = turns + .iter() + .enumerate() + .filter(|(index, _)| Some(*index) != latest_user) + .map(|(_, (role, text))| { + let text = if matches!(role, Role::User | Role::System | Role::Developer) { + text.clone() + } else { + clip_mid(text, ASSISTANT_TURN_BUDGET, 0.5) + }; + format!("[{}] {text}", role_label(*role)) + }) + .collect(); + if let Some(index) = latest_user { + lines.push(format!("[user (latest)] {}", turns[index].1)); + } + lines.push(format!("[assistant attempt] {}", attempt_section(attempt))); + lines.join("\n") +} + +/// The user task and updates, the tool trajectory, and the attempt. +/// +/// Framework instructions are omitted. +pub(super) fn render_agentic_view(request: &Request, turns: &[Turn], attempt: &str) -> String { + let (turns, _) = rendered(turns); + let requirements = turns + .iter() + .filter(|(role, _)| *role == Role::User) + .enumerate() + .map(|(index, (_, text))| match index { + 0 => format!("[user task] {text}"), + _ => format!("[user update {index}] {text}"), + }) + .collect::>() + .join("\n"); + let trajectory = agentic_trajectory(request); + let mut parts = vec![format!("USER TASK AND UPDATES:\n{requirements}")]; + if !trajectory.is_empty() { + parts.push(format!( + "TOOL TRAJECTORY:\n{}", + clip_mid(&trajectory.join("\n"), AGENTIC_TRAJECTORY_BUDGET, 1.0 / 3.0) + )); + } + parts.push(format!("CURRENT ATTEMPT:\n{}", attempt_section(attempt))); + parts.join("\n\n") +} + +/// Bounded assistant and tool events after the first user task. +fn agentic_trajectory(request: &Request) -> Vec { + let mut messages = request.llm_request.messages.iter(); + for message in messages.by_ref() { + if message.role == Role::User && !content_text(&message.content).0.trim().is_empty() { + break; + } + } + let mut events = Vec::new(); + for message in messages { + for block in &message.content { + let event = match block { + ContentBlock::Text { text } | ContentBlock::Refusal { text } => { + match message.role { + Role::Assistant => Some(("assistant", text.clone())), + Role::Tool => Some(("tool result", text.clone())), + _ => None, + } + } + ContentBlock::ToolCall(call) => { + Some(("tool call", format!("{}({})", call.name, call.arguments))) + } + ContentBlock::ToolResult(result) => Some(( + if result.is_error == Some(true) { + "tool error" + } else { + "tool result" + }, + content_text(&result.content).0, + )), + _ => None, + }; + if let Some((label, text)) = event { + events.push(format!( + "[{label}] {}", + clip_mid(&redact(&text), AGENTIC_EVENT_BUDGET, 0.5) + )); + } + } + } + events +} diff --git a/crates/libsy/src/algorithms/vgr/tests.rs b/crates/libsy/src/algorithms/vgr/tests.rs new file mode 100644 index 000000000..37cdadc28 --- /dev/null +++ b/crates/libsy/src/algorithms/vgr/tests.rs @@ -0,0 +1,134 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use switchyard_protocol::{ + ContentBlock, ImageSource, LlmRequest, Message, Request, Role, ToolCall, ToolResult, +}; + +use super::text::ToolRecord; +use super::{Branch, TaskType, derive_capabilities}; + +pub(super) fn request(messages: Vec) -> Request { + Request { + llm_request: LlmRequest { + messages, + ..LlmRequest::default() + }, + ..Default::default() + } +} + +pub(super) fn call(id: &str) -> Message { + Message { + role: Role::Assistant, + content: vec![ContentBlock::ToolCall(ToolCall { + id: id.into(), + name: "bash".into(), + arguments: serde_json::json!({"command": "ls"}), + })], + } +} + +pub(super) fn result(id: &str, text: &str) -> Message { + Message { + role: Role::Tool, + content: vec![ContentBlock::ToolResult(ToolResult { + tool_call_id: id.into(), + content: vec![ContentBlock::Text { text: text.into() }], + is_error: None, + })], + } +} + +fn branch(request: &Request, attempt: &str, task_type: Option) -> Branch { + derive_capabilities(request, attempt, task_type).branch +} + +#[test] +fn router_typing_selects_the_verification_regime() { + let single = request(vec![Message::text(Role::User, "Help with this task")]); + for (task_type, expected) in [ + (Some(TaskType::Coding), Branch::Coding), + (Some(TaskType::Agentic), Branch::Agentic), + (Some(TaskType::Answer), Branch::Answer), + (Some(TaskType::Chat), Branch::Chat), + (None, Branch::DefaultVerified), + ] { + assert_eq!(branch(&single, "done", task_type), expected); + } + + // A tool trajectory outranks a coding type; only conversation types keep it off. + let session = request(vec![ + Message::text(Role::User, "fix the build"), + call("c1"), + result("c1", "ok"), + ]); + assert_eq!( + branch(&session, "done", Some(TaskType::Coding)), + Branch::Agentic + ); + assert_eq!(branch(&session, "done", Some(TaskType::Chat)), Branch::Chat); + assert_eq!(branch(&single, "```\nx\n```", None), Branch::Coding); +} + +#[test] +fn missing_or_unrepresentable_evidence_fails_closed() { + let task = request(vec![Message::text(Role::User, "task")]); + assert_eq!(branch(&task, " ", None), Branch::Unknown); + assert_eq!(branch(&request(Vec::new()), "done", None), Branch::Unknown); + + let image = request(vec![Message { + role: Role::User, + content: vec![ContentBlock::Image { + source: ImageSource::Url { + url: "https://example.test/a.png".into(), + detail: None, + }, + }], + }]); + assert_eq!(branch(&image, "done", None), Branch::Unknown); +} + +#[test] +fn a_zero_exit_code_overrules_every_error_inference() { + let record = ToolRecord::from_request(&request(vec![ + result( + "a", + r#"{"output": "cat: HEAD: No such file", "exit_code": 0, "error": null}"#, + ), + result( + "b", + r#"{"output": "boom", "exit_code": 127, "error": "spawn failed"}"#, + ), + result("c", r#"{"error": "record not found"}"#), + result("d", "Traceback (most recent call last):\n ValueError"), + result("e", "done"), + ])); + assert_eq!( + record, + ToolRecord { + errors: 3, + results: 5, + tail_clean: true, + clean_tail: 1, + } + ); +} + +#[test] +fn the_agentic_view_keeps_the_task_and_redacts_the_trajectory() { + let session = request(vec![ + Message::text(Role::System, "framework boilerplate"), + Message::text(Role::User, "rotate the key"), + call("c1"), + result("c1", "Authorization: Bearer abcdefghijklmnopqrstuvwxyz"), + ]); + let caps = derive_capabilities(&session, "rotated", None); + let view = caps.transcript.unwrap_or_default(); + assert_eq!(caps.branch, Branch::Agentic); + assert!(view.starts_with("USER TASK AND UPDATES:\n[user task] rotate the key")); + assert!(view.contains("[tool call] bash")); + assert!(view.contains("[REDACTED]") && !view.contains("abcdefghij")); + assert!(!view.contains("boilerplate")); + assert!(view.ends_with("CURRENT ATTEMPT:\nrotated")); +} diff --git a/crates/libsy/src/algorithms/vgr/text.rs b/crates/libsy/src/algorithms/vgr/text.rs new file mode 100644 index 000000000..d9f9888dc --- /dev/null +++ b/crates/libsy/src/algorithms/vgr/text.rs @@ -0,0 +1,206 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Pure text primitives shared by derivation and the judged views. +//! +//! Budgets and clipping count characters, not bytes. + +use std::sync::LazyLock; + +use regex::Regex; +use serde_json::Value; +use switchyard_protocol::{ContentBlock, InstructionBlock, Message, Request, Role}; + +use crate::algorithms::util::tool_signals::classify_text; + +/// One flattened turn: instructions first, then messages. +#[derive(Clone, Debug, PartialEq)] +pub(super) struct Turn { + pub(super) role: Role, + pub(super) text: String, +} + +static REDACT_PATTERNS: LazyLock> = LazyLock::new(|| { + [ + r"\b(sk|rk|pk)-[A-Za-z0-9_\-]{16,}\b", + r"\bAKIA[0-9A-Z]{16}\b", + r"(?i)\bbearer\s+[A-Za-z0-9._\-]{16,}", + r"\b[\w.+-]+@[\w-]+\.[\w.]+\b", + ] + .iter() + .filter_map(|pattern| Regex::new(pattern).ok()) + .collect() +}); + +/// Code or test activity in the attempt itself; prose never matches. +static CODE_ACTIVITY: LazyLock> = LazyLock::new(|| { + Regex::new(concat!( + r"(?m)```|^\$ |^diff --git|^@@ |", + r"\bTraceback \(most recent call last\)|\b\d+ (?:passed|failed)\b|", + r"^(?:\$ |> )?(?:pytest|unittest|npm (?:test|run)|cargo (?:test|build)|", + r"go test|make test)\b", + )) + .ok() +}); + +static TOOL_ACTIVITY: LazyLock> = LazyLock::new(|| { + Regex::new(r#"(?m)^\[tool\b|^\[[^\]\n]* calls tools?:|"tool_calls?"\s*:"#).ok() +}); + +/// Text of a content sequence, and whether it carried media or unknown blocks. +pub(super) fn content_text(content: &[ContentBlock]) -> (String, bool) { + let mut parts: Vec<&str> = Vec::new(); + let mut unsupported = false; + for block in content { + match block { + ContentBlock::Text { text } | ContentBlock::Refusal { text } => parts.push(text), + ContentBlock::ToolCall(_) + | ContentBlock::ToolResult(_) + | ContentBlock::Reasoning { .. } => {} + _ => unsupported = true, + } + } + (parts.join("\n"), unsupported) +} + +pub(super) fn turns(request: &Request) -> (Vec, bool) { + let mut flattened = Vec::new(); + let mut unsupported = false; + let mut push = |role: Role, content: &[ContentBlock]| { + let (text, block_unsupported) = content_text(content); + unsupported |= block_unsupported; + flattened.push(Turn { role, text }); + }; + for InstructionBlock { role, content } in &request.llm_request.instructions { + push(*role, content); + } + for Message { role, content } in &request.llm_request.messages { + push(*role, content); + } + (flattened, unsupported) +} + +/// The latest non-empty user turn. +pub(super) fn user_task_text(turns: &[Turn]) -> String { + turns + .iter() + .rev() + .find(|turn| turn.role == Role::User && !turn.text.trim().is_empty()) + .map(|turn| turn.text.clone()) + .unwrap_or_default() +} + +pub(super) fn has_assistant_turn(turns: &[Turn]) -> bool { + turns.iter().any(|turn| turn.role == Role::Assistant) +} + +/// Whether the conversation contains an executed or proposed tool step. +pub(super) fn has_tool_trajectory(request: &Request) -> bool { + request.llm_request.messages.iter().any(|message| { + message.role == Role::Tool + || message.content.iter().any(|block| { + matches!( + block, + ContentBlock::ToolCall(_) | ContentBlock::ToolResult(_) + ) + }) + }) +} + +/// Idempotent secret and PII scrub applied before text reaches a verifier. +pub(super) fn redact(text: &str) -> String { + let mut out = text.to_string(); + for pattern in REDACT_PATTERNS.iter() { + out = pattern.replace_all(&out, "[REDACTED]").into_owned(); + } + out +} + +pub(super) fn char_len(text: &str) -> usize { + text.chars().count() +} + +/// Head-and-tail clip to `budget` characters, keeping `head_frac` as head. +pub(super) fn clip_mid(text: &str, budget: usize, head_frac: f64) -> String { + let chars: Vec = text.chars().collect(); + if chars.len() <= budget { + return text.to_string(); + } + let head = ((budget as f64 * head_frac) as usize).max(1); + let tail = budget.saturating_sub(head); + let omitted = chars.len() - budget; + let head_text: String = chars[..head].iter().collect(); + let tail_text: String = chars[chars.len() - tail..].iter().collect(); + format!("{head_text}\n...[{omitted} chars omitted]...\n{tail_text}") +} + +pub(super) fn observed_hardening(attempt: &str) -> bool { + CODE_ACTIVITY + .as_ref() + .is_some_and(|pattern| pattern.is_match(attempt)) +} + +pub(super) fn observed_tool_activity(attempt: &str) -> bool { + TOOL_ACTIVITY + .as_ref() + .is_some_and(|pattern| pattern.is_match(attempt)) +} + +/// The router's own summary of every tool result in the conversation. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub(super) struct ToolRecord { + pub(super) errors: i32, + pub(super) results: i32, + pub(super) tail_clean: bool, + /// Consecutive clean results ending the conversation. + pub(super) clean_tail: i32, +} + +impl ToolRecord { + pub(super) fn from_request(request: &Request) -> Self { + let mut record = Self::default(); + let results = request + .llm_request + .messages + .iter() + .flat_map(|message| &message.content) + .filter_map(|block| match block { + ContentBlock::ToolResult(result) => Some(result), + _ => None, + }); + for result in results { + let text = result + .content + .iter() + .filter_map(|block| match block { + ContentBlock::Text { text } | ContentBlock::Refusal { text } => Some(text), + _ => None, + }) + .map(String::as_str) + .collect::>() + .join("\n"); + let body = serde_json::from_str::(&text).ok(); + let object = body.as_ref().and_then(Value::as_object); + // The executor's own `exit_code: 0` overrules every inferred failure: + // harnesses send `"error": null` on success, and negative checks print + // error-shaped text. + let exit_ok = object + .and_then(|object| object.get("exit_code")) + .and_then(Value::as_i64) + == Some(0); + let failed = (result.is_error == Some(true) + || object.is_some_and(|object| object.contains_key("error")) + || classify_text(&text).0 > 0.0) + && !exit_ok; + record.results = record.results.saturating_add(1); + record.errors = record.errors.saturating_add(i32::from(failed)); + record.tail_clean = !failed; + record.clean_tail = if failed { + 0 + } else { + record.clean_tail.saturating_add(1) + }; + } + record + } +}