From d11fa8301c29fd29e4491987e5b9c8fb9e28a97b Mon Sep 17 00:00:00 2001 From: baiqing Date: Wed, 29 Jul 2026 22:39:06 +0800 Subject: [PATCH 001/173] feat: add reviewable talking-head cleanup --- crates/opentake-agent/src/chat/loop.rs | 18 +- crates/opentake-agent/src/mcp/dispatch.rs | 383 ++++++++++++++++-- crates/opentake-agent/src/mcp/server.rs | 7 +- crates/opentake-agent/src/tools/args.rs | 14 + .../opentake-agent/src/tools/descriptions.rs | 12 + crates/opentake-agent/src/tools/names.rs | 32 +- .../tests/advertised_tool_acceptance.rs | 3 +- crates/opentake-ops/src/ops/clear_region.rs | 18 +- crates/opentake-ops/src/ops/ripple.rs | 28 ++ docs/architecture/BUGS.md | 4 +- docs/architecture/CAPCUT-GAP.md | 5 +- docs/architecture/FULL_PROJECT_SCAN_REPORT.md | 2 +- docs/architecture/HANDOFF-2026-07.md | 2 +- docs/architecture/ROADMAP.md | 2 +- .../EDITING-AUTOMATION-DOS.md | 6 +- .../EDITING-AUTOMATION/acceptance-tests.md | 2 +- .../agent-editing-suggestions.md | 8 +- .../workflow-plugin-recipes.md | 2 +- ...gent-settings-generation-implementation.md | 4 +- .../talking-head-cleanup-2026-07-29.md | 58 +++ docs/modules/opentake-agent/OVERVIEW.md | 18 +- docs/modules/opentake-agent/SPEC.md | 4 +- docs/modules/opentake-agent/dispatch-tools.md | 15 +- docs/modules/opentake-agent/mcp-server.md | 2 +- docs/specs/agent/10-implementation.md | 2 +- docs/specs/agent/2-tools.md | 2 +- 26 files changed, 555 insertions(+), 98 deletions(-) create mode 100644 docs/audit/2026-07-14/runtime-artifacts/automated/talking-head-cleanup-2026-07-29.md diff --git a/crates/opentake-agent/src/chat/loop.rs b/crates/opentake-agent/src/chat/loop.rs index 8bb3665a..59f824e2 100644 --- a/crates/opentake-agent/src/chat/loop.rs +++ b/crates/opentake-agent/src/chat/loop.rs @@ -29,7 +29,6 @@ use crate::plugin::registry::PluginRegistry; use crate::prompt::assemble::assemble_system_prompt; use crate::signal::engine::build_signal; use crate::tools::descriptions::{description, input_schema}; -use crate::tools::names::ToolName; use crate::tools::panic_boundary::with_redacted_dispatch_panic; use crate::tools::result::ToolResult; @@ -251,7 +250,7 @@ impl ChatLoop { } /// The tool catalog in the OpenAI function-calling shape. Built fresh per - /// turn (cheap; currently 38 live tools) so the model always sees the + /// turn (cheap; currently 39 base live tools) so the model always sees the /// current fail-closed catalog. /// /// When the dispatcher lacks a media bridge, hide the bridge-dependent @@ -260,13 +259,6 @@ impl ChatLoop { self.dispatcher .advertised_tools() .into_iter() - .filter(|tool| { - self.dispatcher.has_media_bridge() - || !matches!( - tool, - ToolName::InspectMedia | ToolName::InspectTimeline | ToolName::ImportMedia - ) - }) .map(|tool| ToolSchema { name: tool.as_str().to_string(), description: description(tool).to_string(), @@ -285,7 +277,10 @@ impl ChatLoop { if let Ok(json) = serde_json::to_value(&signal) { s.push_str("\n\n# Current timeline context signal\n"); s.push_str(&serde_json::to_string_pretty(&json).unwrap_or_default()); - s.push_str("\n\nUse this signal to pick the right tool without re-reading the timeline first. For example, if the user asks to tighten silences on a talking-head timeline, call `tighten_silences` then `ripple_delete_ranges` with the returned ranges."); + s.push_str("\n\nUse this signal to pick the right tool without re-reading the timeline first. For example, if the user asks to tighten silences on a talking-head timeline, call `tighten_silences` then `ripple_delete_ranges` with the accepted returned ranges."); + if self.dispatcher.has_media_bridge() { + s.push_str(" If the user asks to remove filler words, call `remove_filler_words`, let them review the word-aligned cuts, then apply only the accepted ranges with `ripple_delete_ranges`."); + } } s } @@ -607,6 +602,9 @@ mod tests { let loop_ = build_loop(talking_head_timeline(), Arc::new(MemoryKeyStore::new())); let tools = loop_.tool_catalog(); assert!(tools.iter().any(|t| t.name == "tighten_silences")); + assert!(!tools.iter().any(|t| t.name == "remove_filler_words")); + assert!(!tools.iter().any(|t| t.name == "get_transcript")); + assert!(!tools.iter().any(|t| t.name == "search_media")); assert!(!tools.iter().any(|t| t.name == "inspect_media")); assert!(!tools.iter().any(|t| t.name == "inspect_timeline")); assert!(!tools.iter().any(|t| t.name == "import_media")); diff --git a/crates/opentake-agent/src/mcp/dispatch.rs b/crates/opentake-agent/src/mcp/dispatch.rs index 8b094886..3e0582a9 100644 --- a/crates/opentake-agent/src/mcp/dispatch.rs +++ b/crates/opentake-agent/src/mcp/dispatch.rs @@ -17,7 +17,7 @@ //! names for compatibility but stay out of discovery until their backends are //! production-ready. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::sync::{Arc, Mutex, RwLock}; use opentake_domain::{AnimPair, Crop, Interpolation, Keyframe, KeyframeTrack}; @@ -64,16 +64,6 @@ const INSPECT_MEDIA_MAX_FRAMES: usize = 12; const INSPECT_MEDIA_MAX_SEGMENTS: usize = 400; const INSPECT_MEDIA_MAX_WORDS: usize = 10_000; -fn is_generation_tool(tool: ToolName) -> bool { - matches!( - tool, - ToolName::GenerateVideo - | ToolName::GenerateImage - | ToolName::GenerateAudio - | ToolName::UpscaleMedia - ) -} - /// The in-process tool dispatcher. Holds the [`CoreHandle`] boundary, the plugin /// registry (read-locked for the active plugin), and a per-dispatcher agent-undo /// stack so `undo` only reverts edits this session made. @@ -142,6 +132,9 @@ impl Dispatcher { pub fn advertised_tools(&self) -> Vec { let mut tools = ToolName::ALL.to_vec(); + if !self.has_media_bridge() { + tools.retain(|tool| !tool.requires_media_bridge()); + } if self.can_generate() { tools.extend(ToolName::GENERATION); } @@ -184,9 +177,7 @@ impl Dispatcher { error.message, ); } - if !(ToolName::ALL.contains(&tool) - || is_generation_tool(tool) && self.generation_bridge.is_some()) - { + if !self.advertised_tools().contains(&tool) { return ToolResult::public_error( PublicErrorKind::UnknownTool, format!("Tool is not advertised: {}", tool.as_str()), @@ -303,6 +294,7 @@ impl Dispatcher { ToolName::AutoCutToBeats => self.auto_cut_to_beats(args, before), ToolName::SmartReframe => self.smart_reframe(args), ToolName::TightenSilences => self.tighten_silences(args, before), + ToolName::RemoveFillerWords => self.remove_filler_words(args, before, manifest), // --- Render + import + transcript + search (wired to the injected MediaBridge) --- ToolName::InspectTimeline => self.inspect_timeline(args, before), @@ -1481,6 +1473,205 @@ impl Dispatcher { Ok(ToolResult::ok(round_floats_3dp(payload).to_string())) } + fn remove_filler_words( + &self, + args: &Value, + before: &Timeline, + manifest: &MediaManifest, + ) -> Result { + let a: RemoveFillerWordsArgs = decode_tool_args(args, "")?; + if a.clip_ids.is_some() && a.track_index.is_some() { + return Err(ToolError::new( + "remove_filler_words: pass clipIds or trackIndex, not both", + )); + } + if let Some(ids) = a.clip_ids.as_ref() { + if ids.is_empty() { + return Err(ToolError::new("remove_filler_words: clipIds is empty")); + } + for id in ids { + if find_clip(before, id).is_none() { + return Err(ToolError::new(format!( + "remove_filler_words: clip not found: {id}" + ))); + } + } + } + if let Some(track_index) = a.track_index { + if before.tracks.get(track_index).is_none() { + return Err(ToolError::new(format!( + "remove_filler_words: track not found: {track_index}" + ))); + } + } + + let lexicon = a.filler_words.unwrap_or_else(|| { + ["um", "uh", "er", "erm", "ah", "like", "you know"] + .into_iter() + .map(str::to_string) + .collect() + }); + let mut phrases = lexicon + .into_iter() + .filter_map(|phrase| { + let tokens = phrase + .split_whitespace() + .map(normalize_spoken_token) + .filter(|token| !token.is_empty()) + .collect::>(); + (!tokens.is_empty()).then_some(tokens) + }) + .collect::>(); + phrases.sort(); + phrases.dedup(); + phrases.sort_by_key(|tokens| std::cmp::Reverse(tokens.len())); + if phrases.is_empty() { + return Err(ToolError::new( + "remove_filler_words: fillerWords has no usable phrases", + )); + } + + let transcript = self.get_transcript(&serde_json::json!({}), before, manifest)?; + if transcript.is_error { + return Ok(transcript); + } + let transcript_json: Value = serde_json::from_str(&transcript.text_joined()) + .map_err(|_| ToolError::new("remove_filler_words: transcript response is invalid"))?; + let clips = transcript_json["clips"] + .as_array() + .ok_or_else(|| ToolError::new("remove_filler_words: transcript clips are missing"))?; + let requested_ids = a + .clip_ids + .as_ref() + .map(|ids| ids.iter().map(String::as_str).collect::>()) + .or_else(|| { + a.track_index.map(|track_index| { + before.tracks[track_index] + .clips + .iter() + .map(|clip| clip.id.as_str()) + .collect::>() + }) + }); + let selected_ids = requested_ids.map(|requested| { + let mut expanded = requested + .iter() + .map(|id| (*id).to_string()) + .collect::>(); + let link_groups = requested + .iter() + .filter_map(|id| find_clip(before, id)) + .filter_map(|clip| clip.link_group_id.as_deref()) + .collect::>(); + for clip in before.tracks.iter().flat_map(|track| &track.clips) { + if clip + .link_group_id + .as_deref() + .is_some_and(|group| link_groups.contains(group)) + { + expanded.insert(clip.id.clone()); + } + } + expanded + }); + let padding = a.padding_frames.unwrap_or(1).max(0) as i64; + let mut cuts = Vec::new(); + let mut ranges_by_track: BTreeMap> = BTreeMap::new(); + + for clip in clips { + let Some(clip_id) = clip["clipId"].as_str() else { + continue; + }; + let Some(track_index) = clip["trackIndex"].as_u64() else { + continue; + }; + if selected_ids + .as_ref() + .is_some_and(|ids| !ids.contains(clip_id)) + { + continue; + } + let clip_start = clip["startFrame"].as_i64().unwrap_or(0); + let clip_end = clip["endFrame"].as_i64().unwrap_or(clip_start); + let Some(rows) = clip["words"].as_array() else { + continue; + }; + let normalized = rows + .iter() + .map(|row| normalize_spoken_token(row[0].as_str().unwrap_or_default())) + .collect::>(); + let mut word_index = 0; + while word_index < rows.len() { + let Some(phrase) = phrases.iter().find(|phrase| { + word_index + phrase.len() <= normalized.len() + && normalized[word_index..word_index + phrase.len()] == phrase[..] + }) else { + word_index += 1; + continue; + }; + let last_index = word_index + phrase.len() - 1; + let start = (rows[word_index][1].as_i64().unwrap_or(clip_start) + padding) + .clamp(clip_start, clip_end); + let end = (rows[last_index][2].as_i64().unwrap_or(start) - padding) + .clamp(clip_start, clip_end); + if end > start { + let text = rows[word_index..=last_index] + .iter() + .filter_map(|row| row[0].as_str()) + .collect::>() + .join(" "); + let cut_id = format!("filler-{clip_id}-{word_index}"); + cuts.push(serde_json::json!({ + "id": cut_id, + "clipId": clip_id, + "trackIndex": track_index, + "text": text, + "range": [start, end], + "accepted": true, + })); + ranges_by_track + .entry(track_index) + .or_default() + .push([start, end]); + } + word_index += phrase.len(); + } + } + + for ranges in ranges_by_track.values_mut() { + ranges.sort_unstable(); + ranges.dedup(); + } + cuts.sort_by_key(|cut| { + ( + cut["trackIndex"].as_u64().unwrap_or(0), + cut["range"][0].as_i64().unwrap_or(0), + ) + }); + let commands = ranges_by_track + .into_iter() + .map(|(track_index, ranges)| { + serde_json::json!({ + "tool": "ripple_delete_ranges", + "args": { + "trackIndex": track_index, + "units": "frames", + "ranges": ranges, + } + }) + }) + .collect::>(); + Ok(ToolResult::ok( + serde_json::json!({ + "applied": false, + "cuts": cuts, + "commands": commands, + "note": "Review cuts and remove rejected ranges before calling each returned ripple_delete_ranges command. Each command applies as one undoable edit.", + }) + .to_string(), + )) + } + fn detect_beat_hints( &self, timeline: &Timeline, @@ -2050,6 +2241,7 @@ fn validate_tool_args(tool: ToolName, args: &Value) -> Result<(), ToolError> { ToolName::AutoCutToBeats => decode!(AutoCutToBeatsArgs), ToolName::SmartReframe => decode!(SmartReframeArgs), ToolName::TightenSilences => decode!(TightenSilencesArgs), + ToolName::RemoveFillerWords => decode!(RemoveFillerWordsArgs), ToolName::GenerateVideo => decode!(GenerateVideoArgs), ToolName::GenerateImage => decode!(GenerateImageArgs), ToolName::GenerateAudio => decode!(GenerateAudioArgs), @@ -2669,6 +2861,14 @@ fn normalized_speed(clip: &opentake_domain::Clip) -> f64 { } } +fn normalize_spoken_token(value: &str) -> String { + value + .chars() + .flat_map(char::to_lowercase) + .filter(|character| character.is_alphanumeric() || *character == '\'') + .collect() +} + fn source_seconds_to_timeline_frame_clamped( clip: &opentake_domain::Clip, source_seconds: f64, @@ -4543,16 +4743,97 @@ mod tests { } #[test] - fn remove_filler_words_stays_disabled_until_transcript_is_wired() { - let d = dispatcher_with(empty_manifest_handle(vec![])); - let r = d.dispatch("remove_filler_words", serde_json::json!({})); - assert!(r.is_error); - assert!( - r.text_joined() - .contains("Unknown tool: remove_filler_words"), - "{}", - r.text_joined() + fn remove_filler_words_returns_reviewable_word_aligned_ranges() { + let (d, _bridge) = transcript_dispatcher(transcript(vec![ + word("Well", 0.0, 0.2), + word("um", 0.2, 0.4), + word("you", 0.5, 0.7), + word("know", 0.7, 0.9), + word("go", 1.0, 1.2), + ])); + assert!(d.advertised_tools().contains(&ToolName::RemoveFillerWords)); + let r = d.dispatch( + "remove_filler_words", + serde_json::json!({ + "clipIds": ["clip-a"], + "fillerWords": ["um", "you know"], + "paddingFrames": 0 + }), + ); + assert!(!r.is_error, "{}", r.text_joined()); + let json = first_json(&r); + assert_eq!(json["applied"], false); + assert_eq!(json["cuts"].as_array().unwrap().len(), 2); + assert_eq!(json["cuts"][0]["text"], "um"); + assert_eq!(json["cuts"][0]["range"], serde_json::json!([6, 12])); + assert_eq!(json["cuts"][1]["text"], "you know"); + assert_eq!(json["cuts"][1]["range"], serde_json::json!([15, 27])); + assert_eq!( + json["commands"][0]["args"]["ranges"], + serde_json::json!([[6, 12], [15, 27]]) + ); + } + + #[test] + fn reviewed_filler_cut_applies_once_and_undo_restores_the_timeline() { + let (d, _bridge) = linked_talking_head_dispatcher(transcript(vec![ + word("Well", 0.0, 0.2), + word("um", 0.2, 0.4), + word("you", 0.5, 0.7), + word("know", 0.7, 0.9), + word("go", 1.0, 1.2), + ])); + let before = d.handle.timeline(); + let preview = d.dispatch( + "remove_filler_words", + serde_json::json!({ + "clipIds": ["clip-v"], + "fillerWords": ["um", "you know"], + "paddingFrames": 0 + }), + ); + let json = first_json(&preview); + let apply = d.dispatch( + "ripple_delete_ranges", + serde_json::json!({ + "trackIndex": 1, + "units": "frames", + "ranges": [json["cuts"][0]["range"].clone()] + }), ); + assert!(!apply.is_error, "{}", apply.text_joined()); + let after = d.handle.timeline(); + assert_ne!(after, before); + assert_eq!(after.tracks.len(), 2); + let video_ranges = after.tracks[0] + .clips + .iter() + .map(|clip| (clip.start_frame, clip.end_frame())) + .collect::>(); + let audio_ranges = after.tracks[1] + .clips + .iter() + .map(|clip| (clip.start_frame, clip.end_frame())) + .collect::>(); + assert_eq!(video_ranges, audio_ranges, "linked A/V ranges drifted"); + assert_eq!(video_ranges.last().map(|range| range.1), Some(894)); + + let post_cut = d.dispatch("get_transcript", serde_json::json!({})); + assert!(!post_cut.is_error, "{}", post_cut.text_joined()); + let post_cut_json = first_json(&post_cut); + let spoken = post_cut_json["clips"] + .as_array() + .unwrap() + .iter() + .flat_map(|clip| clip["words"].as_array().unwrap()) + .filter_map(|word| word[0].as_str()) + .collect::>(); + assert!(!spoken.contains(&"um"), "{spoken:?}"); + assert!(spoken.windows(2).any(|words| words == ["you", "know"])); + + let undo = d.dispatch("undo", serde_json::json!({})); + assert!(!undo.is_error, "{}", undo.text_joined()); + assert_eq!(d.handle.timeline(), before); } #[test] @@ -5140,14 +5421,12 @@ mod tests { } #[test] - fn inspect_timeline_without_bridge_reports_unavailable() { - // The seeded TestHandle timeline is empty, so first assert the empty guard, - // then a non-empty timeline with no bridge reports "not available". + fn inspect_timeline_without_bridge_is_not_advertised() { let d = dispatcher_with(seeded_handle()); let r = d.dispatch("inspect_timeline", serde_json::json!({ "startFrame": 0 })); assert!(r.is_error); assert!( - r.text_joined().contains("not available"), + r.text_joined().contains("not advertised"), "{}", r.text_joined() ); @@ -5272,7 +5551,7 @@ mod tests { } #[test] - fn import_media_without_bridge_reports_unavailable() { + fn import_media_without_bridge_is_not_advertised() { let d = dispatcher_with(seeded_handle()); let r = d.dispatch( "import_media", @@ -5280,7 +5559,7 @@ mod tests { ); assert!(r.is_error); assert!( - r.text_joined().contains("not available"), + r.text_joined().contains("not advertised"), "{}", r.text_joined() ); @@ -5617,7 +5896,7 @@ mod tests { } #[test] - fn search_media_without_bridge_reports_unavailable() { + fn search_media_without_bridge_is_not_advertised() { let mut m = MediaManifest::new(); m.entries.push(entry("v", "Clip")); let handle = Arc::new(StateHandle::new(Timeline::new(), m)); @@ -5625,7 +5904,7 @@ mod tests { let r = d.dispatch("search_media", serde_json::json!({ "query": "x" })); assert!(r.is_error); assert!( - r.text_joined().contains("not available in this build"), + r.text_joined().contains("not advertised"), "{}", r.text_joined() ); @@ -5674,6 +5953,39 @@ mod tests { (d, bridge) } + /// A fixed 30-second talking-head fixture with linked video/audio clips. + /// Only the audio partner is transcribed, matching production caption target + /// selection, while a reviewed ripple cut must keep both tracks frame-exact. + fn linked_talking_head_dispatcher(t: TranscriptionResult) -> (Dispatcher, Arc) { + let mut tl = Timeline::new(); + tl.fps = 30; + + let mut video_track = Track::new("track-v", ClipType::Video); + let mut video = Clip::new("clip-v", "vid", 0, 30 * 30); + video.link_group_id = Some("talking-head-av".into()); + video_track.clips.push(video); + + let mut audio_track = Track::new("track-a", ClipType::Audio); + let mut audio = Clip::new("clip-a", "aud", 0, 30 * 30); + audio.media_type = ClipType::Audio; + audio.link_group_id = Some("talking-head-av".into()); + audio_track.clips.push(audio); + + tl.tracks.push(video_track); + tl.tracks.push(audio_track); + let mut manifest = MediaManifest::new(); + manifest.entries.push(entry("vid", "Camera")); + manifest.entries.push(audio_entry("aud", "Voice")); + let handle = Arc::new(StateHandle::new(tl, manifest)); + let bridge = Arc::new(FakeBridge::default().with_transcript("aud", t)); + let dispatcher = Dispatcher::with_bridge( + handle, + Arc::new(RwLock::new(PluginRegistry::new())), + Some(bridge.clone() as Arc), + ); + (dispatcher, bridge) + } + #[test] fn get_transcript_maps_words_to_project_frames() { let (d, _b) = transcript_dispatcher(transcript(vec![ @@ -5700,8 +6012,7 @@ mod tests { } #[test] - fn get_transcript_without_bridge_reports_unavailable() { - // Same audio timeline but no bridge wired → honest "not available". + fn get_transcript_without_bridge_is_not_advertised() { let mut tl = Timeline::new(); tl.fps = 30; let mut track = opentake_domain::Track::new("track-a", ClipType::Audio); @@ -5715,7 +6026,7 @@ mod tests { let r = d.dispatch("get_transcript", serde_json::json!({})); assert!(r.is_error); assert!( - r.text_joined().contains("not available"), + r.text_joined().contains("not advertised"), "{}", r.text_joined() ); @@ -6061,7 +6372,7 @@ mod tests { } #[test] - fn add_captions_without_bridge_reports_unavailable() { + fn add_captions_without_bridge_is_not_advertised() { let mut tl = Timeline::new(); tl.fps = 30; tl.width = 1920; @@ -6077,7 +6388,7 @@ mod tests { let r = d.dispatch("add_captions", serde_json::json!({})); assert!(r.is_error); assert!( - r.text_joined().contains("not available"), + r.text_joined().contains("not advertised"), "{}", r.text_joined() ); diff --git a/crates/opentake-agent/src/mcp/server.rs b/crates/opentake-agent/src/mcp/server.rs index 3afc2b06..b3f5226f 100644 --- a/crates/opentake-agent/src/mcp/server.rs +++ b/crates/opentake-agent/src/mcp/server.rs @@ -612,12 +612,17 @@ mod tests { #[test] fn lists_every_advertised_tool() { let server = server(); - assert_eq!(server.tools().len(), ToolName::ALL.len()); + let expected = ToolName::ALL + .iter() + .filter(|tool| !tool.requires_media_bridge()) + .count(); + assert_eq!(server.tools().len(), expected); // Names round-trip to the wire names. let names: Vec = server.tools().iter().map(|t| t.name.to_string()).collect(); assert!(names.contains(&"add_clips".to_string())); assert!(names.contains(&"detect_beats".to_string())); assert!(names.contains(&"activate_workflow".to_string())); + assert!(!names.contains(&"remove_filler_words".to_string())); } #[test] diff --git a/crates/opentake-agent/src/tools/args.rs b/crates/opentake-agent/src/tools/args.rs index cc67a9f2..687192d5 100644 --- a/crates/opentake-agent/src/tools/args.rs +++ b/crates/opentake-agent/src/tools/args.rs @@ -678,6 +678,20 @@ impl ToolArgs for TightenSilencesArgs { ]; } +// --- remove_filler_words --- +#[derive(Debug, Clone, Default, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct RemoveFillerWordsArgs { + pub clip_ids: Option>, + pub track_index: Option, + pub filler_words: Option>, + pub padding_frames: Option, +} +impl ToolArgs for RemoveFillerWordsArgs { + const ALLOWED_KEYS: &'static [&'static str] = + &["clipIds", "trackIndex", "fillerWords", "paddingFrames"]; +} + // --- generate_video --- #[derive(Debug, Clone, Default, Deserialize, PartialEq)] #[serde(rename_all = "camelCase")] diff --git a/crates/opentake-agent/src/tools/descriptions.rs b/crates/opentake-agent/src/tools/descriptions.rs index 74042e7f..44b32623 100644 --- a/crates/opentake-agent/src/tools/descriptions.rs +++ b/crates/opentake-agent/src/tools/descriptions.rs @@ -61,6 +61,8 @@ pub fn description(tool: ToolName) -> &'static str { ToolName::TightenSilences => "Plans silence tightening by finding low-energy PCM spans and converting them into ripple_delete_ranges candidate commands. Returns a preview only; it does not mutate the timeline.", + ToolName::RemoveFillerWords => "Transcribes the current spoken timeline and returns reviewable filler-word cuts aligned to word timestamps. Supports an exact configurable lexicon including multi-word phrases. It does not mutate the timeline: remove rejected cuts, then call each returned ripple_delete_ranges command to apply the accepted ranges as one undoable edit per track.", + ToolName::GenerateVideo => "Starts an async AI video generation. Returns a placeholder asset ID immediately; generation runs in the background and the asset becomes usable in add_clips once ready. Costs real money and is not undoable.", ToolName::GenerateImage => "Starts an async AI image generation. Returns a placeholder asset ID immediately; generation runs in the background. Costs real money and is not undoable.", @@ -400,6 +402,16 @@ pub fn input_schema(tool: ToolName) -> Value { &[], ), + ToolName::RemoveFillerWords => object( + json!({ + "clipIds": {"type": "array", "items": {"type": "string"}, "description": "Optional spoken clip ids to transcribe and analyze."}, + "trackIndex": {"type": "integer", "description": "Optional spoken track index to analyze. Mutually exclusive with clipIds."}, + "fillerWords": {"type": "array", "items": {"type": "string"}, "description": "Optional exact filler lexicon. Multi-word phrases such as 'you know' are supported."}, + "paddingFrames": {"type": "integer", "minimum": 0, "description": "Optional context frames to preserve before and after each matched filler phrase."} + }), + &[], + ), + ToolName::GenerateVideo => object( json!({ "costAuthorized": {"type": "boolean", "description": "Required true. Set only after the user explicitly approves the paid generation request in this conversation."}, diff --git a/crates/opentake-agent/src/tools/names.rs b/crates/opentake-agent/src/tools/names.rs index 52c48d39..cc674e9f 100644 --- a/crates/opentake-agent/src/tools/names.rs +++ b/crates/opentake-agent/src/tools/names.rs @@ -34,6 +34,7 @@ pub enum ToolName { AutoCutToBeats, SmartReframe, TightenSilences, + RemoveFillerWords, // --- Media generation / import (5) --- GenerateVideo, GenerateImage, @@ -63,6 +64,22 @@ pub enum ToolName { } impl ToolName { + /// Whether discovery of this tool requires a live host media bridge. + /// Keeping this predicate next to the catalog prevents MCP and in-app Chat + /// from drifting into different fail-closed capability sets. + pub const fn requires_media_bridge(self) -> bool { + matches!( + self, + ToolName::InspectMedia + | ToolName::GetTranscript + | ToolName::InspectTimeline + | ToolName::SearchMedia + | ToolName::AddCaptions + | ToolName::RemoveFillerWords + | ToolName::ImportMedia + ) + } + /// The wire name (matches upstream / spec exactly). pub fn as_str(self) -> &'static str { match self { @@ -89,6 +106,7 @@ impl ToolName { ToolName::AutoCutToBeats => "auto_cut_to_beats", ToolName::SmartReframe => "smart_reframe", ToolName::TightenSilences => "tighten_silences", + ToolName::RemoveFillerWords => "remove_filler_words", ToolName::GenerateVideo => "generate_video", ToolName::GenerateImage => "generate_image", ToolName::GenerateAudio => "generate_audio", @@ -116,7 +134,7 @@ impl ToolName { /// Tools advertised to MCP and in-app Chat in registration order. Provider- /// backed generation and Motion Canvas tools remain known wire names, but /// stay out of discovery until their production backends are connected. - pub const ALL: [ToolName; 38] = [ + pub const ALL: [ToolName; 39] = [ ToolName::GetTimeline, ToolName::GetMedia, ToolName::InspectMedia, @@ -140,6 +158,7 @@ impl ToolName { ToolName::AutoCutToBeats, ToolName::SmartReframe, ToolName::TightenSilences, + ToolName::RemoveFillerWords, ToolName::ImportMedia, ToolName::ListFolders, ToolName::CreateFolder, @@ -170,7 +189,7 @@ impl ToolName { /// hidden from discovery until a real backend exists. Keeping this set lets /// strict argument validation and compatibility tests cover future tools /// without advertising placeholder behavior to models. - pub const KNOWN: [ToolName; 44] = [ + pub const KNOWN: [ToolName; 45] = [ ToolName::GetTimeline, ToolName::GetMedia, ToolName::InspectMedia, @@ -194,6 +213,7 @@ impl ToolName { ToolName::AutoCutToBeats, ToolName::SmartReframe, ToolName::TightenSilences, + ToolName::RemoveFillerWords, ToolName::GenerateVideo, ToolName::GenerateImage, ToolName::GenerateAudio, @@ -274,9 +294,9 @@ mod tests { } #[test] - fn advertised_set_is_38_and_known_set_is_44() { - assert_eq!(ToolName::ALL.len(), 38); - assert_eq!(ToolName::KNOWN.len(), 44); + fn advertised_set_is_39_and_known_set_is_45() { + assert_eq!(ToolName::ALL.len(), 39); + assert_eq!(ToolName::KNOWN.len(), 45); assert!(ToolName::ALL .iter() .all(|tool| ToolName::KNOWN.contains(tool))); @@ -288,11 +308,13 @@ mod tests { assert_eq!(ToolName::AutoCutToBeats.as_str(), "auto_cut_to_beats"); assert_eq!(ToolName::SmartReframe.as_str(), "smart_reframe"); assert_eq!(ToolName::TightenSilences.as_str(), "tighten_silences"); + assert_eq!(ToolName::RemoveFillerWords.as_str(), "remove_filler_words"); for t in [ ToolName::DetectBeats, ToolName::AutoCutToBeats, ToolName::SmartReframe, ToolName::TightenSilences, + ToolName::RemoveFillerWords, ] { assert_eq!(ToolName::from_str(t.as_str()), Ok(t)); assert!(!ToolName::UPSTREAM.contains(&t)); diff --git a/crates/opentake-agent/tests/advertised_tool_acceptance.rs b/crates/opentake-agent/tests/advertised_tool_acceptance.rs index 19526565..d4513c9e 100644 --- a/crates/opentake-agent/tests/advertised_tool_acceptance.rs +++ b/crates/opentake-agent/tests/advertised_tool_acceptance.rs @@ -68,9 +68,10 @@ fn every_advertised_tool_is_live_or_absent() { serde_json::json!({"clipId": "clip", "code": "export default {}"}), ), ]; + let advertised = dispatcher.advertised_tools(); for (tool, args) in cases { - if !ToolName::ALL.contains(&tool) { + if !advertised.contains(&tool) { let result = dispatcher.dispatch(tool.as_str(), args); assert!( result.text_joined().contains("not advertised"), diff --git a/crates/opentake-ops/src/ops/clear_region.rs b/crates/opentake-ops/src/ops/clear_region.rs index 77590901..b3de6b29 100644 --- a/crates/opentake-ops/src/ops/clear_region.rs +++ b/crates/opentake-ops/src/ops/clear_region.rs @@ -71,13 +71,17 @@ pub fn clear_region( if find(timeline, &clip_id).is_some() { // Split at `start`; the right half is what now covers the region. split_clip(timeline, &clip_id, start, ids); - // Locate the freshly created right half (starts at `start`, not the original id). - let right = timeline - .tracks - .iter() - .flat_map(|t| &t.clips) - .find(|c| c.start_frame == start && c.id != clip_id) - .map(|c| (c.id.clone(), c.end_frame())); + // Locate the freshly created right half on the original + // clip's track. Linked splits mint a right half on every + // partner track; a global search can select the wrong + // partner and leave a duplicate middle fragment behind. + let right = find(timeline, &clip_id).and_then(|(ti, _)| { + timeline.tracks[ti] + .clips + .iter() + .find(|c| c.start_frame == start && c.id != clip_id) + .map(|c| (c.id.clone(), c.end_frame())) + }); if let Some((right_id, right_end)) = right { if right_end > end { // Right half overruns the region — split again at `end`, diff --git a/crates/opentake-ops/src/ops/ripple.rs b/crates/opentake-ops/src/ops/ripple.rs index 8e6b7629..7f9dadf7 100644 --- a/crates/opentake-ops/src/ops/ripple.rs +++ b/crates/opentake-ops/src/ops/ripple.rs @@ -488,6 +488,34 @@ mod tests { } } + #[test] + fn ripple_delete_ranges_keeps_linked_av_frame_exact() { + let mut tl = Timeline::new(); + let mut video = Track::new("video", ClipType::Video); + let mut video_clip = clip("video-clip", 0, 900); + video_clip.link_group_id = Some("av".into()); + video.clips.push(video_clip); + let mut audio = Track::new("audio", ClipType::Audio); + let mut audio_clip = clip("audio-clip", 0, 900); + audio_clip.media_type = ClipType::Audio; + audio_clip.link_group_id = Some("av".into()); + audio.clips.push(audio_clip); + tl.tracks.extend([video, audio]); + + let g = SeqIdGen::new("r-"); + let out = ripple_delete_ranges_on_track(&mut tl, 1, &[FrameRange::new(6, 12)], &label, &g); + assert!(matches!(out, RippleOutcome::Ok(_))); + let spans = |track: &Track| { + track + .clips + .iter() + .map(|clip| (clip.start_frame, clip.end_frame())) + .collect::>() + }; + assert_eq!(spans(&tl.tracks[0]), vec![(0, 6), (6, 894)]); + assert_eq!(spans(&tl.tracks[1]), vec![(0, 6), (6, 894)]); + } + #[test] fn ripple_delete_ranges_refuses_on_locked_follower_collision() { let mut tl = Timeline::new(); diff --git a/docs/architecture/BUGS.md b/docs/architecture/BUGS.md index 4dda349b..7009be82 100644 --- a/docs/architecture/BUGS.md +++ b/docs/architecture/BUGS.md @@ -59,12 +59,12 @@ | **描述** | GPU 合成的 infrastructure 已就绪（`composite_frame` Tauri 命令、`useTimelineFrame` hook 均存在），但 `Preview.tsx` 仍然使用 DOM `