From c6064b6e6979c28bbc1c65f9e8f4215e1ca50e39 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Mon, 21 Sep 2026 21:02:47 +0000 Subject: [PATCH 1/3] fix(sed): GNU-align the stream editor and add the missing commands The builtin was GNU-aligned only for the subset it implemented; outside it a script could not tell it had diverged. A differential audit of 128 common invocations found 46 mismatches, 21 of them silent (exit 0, empty stderr). Rewrites the builtin as `builtins/sed/` (pattern translation, script parser, execution engine) and fixes every finding in the audit: - Replacement text is expanded by sed's own scanner instead of being handed to `Regex::replace*`, so a literal `$` stays literal (`s/a/$x/` emitted `b`, now `$xb`) and `\U \L \u \l \E` work. - One BRE/ERE translation pass now serves both `s///` patterns and address regexes. `+ ? | ^ $` and a leading `*` are positional literals in BRE, and bracket expressions keep POSIX semantics (the delimiter is ordinary inside `[...]`). - Range addresses carry explicit state: `/re/,N` keeps its regex, a closed range stays closed, `addr,+N` / `addr,~N` / `$,addr` parse, `0,/re/` is the only form testing its end regex on the start line, and `c` on a range emits its text once. - Operands are one continuous stream unless `-s`/`-i` asks otherwise, so line numbers, `$` and `q` span all files. - A missing final newline is preserved rather than invented, including through `-i`, which now rewrites via the atomic helper factored out of `yq` (mode preserved, original intact on failure, THREAT[TM-FS-016]) and refuses non-UTF-8 input instead of corrupting it. - `s///Ng` means "the Nth match and later". - Adds `y = l n N P D T z F r R w W`, `#` comments and `#n`, `q`/`Q` exit status, `//` regex reuse, and `\cREc` addresses. `e` is refused explicitly. - The script parser is char-based end to end, so a multi-byte `s` delimiter no longer panics (TM-UNI-002, now FIXED). `fuzz_exec` additionally rejects stderr reporting a caught panic, which previously let these pass. - Option parsing is a real getopt: clustering (`-ne`), `-f`, `-s`, `-z`, `-l N`, attached `-i.bak`, `--` terminating options, long spellings, and an empty script as a passthrough. Coverage: 43 builtin unit tests, 31 new spec cases (the sed spec suite is now verified against host GNU sed by `sed_comparison_tests`, the same strict gate the bash suite uses), a sed fuzz scaffold, and `sed::temp_*` failpoints. Closes #2427 --- crates/bashkit/docs/compatibility.md | 7 +- crates/bashkit/docs/threat-model.md | 2 +- crates/bashkit/fuzz/fuzz_targets/sed_fuzz.rs | 16 + crates/bashkit/src/builtins/atomic_write.rs | 92 + crates/bashkit/src/builtins/limits.rs | 4 + crates/bashkit/src/builtins/mod.rs | 1 + crates/bashkit/src/builtins/sed.rs | 1475 ----------------- crates/bashkit/src/builtins/sed/exec.rs | 773 +++++++++ crates/bashkit/src/builtins/sed/mod.rs | 487 ++++++ crates/bashkit/src/builtins/sed/pattern.rs | 369 +++++ crates/bashkit/src/builtins/sed/script.rs | 869 ++++++++++ crates/bashkit/src/builtins/sed/tests.rs | 648 ++++++++ crates/bashkit/src/builtins/yq.rs | 85 +- crates/bashkit/src/testing.rs | 6 + crates/bashkit/tests/integration/main.rs | 1 + .../integration/sed_fuzz_scaffold_tests.rs | 114 ++ .../bashkit/tests/integration/spec_tests.rs | 17 +- .../bashkit/tests/security_failpoint_tests.rs | 81 + .../bashkit/tests/spec_cases/sed/sed.test.sh | 313 ++++ knowledge/operations/limitations.md | 4 + knowledge/security/security-testing.md | 3 + knowledge/security/threat-model.md | 2 +- scripts/update-spec-expected.sh | 6 +- 23 files changed, 3821 insertions(+), 1554 deletions(-) create mode 100644 crates/bashkit/src/builtins/atomic_write.rs delete mode 100644 crates/bashkit/src/builtins/sed.rs create mode 100644 crates/bashkit/src/builtins/sed/exec.rs create mode 100644 crates/bashkit/src/builtins/sed/mod.rs create mode 100644 crates/bashkit/src/builtins/sed/pattern.rs create mode 100644 crates/bashkit/src/builtins/sed/script.rs create mode 100644 crates/bashkit/src/builtins/sed/tests.rs create mode 100644 crates/bashkit/tests/integration/sed_fuzz_scaffold_tests.rs diff --git a/crates/bashkit/docs/compatibility.md b/crates/bashkit/docs/compatibility.md index f0e5f36a2..c9dda5674 100644 --- a/crates/bashkit/docs/compatibility.md +++ b/crates/bashkit/docs/compatibility.md @@ -72,7 +72,7 @@ for sandbox security reasons. See the compliance spec for details. | `times` | - | POSIX display process times | | `grep` | `-i`, `-v`, `-c`, `-n`, `-E`, `-q` | Pattern matching | | `rg` | `-i/-S/-s`, `-n/-N`, `--column`, `-b/--byte-offset`, `--vimgrep`, basic `--json`, `--stats`, `--null`, `-c`, `--count-matches`, `--include-zero`, `-l`, `--files-with-matches`, `--files-without-match`, `--files`, `-F`, `-w`, `-x`, `-a/--text`, `--binary`, `-m`, `-M/--max-columns`, `--max-columns-preview`, `--max-depth`, `-A`, `-B`, `-C`, separator flags, `--heading/--no-heading`, `--sort/--sortr`, `--path-separator`, `-g/--glob`, `--ignore-file`, `.ignore`/`.gitignore`, `--no-ignore*`, `--require-git/--no-require-git`, `-u/--unrestricted`, `--messages/--no-messages`, `-t/--type`, `-T/--type-not`, `--type-add`, `--type-clear`, `--type-list`, `--hidden/--no-hidden`, `-o`, `-q`, `-e`, `-f`, `-E/--encoding` (`auto`, `none`, UTF-8, UTF-16LE/BE), `-r`, `--passthru`, `--trim`, `-H/-I`, `--no-config`, `--line-buffered`, `--block-buffered`, `--mmap/--no-mmap`, `-P/--pcre2`, `--no-pcre2`, `--engine`, `--auto-hybrid-regex`, `--no-auto-hybrid-regex`, `--help`, `--version` | Recursive ripgrep-style search | -| `sed` | `s///[g]`, `d`, `p`, `q`, `a`, `i`, `c`, `h/H/g/G/x`, `-E`, `-n`, `!` | Stream editing | +| `sed` | `s///[gpiImMN w]`, `y`, `d`, `D`, `p`, `P`, `n`, `N`, `q[N]`, `Q[N]`, `=`, `l[N]`, `z`, `F`, `a`, `i`, `c`, `r`, `R`, `w`, `W`, `h/H/g/G/x`, `b`/`t`/`T`/`:label`, `{}`, `#`/`#n`; addresses `N`, `$`, `/re/[IM]`, `\cREc`, `first~step`, `a,b`, `a,+N`, `a,~N`, `0,/re/`, `!`; options `-n -e -f -i[SUF] -E/-r -s -z -l N -u --` and long forms | Stream editing; one stream across operands unless `-s`/`-i` | | `awk` | `'{print}'`, `-F`, `-v`, loops, arrays, increment, ternary | Text processing | | `jq` | `.field`, `.[n]`, pipes, file args, `-r`, `-c`, `-n`, `-s`, `-S`, `-e`, `-j`, `--tab`, `--arg`, `--argjson`, `-V`, combined flags | JSON processing | | `sleep` | `N`, `N.N` | Pause execution (max 60s) | @@ -502,6 +502,11 @@ Identified from eval analysis, all items now implemented: - [x] `sed` `0~2` step addressing, even/odd line processing - [x] `sed` `Q` quiet quit command - [x] `sed` `0,/pattern/` first match addressing +- [x] `sed` GNU rewrite, issue #2427: literal `$` in the replacement, BRE/ERE + translation shared by `s///` and address regexes, stateful range addresses, + single-stream multi-file input, preserved missing final newline, `s///Ng`, + the `y = l n N P D T z F r R w W` commands, `#` comments and `#n`, atomic + mode-preserving `-i`, and full getopt-style option parsing - [x] AWK `$0` modification with field re-splitting ### Not Planned diff --git a/crates/bashkit/docs/threat-model.md b/crates/bashkit/docs/threat-model.md index ebea4d988..57aee5244 100644 --- a/crates/bashkit/docs/threat-model.md +++ b/crates/bashkit/docs/threat-model.md @@ -883,7 +883,7 @@ builtin silently fails. | Threat | Attack Example | Mitigation | Status | |--------|---------------|------------|--------| | Awk byte-boundary panic (TM-UNI-001) | Multi-byte chars in awk input | `catch_unwind` catches panic | PARTIAL | -| Sed byte-boundary panic (TM-UNI-002) | Box-drawing chars in sed pattern | `catch_unwind` catches panic | PARTIAL | +| Sed byte-boundary panic (TM-UNI-002) | Multi-byte `s` delimiter, e.g. `sed 's≠a≠X≠'` | Char-based script parser; no byte slicing of script text | FIXED | | Expr substr panic (TM-UNI-015) | `expr substr "café" 4 1` | `catch_unwind` catches panic | PARTIAL | | Printf precision panic (TM-UNI-016) | `printf "%.1s" "é"` | `catch_unwind` catches panic | PARTIAL | | Cut/tr byte-level parsing (TM-UNI-017) | `tr 'é' 'e'`, multi-byte in char set | `catch_unwind` catches; silent data loss | PARTIAL | diff --git a/crates/bashkit/fuzz/fuzz_targets/sed_fuzz.rs b/crates/bashkit/fuzz/fuzz_targets/sed_fuzz.rs index 78e77e0d5..292ce20dd 100644 --- a/crates/bashkit/fuzz/fuzz_targets/sed_fuzz.rs +++ b/crates/bashkit/fuzz/fuzz_targets/sed_fuzz.rs @@ -90,6 +90,22 @@ fuzz_target!(|data: &[u8]| { expr.replace('\'', "'\\''"), ); bashkit::testing::fuzz_exec(&mut bash, &script3, "sed_fuzz", &[]).await; + + // Test 4: seeds that historically panicked or diverged (issue #2427): + // multi-byte `s` delimiters (TM-UNI-002), truncated scripts, and + // option clustering. + for seed in [ + "s≠a≠X≠", + "s😀a😀X😀g", + "y≠a≠b≠", + "s/a", + "s/[//", + ":a;ba", + "b missing", + ] { + let script = format!("echo 'a' | sed '{seed}'"); + bashkit::testing::fuzz_exec(&mut bash, &script, "sed_fuzz_seed", &[]).await; + } }); } }); diff --git a/crates/bashkit/src/builtins/atomic_write.rs b/crates/bashkit/src/builtins/atomic_write.rs new file mode 100644 index 000000000..591085860 --- /dev/null +++ b/crates/bashkit/src/builtins/atomic_write.rs @@ -0,0 +1,92 @@ +//! Shared atomic in-place file replacement for builtins that rewrite files. +//! +//! THREAT[TM-FS-016]: write a sibling temporary file, copy the original mode +//! onto it, and rename only after the new content is fully written. Any failure +//! leaves the source file untouched and removes the temporary. +//! +//! Extracted from `yq` so `sed -i` gets the same guarantees (issue #2427, +//! finding E): the previous `sed -i` wrote straight through +//! `FileSystem::write_file`, which recreates the entry with mode 0644 and loses +//! the original content if the write fails part-way. + +use std::path::Path; + +use crate::fs::vfs_join; + +/// Failpoint names for one call site. Each builtin supplies its own so the +/// security failpoint suite can target them independently. +#[cfg_attr(not(feature = "failpoints"), allow(dead_code))] +pub(crate) struct AtomicFailpoints { + pub(crate) allocate: &'static str, + pub(crate) chmod: &'static str, + pub(crate) rename: &'static str, +} + +pub(crate) async fn atomic_replace( + fs: &dyn crate::fs::FileSystem, + target: &Path, + content: &[u8], + tool: &str, + failpoints: AtomicFailpoints, +) -> std::result::Result<(), String> { + let parent = target.parent().unwrap_or_else(|| Path::new("/")); + #[cfg(feature = "failpoints")] + if injected_failure(failpoints.allocate, "exhausted") { + return Err("cannot allocate temporary file".to_string()); + } + let mut temp = None; + for _ in 0..16 { + let mut suffix = [0u8; 8]; + getrandom::fill(&mut suffix) + .map_err(|_| "cannot generate temporary filename".to_string())?; + let suffix = suffix + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::(); + let candidate = vfs_join(parent, format!(".bashkit-{tool}-{suffix}.tmp")); + if !fs + .exists(&candidate) + .await + .map_err(|error| error.to_string())? + { + temp = Some(candidate); + break; + } + } + let temp = temp.ok_or_else(|| "cannot allocate temporary file".to_string())?; + if let Err(error) = fs.write_file(&temp, content).await { + let _ = fs.remove(&temp, false).await; + return Err(format!("cannot write temporary file: {error}")); + } + #[cfg(feature = "failpoints")] + if injected_failure(failpoints.chmod, "error") { + let _ = fs.remove(&temp, false).await; + return Err("cannot preserve file mode: injected failure".to_string()); + } + if let Ok(metadata) = fs.stat(target).await + && let Err(error) = fs.chmod(&temp, metadata.mode).await + { + let _ = fs.remove(&temp, false).await; + return Err(format!("cannot preserve file mode: {error}")); + } + #[cfg(feature = "failpoints")] + if injected_failure(failpoints.rename, "error") { + let _ = fs.remove(&temp, false).await; + return Err(format!( + "cannot replace '{}': injected failure", + target.display() + )); + } + if let Err(error) = fs.rename(&temp, target).await { + let _ = fs.remove(&temp, false).await; + return Err(format!("cannot replace '{}': {error}", target.display())); + } + #[cfg(not(feature = "failpoints"))] + let _ = failpoints; + Ok(()) +} + +#[cfg(feature = "failpoints")] +fn injected_failure(name: &str, expected: &str) -> bool { + fail::eval(name, |action| action.as_deref() == Some(expected)).unwrap_or(false) +} diff --git a/crates/bashkit/src/builtins/limits.rs b/crates/bashkit/src/builtins/limits.rs index cd17fefab..c6d3fc7c6 100644 --- a/crates/bashkit/src/builtins/limits.rs +++ b/crates/bashkit/src/builtins/limits.rs @@ -74,6 +74,10 @@ pub(crate) const RETRY_MAX_ATTEMPTS: u32 = 10_000; /// sed: max group-nesting depth in `s` replacements. pub(crate) const SED_MAX_GROUP_NESTING_DEPTH: usize = 128; +/// sed: max instructions executed for one input line, bounding `b`/`t` loops. +/// THREAT[TM-DOS]: sed scripts are untrusted text and `:a;ba` never terminates. +pub(crate) const SED_MAX_CYCLE_STEPS: usize = 1_000; + /// sleep: max sleep duration. pub(crate) const SLEEP_MAX_SECONDS: f64 = 60.0; diff --git a/crates/bashkit/src/builtins/mod.rs b/crates/bashkit/src/builtins/mod.rs index 6be483304..4f9a6f64c 100644 --- a/crates/bashkit/src/builtins/mod.rs +++ b/crates/bashkit/src/builtins/mod.rs @@ -26,6 +26,7 @@ mod alias; mod archive; pub(crate) mod arg_parser; mod assert; +mod atomic_write; mod awk; mod base64; mod bc; diff --git a/crates/bashkit/src/builtins/sed.rs b/crates/bashkit/src/builtins/sed.rs deleted file mode 100644 index 0680054d1..000000000 --- a/crates/bashkit/src/builtins/sed.rs +++ /dev/null @@ -1,1475 +0,0 @@ -//! sed - Stream editor builtin -//! -//! Implements basic sed functionality. -//! -//! Usage: -//! sed 's/pattern/replacement/' file -//! sed 's/pattern/replacement/g' file # global replacement -//! sed 's/pattern/replacement/2' file # nth occurrence -//! sed -E 's/pattern+/replacement/' file # extended regex -//! sed -i 's/pattern/replacement/' file # in-place edit -//! echo "text" | sed 's/pattern/replacement/' -//! sed -n '2p' file # print line 2 -//! sed '2d' file # delete line 2 -//! sed '/bar/!d' file # delete lines not matching bar -//! sed -e 's/a/b/' -e 's/c/d/' file # multiple commands - -// sed command parser uses chars().next().unwrap() after validating. -// This is safe because we check for non-empty strings before accessing. -#![allow(clippy::unwrap_used)] - -use async_trait::async_trait; -use regex::Regex; - -use super::search_common::{REGEX_DFA_SIZE_LIMIT, REGEX_SIZE_LIMIT, build_regex, build_regex_opts}; - -use super::limits::SED_MAX_GROUP_NESTING_DEPTH as MAX_GROUP_NESTING_DEPTH; -use super::{Builtin, Context, read_text_file}; -use crate::error::{Error, Result}; -use crate::fs::vfs_join; -use crate::interpreter::ExecResult; - -/// Regex wrapper that falls back to fancy-regex for patterns with backreferences. -/// The standard `regex` crate doesn't support backreferences in search patterns -/// (e.g. `\(.\)\1`). When such patterns are detected, we use `fancy_regex` instead. -#[derive(Debug)] -enum SedRegex { - Standard(Regex), - Fancy(fancy_regex::Regex), -} - -impl SedRegex { - /// Backtracking step cap for fancy-regex fallback. - const FANCY_BACKTRACK_LIMIT: usize = 1_000_000; - - fn build_fancy_with_limit( - pattern: &str, - case_insensitive: bool, - backtrack_limit: usize, - ) -> std::result::Result { - fancy_regex::RegexBuilder::new(pattern) - .case_insensitive(case_insensitive) - .delegate_size_limit(REGEX_SIZE_LIMIT) - .delegate_dfa_size_limit(REGEX_DFA_SIZE_LIMIT) - .backtrack_limit(backtrack_limit) - .build() - .map(SedRegex::Fancy) - .map_err(|e| e.to_string()) - } - - fn build_fancy(pattern: &str, case_insensitive: bool) -> std::result::Result { - Self::build_fancy_with_limit(pattern, case_insensitive, Self::FANCY_BACKTRACK_LIMIT) - } - - /// Build a regex, falling back to fancy-regex if backreferences are present. - fn new(pattern: &str, case_insensitive: bool) -> std::result::Result { - match build_regex_opts(pattern, case_insensitive) { - Ok(re) => Ok(SedRegex::Standard(re)), - Err(e) => { - let err_msg = e.to_string(); - if err_msg.contains("backreference") { - Self::build_fancy(pattern, case_insensitive) - } else { - Err(err_msg) - } - } - } - } - - fn replace<'t>(&self, text: &'t str, rep: &str) -> std::borrow::Cow<'t, str> { - match self { - SedRegex::Standard(re) => re.replace(text, rep), - SedRegex::Fancy(re) => re.replace(text, rep), - } - } - - fn replace_all<'t>(&self, text: &'t str, rep: &str) -> std::borrow::Cow<'t, str> { - match self { - SedRegex::Standard(re) => re.replace_all(text, rep), - SedRegex::Fancy(re) => re.replace_all(text, rep), - } - } -} - -/// Convert a BRE (Basic Regular Expression) pattern to ERE for the regex crate. -/// In BRE: ( ) { } are literal; \( \) \{ \} \+ \? \| are metacharacters. -fn bre_to_ere(pattern: &str) -> String { - let mut result = String::with_capacity(pattern.len()); - let chars: Vec = pattern.chars().collect(); - let mut i = 0; - - while i < chars.len() { - if chars[i] == '\\' && i + 1 < chars.len() { - match chars[i + 1] { - '(' | ')' | '{' | '}' | '+' | '?' | '|' => { - result.push(chars[i + 1]); - i += 2; - } - _ => { - result.push('\\'); - result.push(chars[i + 1]); - i += 2; - } - } - } else if chars[i] == '(' || chars[i] == ')' || chars[i] == '{' || chars[i] == '}' { - result.push('\\'); - result.push(chars[i]); - i += 1; - } else { - result.push(chars[i]); - i += 1; - } - } - - result -} - -/// sed command - stream editor -pub struct Sed; - -#[derive(Debug)] -enum SedCommand { - Substitute { - pattern: SedRegex, - replacement: String, - global: bool, - nth: Option, // Replace nth occurrence (1-indexed) - print_only: bool, - }, - Delete, - Print, - Quit, - QuitNoprint, // Q - quit without printing - Append(String), - Insert(String), - Change(String), // c\text - replace line - HoldCopy, // h - copy pattern to hold - HoldAppend, // H - append pattern to hold - GetCopy, // g - copy hold to pattern - GetAppend, // G - append hold to pattern - Exchange, // x - swap pattern and hold - Group(Vec<(Option
, bool, SedCommand)>), // { cmd1; cmd2; ... } - Label(String), // :label - Branch(Option), // b [label] - unconditional branch - BranchIfSub(Option), // t [label] - branch if substitution succeeded -} - -#[derive(Debug, Clone)] -enum Address { - All, - Line(usize), - Range(usize, usize), - Regex(Regex), - Last, - RegexRange(Regex, Regex), // /start/,/end/ - regex range - LineRegexRange(usize, Regex), // N,/end/ - Step(usize, usize), // first~step - every step-th line starting at first - ZeroRegex(Regex), // 0,/pattern/ - from line 0 to first match -} - -impl Address { - fn matches_simple(&self, line_num: usize, total_lines: usize, line: &str) -> bool { - match self { - Address::All => true, - Address::Line(n) => line_num == *n, - Address::Range(start, end) => line_num >= *start && line_num <= *end, - Address::Regex(re) => re.is_match(line), - Address::Last => line_num == total_lines, - Address::Step(first, step) => { - if *step == 0 { - line_num == *first - } else if *first == 0 { - line_num.is_multiple_of(*step) - } else { - line_num >= *first && (line_num - *first).is_multiple_of(*step) - } - } - // Ranges with state handled separately - Address::RegexRange(_, _) | Address::LineRegexRange(_, _) | Address::ZeroRegex(_) => { - false - } - } - } - - fn matches_with_state( - &self, - line_num: usize, - total_lines: usize, - line: &str, - in_range: &mut bool, - ) -> bool { - match self { - Address::RegexRange(start_re, end_re) => { - if *in_range { - if end_re.is_match(line) { - *in_range = false; - } - true - } else if start_re.is_match(line) { - *in_range = true; - true - } else { - false - } - } - Address::LineRegexRange(start_line, end_re) => { - if *in_range { - if end_re.is_match(line) { - *in_range = false; - } - true - } else if line_num >= *start_line { - *in_range = true; - true - } else { - false - } - } - Address::ZeroRegex(end_re) => { - // 0,/pattern/ — always match from line 1; stop after first match - if *in_range { - if end_re.is_match(line) { - *in_range = false; - } - true - } else if line_num == 1 { - // Start matching from line 1 (0 is virtual start) - *in_range = true; - // Check if first line already matches end pattern - if end_re.is_match(line) { - *in_range = false; - } - true - } else { - false - } - } - _ => self.matches_simple(line_num, total_lines, line), - } - } -} - -struct SedOptions { - commands: Vec<(Option
, bool, SedCommand)>, // (address, negate, command) - files: Vec, - in_place: bool, - quiet: bool, - extended_regex: bool, -} - -impl SedOptions { - fn parse(args: &[String]) -> Result> { - let mut opts = SedOptions { - commands: Vec::new(), - files: Vec::new(), - in_place: false, - quiet: false, - extended_regex: false, - }; - - // First pass: check for -E flag - for arg in args { - if arg == "-E" || arg == "-r" { - opts.extended_regex = true; - } - } - - let mut i = 0; - while i < args.len() { - let arg = &args[i]; - if arg == "-n" { - opts.quiet = true; - } else if arg == "-i" { - opts.in_place = true; - } else if arg == "-E" || arg == "-r" { - // Already handled - } else if arg == "-e" { - i += 1; - if i < args.len() { - let (addr, negate, cmd) = parse_sed_command(&args[i], opts.extended_regex)?; - opts.commands.push((addr, negate, cmd)); - } - } else if arg == "-" || arg == "--" { - // "-" is the stdin operand; "--" ends options. Neither is a flag. - } else if arg.starts_with('-') && arg.len() > 1 { - // Unknown option-shaped token → reject (sed exits 1). - return Ok(Err(super::invalid_option("sed", arg, 1))); - } else if opts.commands.is_empty() { - // First non-option is the command (may contain multiple commands separated by ;) - for cmd_str in split_sed_commands(arg) { - let trimmed = cmd_str.trim(); - if !trimmed.is_empty() { - let (addr, negate, cmd) = parse_sed_command(trimmed, opts.extended_regex)?; - opts.commands.push((addr, negate, cmd)); - } - } - } else { - // Rest are files - opts.files.push(arg.clone()); - } - i += 1; - } - - if opts.commands.is_empty() { - return Err(Error::Execution("sed: no command given".to_string())); - } - - Ok(Ok(opts)) - } -} - -/// Split a sed command string into individual commands separated by semicolons. -/// This is careful to not split inside s/pattern/replacement/ or { } blocks. -fn split_sed_commands(s: &str) -> Vec<&str> { - let mut result = Vec::new(); - let mut start = 0; - let mut in_subst = false; - let mut delim_count = 0; - let mut delim: Option = None; - let mut escaped = false; - let mut brace_depth = 0; - let chars: Vec<(usize, char)> = s.char_indices().collect(); - - for (idx, &(byte_pos, c)) in chars.iter().enumerate() { - if escaped { - escaped = false; - continue; - } - - if c == '\\' { - escaped = true; - continue; - } - - if !in_subst && c == 's' && idx + 1 < chars.len() { - // Start of substitution command - in_subst = true; - delim = Some(chars[idx + 1].1); - delim_count = 0; - } else if in_subst { - if Some(c) == delim { - delim_count += 1; - if delim_count >= 3 { - // After third delimiter, we might have flags then end - in_subst = false; - } - } - } else if c == '{' { - brace_depth += 1; - } else if c == '}' { - brace_depth -= 1; - } else if c == ';' && brace_depth == 0 { - result.push(&s[start..byte_pos]); - start = byte_pos + c.len_utf8(); - } - } - - if start < s.len() { - result.push(&s[start..]); - } - - result -} - -fn parse_address(s: &str) -> Result<(Option
, &str)> { - if s.is_empty() { - return Ok((None, s)); - } - - let first_char = s.chars().next().unwrap(); - - // Line number - if first_char.is_ascii_digit() { - let end = s.find(|c: char| !c.is_ascii_digit()).unwrap_or(s.len()); - let num: usize = s[..end] - .parse() - .map_err(|_| Error::Execution("sed: invalid address".to_string()))?; - let rest = &s[end..]; - - // Check for step address: first~step - if let Some(after_tilde) = rest.strip_prefix('~') { - let end2 = after_tilde - .find(|c: char| !c.is_ascii_digit()) - .unwrap_or(after_tilde.len()); - if end2 > 0 { - let step: usize = after_tilde[..end2] - .parse() - .map_err(|_| Error::Execution("sed: invalid step address".to_string()))?; - return Ok((Some(Address::Step(num, step)), &after_tilde[end2..])); - } - } - - // Check for range - if let Some(rest) = rest.strip_prefix(',') { - if let Some(after_dollar) = rest.strip_prefix('$') { - return Ok((Some(Address::Range(num, usize::MAX)), after_dollar)); - } - // N,/pattern/ range — 0,/pat/ is special (matches first occurrence) - if let Some(after_slash) = rest.strip_prefix('/') { - let end2 = after_slash.find('/').ok_or_else(|| { - Error::Execution("sed: unterminated address regex".to_string()) - })?; - let pattern = &after_slash[..end2]; - let regex = build_regex(pattern) - .map_err(|e| Error::Execution(format!("sed: invalid regex: {}", e)))?; - if num == 0 { - return Ok((Some(Address::ZeroRegex(regex)), &after_slash[end2 + 1..])); - } - return Ok(( - Some(Address::LineRegexRange(num, regex)), - &after_slash[end2 + 1..], - )); - } - let end2 = rest - .find(|c: char| !c.is_ascii_digit()) - .unwrap_or(rest.len()); - if end2 > 0 { - let num2: usize = rest[..end2] - .parse() - .map_err(|_| Error::Execution("sed: invalid address".to_string()))?; - return Ok((Some(Address::Range(num, num2)), &rest[end2..])); - } - return Ok((Some(Address::Line(num)), rest)); - } - - return Ok((Some(Address::Line(num)), rest)); - } - - // Last line - if let Some(after_dollar) = s.strip_prefix('$') { - return Ok((Some(Address::Last), after_dollar)); - } - - // Regex address /pattern/ - if first_char == '/' { - let end = s[1..] - .find('/') - .ok_or_else(|| Error::Execution("sed: unterminated address regex".to_string()))?; - let pattern = &s[1..end + 1]; - let regex = build_regex(pattern) - .map_err(|e| Error::Execution(format!("sed: invalid regex: {}", e)))?; - let rest = &s[end + 2..]; - - // Check for regex range: /start/,/end/ or /start/,$ - if let Some(after_comma) = rest.strip_prefix(',') { - if let Some(after_dollar) = after_comma.strip_prefix('$') { - // /pattern/,$ — from regex to end - return Ok(( - Some(Address::RegexRange( - regex, - Regex::new("$^").unwrap(), // Never matches - range goes to end - )), - after_dollar, - )); - } - if let Some(after_slash) = after_comma.strip_prefix('/') { - let end2 = after_slash.find('/').ok_or_else(|| { - Error::Execution("sed: unterminated address regex".to_string()) - })?; - let pattern2 = &after_slash[..end2]; - let regex2 = build_regex(pattern2) - .map_err(|e| Error::Execution(format!("sed: invalid regex: {}", e)))?; - return Ok(( - Some(Address::RegexRange(regex, regex2)), - &after_slash[end2 + 1..], - )); - } - // /pattern/,N - let end2 = after_comma - .find(|c: char| !c.is_ascii_digit()) - .unwrap_or(after_comma.len()); - if end2 > 0 - && let Ok(line_num) = after_comma[..end2].parse::() - { - // Create regex that matches at line N (handled as RegexRange with a line check) - return Ok(( - Some(Address::Range(0, line_num)), // Approximate - &after_comma[end2..], - )); - } - } - - return Ok((Some(Address::Regex(regex)), rest)); - } - - Ok((None, s)) -} - -fn validate_group_nesting(s: &str, base_depth: usize) -> Result<()> { - let mut in_subst = false; - let mut subst_delim = None; - let mut subst_delim_count = 0; - let mut escaped = false; - let mut group_depth = 0; - - for c in s.chars() { - if escaped { - escaped = false; - continue; - } - - if c == '\\' { - escaped = true; - continue; - } - - if !in_subst && c == 's' { - in_subst = true; - subst_delim = None; - subst_delim_count = 0; - continue; - } - - if in_subst { - if subst_delim.is_none() { - subst_delim = Some(c); - continue; - } - - if Some(c) == subst_delim { - subst_delim_count += 1; - if subst_delim_count >= 3 { - in_subst = false; - subst_delim = None; - subst_delim_count = 0; - } - } - continue; - } - - match c { - '{' => { - group_depth += 1; - if base_depth + group_depth > MAX_GROUP_NESTING_DEPTH { - return Err(Error::Execution(format!( - "sed: grouped command nesting exceeds max depth {}", - MAX_GROUP_NESTING_DEPTH - ))); - } - } - '}' => { - group_depth = group_depth.saturating_sub(1); - } - _ => {} - } - } - - Ok(()) -} - -fn parse_sed_command(s: &str, extended_regex: bool) -> Result<(Option
, bool, SedCommand)> { - parse_sed_command_with_depth(s, extended_regex, 0) -} - -fn parse_sed_command_with_depth( - s: &str, - extended_regex: bool, - depth: usize, -) -> Result<(Option
, bool, SedCommand)> { - let (address, rest) = parse_address(s)?; - - if rest.is_empty() { - return Err(Error::Execution("sed: missing command".to_string())); - } - - // Check for address negation (!) - let (negate, rest) = if let Some(r) = rest.strip_prefix('!') { - (true, r) - } else { - (false, rest) - }; - - if rest.is_empty() { - return Err(Error::Execution("sed: missing command".to_string())); - } - - let first_char = rest.chars().next().unwrap(); - - match first_char { - 's' => { - // Substitution: s/pattern/replacement/flags - if rest.len() < 4 { - return Err(Error::Execution("sed: invalid substitution".to_string())); - } - let delim = rest.chars().nth(1).unwrap(); - - // Find the parts between delimiters - let rest = &rest[2..]; - let mut parts = Vec::new(); - let mut current = String::new(); - let mut escaped = false; - - for c in rest.chars() { - if escaped { - current.push(c); - escaped = false; - } else if c == '\\' { - escaped = true; - current.push(c); - } else if c == delim { - parts.push(current); - current = String::new(); - } else { - current.push(c); - } - } - parts.push(current); - - if parts.len() < 2 { - return Err(Error::Execution("sed: invalid substitution".to_string())); - } - - let pattern = &parts[0]; - let replacement = &parts[1]; - let flags = parts.get(2).map(|s| s.as_str()).unwrap_or(""); - - // Convert POSIX sed regex to Rust regex syntax - // In BRE mode: \( \) are groups, ( ) are literal, \+ \? are quantifiers - // In ERE mode: ( ) are groups, + ? work directly - let pattern = if extended_regex { - // ERE mode: no conversion needed for groups/quantifiers - pattern.clone() - } else { - // BRE mode: proper char-by-char conversion - bre_to_ere(pattern) - }; - // Build regex with optional case-insensitive flag. - // Falls back to fancy-regex for patterns with backreferences. - let case_insensitive = flags.contains('i'); - let regex = SedRegex::new(&pattern, case_insensitive) - .map_err(|e| Error::Execution(format!("sed: invalid pattern: {}", e)))?; - - // Convert sed replacement syntax to regex replacement syntax - // sed uses \1, \2, etc. and & for full match - // regex crate uses ${N} format to avoid ambiguity - let replacement = replacement - .replace("\\&", "\x00") // Temporarily escape literal & - .replace('&', "${0}") - .replace("\x00", "&"); - - // Use ${N} format instead of $N to avoid ambiguity with following chars - let replacement = Regex::new(r"\\(\d+)") - .unwrap() - .replace_all(&replacement, r"$${$1}") - .to_string(); - - // Convert \n → newline, \t → tab, \/ → /, \\ → \ in replacement - let replacement = replacement - .replace("\\\\", "\x01") // Temporarily escape literal \\ - .replace("\\n", "\n") - .replace("\\t", "\t") - .replace("\\/", "/") - .replace("\x01", "\\"); // Restore literal backslash - - // Parse nth occurrence from flags (e.g., "2" in s/a/b/2) - let nth = flags - .chars() - .filter(|c| c.is_ascii_digit()) - .collect::() - .parse::() - .ok() - .filter(|&n| n > 0); - - Ok(( - address, - negate, - SedCommand::Substitute { - pattern: regex, - replacement, - global: flags.contains('g'), - nth, - print_only: flags.contains('p'), - }, - )) - } - 'd' => Ok((address.or(Some(Address::All)), negate, SedCommand::Delete)), - 'p' => Ok((address.or(Some(Address::All)), negate, SedCommand::Print)), - 'q' => Ok((address, negate, SedCommand::Quit)), - 'a' => { - // Append command: a\text or a text (after backslash) - let text = if rest.len() > 1 && rest.chars().nth(1) == Some('\\') { - rest[2..].to_string() - } else { - rest[1..].to_string() - }; - Ok((address, negate, SedCommand::Append(text))) - } - 'i' => { - // Insert command: i\text or i text (after backslash) - let text = if rest.len() > 1 && rest.chars().nth(1) == Some('\\') { - rest[2..].to_string() - } else { - rest[1..].to_string() - }; - Ok((address, negate, SedCommand::Insert(text))) - } - 'c' => { - // Change command: c\text - let text = if rest.len() > 1 && rest.chars().nth(1) == Some('\\') { - rest[2..].to_string() - } else { - rest[1..].to_string() - }; - Ok((address, negate, SedCommand::Change(text))) - } - 'h' => Ok((address, negate, SedCommand::HoldCopy)), - 'H' => Ok((address, negate, SedCommand::HoldAppend)), - 'g' if rest.len() == 1 || !rest[1..].starts_with('/') => { - // 'g' alone is get-from-hold; 'g' after s// is global flag (handled in Substitute) - Ok((address, negate, SedCommand::GetCopy)) - } - 'G' => Ok((address, negate, SedCommand::GetAppend)), - 'x' => Ok((address, negate, SedCommand::Exchange)), - 'Q' => Ok((address, negate, SedCommand::QuitNoprint)), - ':' => { - // Label: :name - let label = rest[1..].trim().to_string(); - Ok((None, false, SedCommand::Label(label))) - } - 'b' => { - // Branch: b [label] — unconditional jump - let label = rest[1..].trim(); - let label = if label.is_empty() { - None - } else { - Some(label.to_string()) - }; - Ok((address, negate, SedCommand::Branch(label))) - } - 't' => { - // Branch if substitution: t [label] - let label = rest[1..].trim(); - let label = if label.is_empty() { - None - } else { - Some(label.to_string()) - }; - Ok((address, negate, SedCommand::BranchIfSub(label))) - } - '{' => { - validate_group_nesting(rest, depth)?; - if depth + 1 > MAX_GROUP_NESTING_DEPTH { - return Err(Error::Execution(format!( - "sed: grouped command nesting exceeds max depth {}", - MAX_GROUP_NESTING_DEPTH - ))); - } - // Grouped commands: { cmd1; cmd2; ... } - // Find matching closing brace - let inner = rest[1..].trim(); - let inner = inner.strip_suffix('}').unwrap_or(inner); - let mut group_cmds = Vec::new(); - for cmd_str in split_sed_commands(inner) { - let trimmed = cmd_str.trim(); - if !trimmed.is_empty() { - let (a, n, c) = - parse_sed_command_with_depth(trimmed, extended_regex, depth + 1)?; - group_cmds.push((a, n, c)); - } - } - Ok((address, negate, SedCommand::Group(group_cmds))) - } - _ => Err(Error::Execution(format!( - "sed: unknown command: {}", - first_char - ))), - } -} - -/// Replace the nth occurrence of a pattern in a string -fn replace_nth<'a>( - pattern: &SedRegex, - text: &'a str, - replacement: &str, - n: usize, -) -> std::borrow::Cow<'a, str> { - match pattern { - SedRegex::Standard(re) => { - let mut count = 0; - for mat in re.find_iter(text) { - count += 1; - if count == n { - let mut result = String::new(); - result.push_str(&text[..mat.start()]); - let replaced = re.replace(mat.as_str(), replacement); - result.push_str(&replaced); - result.push_str(&text[mat.end()..]); - return std::borrow::Cow::Owned(result); - } - } - std::borrow::Cow::Borrowed(text) - } - SedRegex::Fancy(re) => { - let mut count = 0; - for mat in re.find_iter(text).filter_map(|m| m.ok()) { - count += 1; - if count == n { - let mut result = String::new(); - result.push_str(&text[..mat.start()]); - let replaced = re.replace(mat.as_str(), replacement); - result.push_str(&replaced); - result.push_str(&text[mat.end()..]); - return std::borrow::Cow::Owned(result); - } - } - std::borrow::Cow::Borrowed(text) - } - } -} - -/// Mutable state passed through command execution -struct LineState { - current_line: String, - should_print: bool, - deleted: bool, - extra_output: Vec, // lines printed by 'p' command (printed immediately) - insert_text: Option, - append_text: Option, - quit: bool, - quit_noprint: bool, - hold_space: String, - sub_happened: bool, // track if any substitution succeeded (for t command) -} - -/// Execute a single sed command against the current line state. -/// Returns true if the command was applied (for branching logic). -fn exec_sed_cmd(cmd: &SedCommand, state: &mut LineState, _line_num: usize, _total_lines: usize) { - exec_sed_cmd_with_depth(cmd, state, _line_num, _total_lines, 0); -} - -fn exec_sed_cmd_with_depth( - cmd: &SedCommand, - state: &mut LineState, - _line_num: usize, - _total_lines: usize, - depth: usize, -) { - match cmd { - SedCommand::Substitute { - pattern, - replacement, - global, - nth, - print_only, - } => { - let new_line = if *global { - pattern.replace_all(&state.current_line, replacement.as_str()) - } else if let Some(n) = nth { - replace_nth(pattern, &state.current_line, replacement, *n) - } else { - pattern.replace(&state.current_line, replacement.as_str()) - }; - - if new_line != state.current_line { - state.current_line = new_line.into_owned(); - state.sub_happened = true; - if *print_only { - state.extra_output.push(state.current_line.clone()); - } - } - } - SedCommand::Delete => { - state.deleted = true; - state.should_print = false; - } - SedCommand::Print => { - // Print current pattern space immediately (snapshot at this point) - state.extra_output.push(state.current_line.clone()); - } - SedCommand::Quit => { - state.quit = true; - } - SedCommand::QuitNoprint => { - state.quit_noprint = true; - } - SedCommand::Append(text) => { - state.append_text = Some(text.clone()); - } - SedCommand::Insert(text) => { - state.insert_text = Some(text.clone()); - } - SedCommand::Change(text) => { - state.current_line = text.clone(); - state.deleted = false; - state.should_print = true; - } - SedCommand::HoldCopy => { - state.hold_space = state.current_line.clone(); - } - SedCommand::HoldAppend => { - state.hold_space.push('\n'); - state.hold_space.push_str(&state.current_line); - } - SedCommand::GetCopy => { - state.current_line = state.hold_space.clone(); - } - SedCommand::GetAppend => { - state.current_line.push('\n'); - state.current_line.push_str(&state.hold_space); - } - SedCommand::Exchange => { - std::mem::swap(&mut state.current_line, &mut state.hold_space); - } - SedCommand::Group(cmds) => { - if depth >= MAX_GROUP_NESTING_DEPTH { - return; - } - for (_addr, _negate, sub_cmd) in cmds { - exec_sed_cmd_with_depth(sub_cmd, state, _line_num, _total_lines, depth + 1); - if state.deleted || state.quit || state.quit_noprint { - break; - } - } - } - // Labels and branches are handled at the top-level command loop - SedCommand::Label(_) | SedCommand::Branch(_) | SedCommand::BranchIfSub(_) => {} - } -} - -/// Count total commands (including nested) for range state tracking -fn count_commands(cmds: &[(Option
, bool, SedCommand)]) -> usize { - let mut count = 0; - for (_, _, cmd) in cmds { - count += 1; - if let SedCommand::Group(sub) = cmd { - count += count_commands(sub); - } - } - count -} - -#[async_trait] -impl Builtin for Sed { - async fn execute(&self, ctx: Context<'_>) -> Result { - if let Some(r) = super::check_help_version( - ctx.args, - "Usage: sed [OPTION]... {script} [FILE]...\nStream editor for filtering and transforming text.\n\n -n\t\tsuppress automatic printing of pattern space\n -i\t\tedit files in place\n -E, -r\tuse extended regular expressions\n -e script\tadd the script to the commands to be executed\n --help\tdisplay this help and exit\n --version\toutput version information and exit\n", - Some("sed (bashkit) 0.1"), - ) { - return Ok(r); - } - let opts = match SedOptions::parse(ctx.args)? { - Ok(o) => o, - Err(e) => return Ok(e), - }; - - // Determine input - let inputs: Vec<(Option, String)> = if opts.files.is_empty() { - vec![(None, ctx.stdin.map(ToString::to_string).unwrap_or_default())] - } else { - let mut inputs = Vec::new(); - for file in &opts.files { - let path = if file.starts_with('/') { - std::path::PathBuf::from(file) - } else { - vfs_join(ctx.cwd, file) - }; - - let text = match read_text_file(&*ctx.fs, &path, "sed").await { - Ok(t) => t, - Err(e) => return Ok(e), - }; - inputs.push((Some(file.clone()), text)); - } - inputs - }; - - let mut output = String::new(); - let mut warnings = String::new(); - let mut modified_files: Vec<(String, String)> = Vec::new(); - - for (filename, content) in inputs { - let lines: Vec<&str> = content.lines().collect(); - let total_lines = lines.len(); - let mut file_output = String::new(); - let mut hold_space = String::new(); - let mut global_quit = false; - // Track range state per command - let mut range_state: Vec = vec![false; count_commands(&opts.commands)]; - - for (idx, line) in lines.iter().enumerate() { - if global_quit { - break; - } - - let line_num = idx + 1; - let mut state = LineState { - current_line: line.to_string(), - should_print: !opts.quiet, - deleted: false, - extra_output: Vec::new(), - insert_text: None, - append_text: None, - quit: false, - quit_noprint: false, - hold_space: hold_space.clone(), - sub_happened: false, - }; - - // Execute commands with branch/label support - let mut cmd_idx = 0; - let max_iterations = 1000; // prevent infinite loops - let mut iterations = 0; - while cmd_idx < opts.commands.len() && iterations < max_iterations { - iterations += 1; - let (addr, negate, cmd) = &opts.commands[cmd_idx]; - - let addr_matches = addr - .as_ref() - .map(|a| { - a.matches_with_state( - line_num, - total_lines, - &state.current_line, - &mut range_state[cmd_idx], - ) - }) - .unwrap_or(true); - - let should_apply = if *negate { !addr_matches } else { addr_matches }; - - if !should_apply { - cmd_idx += 1; - continue; - } - - match cmd { - SedCommand::Label(_) => { - // Labels are just markers, skip - cmd_idx += 1; - } - SedCommand::Branch(label) => { - if let Some(label) = label { - // Jump to label - if let Some(pos) = find_label(&opts.commands, label) { - cmd_idx = pos; - } else { - cmd_idx += 1; - } - } else { - // b with no label = jump to end (skip remaining) - break; - } - } - SedCommand::BranchIfSub(label) => { - if state.sub_happened { - state.sub_happened = false; - if let Some(label) = label { - if let Some(pos) = find_label(&opts.commands, label) { - cmd_idx = pos; - } else { - cmd_idx += 1; - } - } else { - break; - } - } else { - cmd_idx += 1; - } - } - _ => { - exec_sed_cmd(cmd, &mut state, line_num, total_lines); - cmd_idx += 1; - } - } - - if state.deleted || state.quit || state.quit_noprint { - break; - } - } - - if iterations >= max_iterations { - warnings.push_str(&format!( - "sed: warning: branch/label loop limit ({max_iterations}) reached on line {line_num}; output may be truncated\n" - )); - } - - hold_space = state.hold_space; - - // Insert text comes before the line - if let Some(text) = state.insert_text { - file_output.push_str(&text); - file_output.push('\n'); - } - - if state.quit_noprint { - global_quit = true; - // Q does NOT print the current line - } else { - // Extra output from p command comes before auto-print - for extra in &state.extra_output { - file_output.push_str(extra); - file_output.push('\n'); - } - - if !state.deleted && state.should_print { - file_output.push_str(&state.current_line); - file_output.push('\n'); - } - - // Append text comes after the line - if let Some(text) = state.append_text { - file_output.push_str(&text); - file_output.push('\n'); - } - - if state.quit { - global_quit = true; - } - } - } - - if opts.in_place { - if let Some(fname) = filename { - modified_files.push((fname, file_output)); - } - } else { - output.push_str(&file_output); - } - } - - // Write back in-place modifications - for (filename, content) in modified_files { - let path = if filename.starts_with('/') { - std::path::PathBuf::from(&filename) - } else { - vfs_join(ctx.cwd, &filename) - }; - - if let Err(e) = ctx.fs.write_file(&path, content.as_bytes()).await { - return Ok(ExecResult::err(format!("sed: {}: {}", filename, e), 1)); - } - } - - if warnings.is_empty() { - Ok(ExecResult::ok(output)) - } else { - Ok(ExecResult { - stdout: output.into(), - stderr: warnings.into(), - exit_code: 0, - ..Default::default() - }) - } - } -} - -/// Find the index of a label command in the command list -fn find_label(cmds: &[(Option
, bool, SedCommand)], target: &str) -> Option { - for (i, (_, _, cmd)) in cmds.iter().enumerate() { - if let SedCommand::Label(name) = cmd - && name == target - { - return Some(i); - } - } - None -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::fs::InMemoryFs; - use std::collections::HashMap; - use std::path::PathBuf; - use std::sync::Arc; - - async fn run_sed(args: &[&str], stdin: Option<&str>) -> Result { - let sed = Sed; - let fs = Arc::new(InMemoryFs::new()); - let mut vars = HashMap::new(); - let mut cwd = PathBuf::from("/"); - let args: Vec = args.iter().map(|s| s.to_string()).collect(); - - let ctx = Context { - args: &args, - env: &HashMap::new(), - variables: &mut vars, - cwd: &mut cwd, - fs, - stdin: crate::builtins::test_stream_opt(stdin), - #[cfg(feature = "http_client")] - http_client: None, - #[cfg(feature = "git")] - git_client: None, - #[cfg(feature = "ssh")] - ssh_client: None, - shell: None, - }; - - sed.execute(ctx).await - } - - #[tokio::test] - async fn test_sed_substitute() { - let result = run_sed(&["s/hello/goodbye/"], Some("hello world\nhello again")) - .await - .unwrap(); - assert_eq!(result.stdout, "goodbye world\ngoodbye again\n"); - } - - #[tokio::test] - async fn test_sed_invalid_option() { - let result = run_sed(&["-Q", "s/a/b/"], Some("a\n")).await.unwrap(); - assert_eq!(result.exit_code, 1); - assert!(result.stderr.contains("invalid option -- 'Q'")); - } - - #[tokio::test] - async fn test_sed_substitute_global() { - let result = run_sed(&["s/o/0/g"], Some("hello world")).await.unwrap(); - assert_eq!(result.stdout, "hell0 w0rld\n"); - } - - #[tokio::test] - async fn test_sed_substitute_first_only() { - let result = run_sed(&["s/o/0/"], Some("hello world")).await.unwrap(); - assert_eq!(result.stdout, "hell0 world\n"); - } - - #[tokio::test] - async fn test_sed_delete_line() { - let result = run_sed(&["2d"], Some("line1\nline2\nline3")).await.unwrap(); - assert_eq!(result.stdout, "line1\nline3\n"); - } - - #[tokio::test] - async fn test_sed_print_line() { - let result = run_sed(&["-n", "2p"], Some("line1\nline2\nline3")) - .await - .unwrap(); - assert_eq!(result.stdout, "line2\n"); - } - - #[tokio::test] - async fn test_sed_regex_groups() { - let result = run_sed(&["s/\\(hello\\) \\(world\\)/\\2 \\1/"], Some("hello world")) - .await - .unwrap(); - assert_eq!(result.stdout, "world hello\n"); - } - - #[tokio::test] - async fn test_sed_search_backref() { - // Backreference in search pattern: match repeated character - let result = run_sed(&["s/\\(.\\)\\1/X/g"], Some("aabbc")).await.unwrap(); - assert_eq!(result.stdout, "XXc\n"); - } - - #[test] - fn test_sed_fancy_regex_backtrack_limit_enforced() { - let Ok(SedRegex::Fancy(re)) = SedRegex::build_fancy_with_limit(r"(a+)+\1b", false, 100) - else { - panic!("expected fancy-regex fallback"); - }; - let err = re - .is_match("a".repeat(5_000).as_str()) - .expect_err("expected backtrack limit error"); - assert!(!err.to_string().is_empty()); - } - - #[test] - fn test_sed_grouped_command_nesting_depth_limit() { - let mut script = String::new(); - for _ in 0..=MAX_GROUP_NESTING_DEPTH { - script.push('{'); - } - script.push('p'); - for _ in 0..=MAX_GROUP_NESTING_DEPTH { - script.push('}'); - } - - let err = parse_sed_command(&script, false).expect_err("expected nesting depth error"); - assert!(err.to_string().contains("nesting exceeds max depth")); - } - - #[test] - fn test_sed_grouped_command_nesting_depth_limit_with_address_bypass_shape() { - let script = "{/s\u{E000}/{p}}"; - let err = parse_sed_command_with_depth(script, false, MAX_GROUP_NESTING_DEPTH) - .expect_err("expected nesting depth error"); - assert!(err.to_string().contains("nesting exceeds max depth")); - } - - #[tokio::test] - async fn test_sed_search_backref_html() { - // Backreference in search pattern: match tag content matching href - let result = run_sed( - &[r#"s|\1|\1|g"#], - Some(r#"hello"#), - ) - .await - .unwrap(); - assert_eq!(result.stdout, "hello\n"); - } - - #[tokio::test] - async fn test_sed_backref_single() { - // Test single backreference: capture "hel", replace entire match with captured + "p" - let result = run_sed(&["s/\\(hel\\)lo/\\1p/"], Some("hello")) - .await - .unwrap(); - assert_eq!(result.stdout, "help\n"); - } - - #[tokio::test] - async fn test_sed_ampersand() { - let result = run_sed(&["s/world/[&]/"], Some("hello world")) - .await - .unwrap(); - assert_eq!(result.stdout, "hello [world]\n"); - } - - #[tokio::test] - async fn test_sed_address_range() { - let result = run_sed(&["2,3d"], Some("line1\nline2\nline3\nline4")) - .await - .unwrap(); - assert_eq!(result.stdout, "line1\nline4\n"); - } - - #[tokio::test] - async fn test_sed_last_line() { - let result = run_sed(&["$d"], Some("line1\nline2\nline3")).await.unwrap(); - assert_eq!(result.stdout, "line1\nline2\n"); - } - - #[tokio::test] - async fn test_sed_case_insensitive() { - let result = run_sed(&["s/hello/hi/i"], Some("Hello World")) - .await - .unwrap(); - assert_eq!(result.stdout, "hi World\n"); - } - - #[tokio::test] - async fn test_sed_multiple_commands() { - let result = run_sed(&["s/hello/hi/; s/world/there/"], Some("hello world")) - .await - .unwrap(); - assert_eq!(result.stdout, "hi there\n"); - } - - #[tokio::test] - async fn test_sed_multiple_commands_with_unicode_delimiter_split() { - let result = run_sed(&["s/café/coffee/; s/latte/milk/"], Some("café latte")) - .await - .unwrap(); - assert_eq!(result.stdout, "coffee milk\n"); - } - - #[tokio::test] - async fn test_sed_append() { - let result = run_sed(&["/one/a\\inserted"], Some("one\ntwo")) - .await - .unwrap(); - assert_eq!(result.stdout, "one\ninserted\ntwo\n"); - } - - #[tokio::test] - async fn test_sed_insert() { - let result = run_sed(&["/two/i\\inserted"], Some("one\ntwo")) - .await - .unwrap(); - assert_eq!(result.stdout, "one\ninserted\ntwo\n"); - } - - // === Hold space tests === - - #[tokio::test] - async fn test_sed_hold_copy() { - // h copies pattern to hold, g retrieves it - let result = run_sed(&["-e", "1h", "-e", "2g"], Some("first\nsecond\nthird")) - .await - .unwrap(); - assert_eq!(result.stdout, "first\nfirst\nthird\n"); - } - - #[tokio::test] - async fn test_sed_hold_append() { - // H appends to hold, G appends hold to pattern - let result = run_sed(&["-e", "1h", "-e", "2H", "-e", "3G"], Some("a\nb\nc")) - .await - .unwrap(); - // Line 1: prints "a", holds "a" - // Line 2: prints "b", hold becomes "a\nb" - // Line 3: prints "c\na\nb" (G appends hold) - assert_eq!(result.stdout, "a\nb\nc\na\nb\n"); - } - - #[tokio::test] - async fn test_sed_exchange() { - // x swaps pattern and hold - let result = run_sed(&["-e", "1h", "-e", "2x"], Some("first\nsecond\nthird")) - .await - .unwrap(); - // Line 1: prints "first", hold = "first" - // Line 2: exchange => pattern = "first", hold = "second", prints "first" - // Line 3: prints "third" - assert_eq!(result.stdout, "first\nfirst\nthird\n"); - } - - #[tokio::test] - async fn test_sed_change_command() { - let result = run_sed(&["2c\\replaced"], Some("line1\nline2\nline3")) - .await - .unwrap(); - assert_eq!(result.stdout, "line1\nreplaced\nline3\n"); - } - - #[tokio::test] - async fn test_sed_regex_range() { - let result = run_sed( - &["/start/,/end/d"], - Some("before\nstart\nmiddle\nend\nafter"), - ) - .await - .unwrap(); - assert_eq!(result.stdout, "before\nafter\n"); - } - - #[tokio::test] - async fn test_sed_regex_range_substitute() { - let result = run_sed( - &["/begin/,/end/s/x/y/g"], - Some("ax\nbeginx\nmiddlex\nendx\nax"), - ) - .await - .unwrap(); - assert_eq!(result.stdout, "ax\nbeginy\nmiddley\nendy\nax\n"); - } - - #[tokio::test] - async fn test_sed_branch_loop_limit_emits_warning() { - // This sed script loops via branch, doubling 'a' each iteration. - // With 1000 iteration limit, it should emit a warning. - let result = run_sed(&[":loop; s/a/aa/; /a\\{2000\\}/!b loop"], Some("a")) - .await - .unwrap(); - assert!( - result.stderr.contains("loop limit"), - "expected warning on stderr, got: {}", - result.stderr - ); - assert_eq!(result.exit_code, 0); - } - - #[tokio::test] - async fn test_sed_normal_branch_no_warning() { - // A simple branch that completes well under the limit - let result = run_sed(&["s/hello/world/"], Some("hello")).await.unwrap(); - assert!(result.stderr.is_empty()); - assert_eq!(result.stdout, "world\n"); - } - - // TM-INF-022: malformed-regex stderr must not leak `regex` crate Debug. - #[tokio::test] - async fn no_leak_invalid_regex() { - let r = crate::builtins::debug_leak_check::run(r"echo 1 | sed -E 's/[//'").await; - crate::builtins::debug_leak_check::assert_no_leak( - &r, - "sed_invalid_regex", - &["regex::Error", "ParseError {"], - ); - } -} diff --git a/crates/bashkit/src/builtins/sed/exec.rs b/crates/bashkit/src/builtins/sed/exec.rs new file mode 100644 index 000000000..da634c6ed --- /dev/null +++ b/crates/bashkit/src/builtins/sed/exec.rs @@ -0,0 +1,773 @@ +//! sed execution engine. +//! +//! Important decisions (issue #2427): +//! +//! - Output goes through [`Sink`], which mirrors GNU's "missing newline" +//! bookkeeping: a line that arrived without a trailing newline is written +//! without one, and the newline is only emitted if more output follows. That +//! is what stops `printf 'a' | sed 's/a/b/'` from inventing a terminator and, +//! with `-i`, writing it back to the file (finding E). +//! - Range addresses carry explicit [`RangeState`] per program counter, with a +//! resolved end line for `,N`, `,+N` and `,~N`, so a closed range cannot +//! silently re-open (finding C). +//! - `s///` walks the match list itself instead of delegating to +//! `Regex::replace*`, which is what makes `s/x/y/Ng` mean "the Nth and later" +//! (finding F) and keeps `$` literal in the replacement (finding A). +//! - Input is one continuous stream across operands unless `-s`/`-i` asks for +//! per-file streams, so `$`, line numbers and `q` behave like GNU (finding D). + +use std::collections::HashMap; + +use super::pattern::SedRegex; +use super::script::{Addr, CaseOp, EndAddr, Kind, Program, RepPart, StartAddr, Subst}; +use crate::builtins::limits::SED_MAX_CYCLE_STEPS; + +/// One input line plus the two facts the engine needs about it. +pub(super) struct InputLine { + pub(super) text: String, + /// Whether the line was terminated in the input. + pub(super) had_newline: bool, + /// Index into the operand list, for `F` and for `-s`/`-i` segmentation. + pub(super) file: usize, +} + +/// Output buffer that reproduces GNU sed's missing-newline handling. +pub(super) struct Sink { + pub(super) buf: String, + missing_newline: bool, + /// Line terminator: `\n`, or NUL under `-z`. + sep: char, +} + +impl Sink { + fn with_sep(sep: char) -> Self { + Sink { + buf: String::new(), + missing_newline: false, + sep, + } + } +} + +impl Sink { + /// Write one line, adding a terminator only when the source had one. + fn line(&mut self, text: &str, had_newline: bool) { + self.pending(); + self.buf.push_str(text); + if had_newline { + self.buf.push(self.sep); + } else { + self.missing_newline = true; + } + } + + /// Write text that already carries its own terminator. + fn raw(&mut self, text: &str) { + self.pending(); + self.buf.push_str(text); + } + + fn pending(&mut self) { + if self.missing_newline { + self.buf.push(self.sep); + self.missing_newline = false; + } + } +} + +#[derive(Default, Clone)] +struct RangeState { + active: bool, + /// Resolved last line of the range for numeric / `+N` / `~N` ends. + end_line: Option, +} + +enum Append { + Text(String), + File(String), + FileLine(String), +} + +enum Cycle { + /// Fell off the end of the script: auto-print unless `-n`. + Auto, + /// `d`, `n` at end of input, `Q`: no auto-print. + Silent, + /// `D` with an embedded newline: rerun the script without reading input. + Restart, +} + +pub(super) struct Machine<'a> { + prog: &'a Program, + names: &'a [String], + read_files: &'a HashMap, + quiet: bool, + default_line_len: usize, + sep: char, + + out: Sink, + pub(super) stderr: String, + pub(super) write_files: HashMap, + + range: Vec, + hold: String, + appends: Vec, + read_cursor: HashMap, + last_regex: Option>, + pub(super) exit_code: Option, + warned: bool, + + // Per-segment state. + lines: &'a [InputLine], + cursor: usize, + line_no: usize, + ps: String, + ps_had_newline: bool, + cur_file: usize, + sub_made: bool, +} + +impl<'a> Machine<'a> { + pub(super) fn new( + prog: &'a Program, + names: &'a [String], + read_files: &'a HashMap, + quiet: bool, + line_len: usize, + sep: char, + ) -> Self { + Machine { + prog, + names, + read_files, + quiet: quiet || prog.quiet, + default_line_len: line_len, + sep, + out: Sink::with_sep(sep), + stderr: String::new(), + write_files: HashMap::new(), + range: Vec::new(), + hold: String::new(), + appends: Vec::new(), + read_cursor: HashMap::new(), + last_regex: None, + exit_code: None, + warned: false, + lines: &[], + cursor: 0, + line_no: 0, + ps: String::new(), + ps_had_newline: true, + cur_file: 0, + sub_made: false, + } + } + + pub(super) fn finished(&self) -> bool { + self.exit_code.is_some() + } + + /// Run one input stream and return everything it wrote to stdout. + pub(super) fn run_segment(&mut self, lines: &'a [InputLine]) -> String { + self.lines = lines; + self.cursor = 0; + self.line_no = 0; + self.range = vec![RangeState::default(); self.prog.cmds.len()]; + self.out = Sink::with_sep(self.sep); + + let mut restart = false; + loop { + if self.exit_code.is_some() { + break; + } + // `D` restarts the script on what is left of the pattern space + // without reading input, and GNU only clears the `t` flag when a + // line is actually read, so the reset lives in `read_next`. + if !restart && !self.read_next() { + break; + } + restart = false; + + match self.cycle() { + Cycle::Auto => { + if !self.quiet { + let (ps, nl) = (std::mem::take(&mut self.ps), self.ps_had_newline); + self.out.line(&ps, nl); + self.ps = ps; + } + self.flush_appends(); + } + Cycle::Silent => self.flush_appends(), + Cycle::Restart => { + self.flush_appends(); + restart = true; + } + } + } + + std::mem::replace(&mut self.out, Sink::with_sep(self.sep)).buf + } + + fn cycle(&mut self) -> Cycle { + let cmds = &self.prog.cmds; + let mut pc = 0usize; + let mut steps = 0usize; + + while pc < cmds.len() { + steps += 1; + if steps > SED_MAX_CYCLE_STEPS { + if !self.warned { + self.warned = true; + self.stderr.push_str(&format!( + "sed: warning: branch/label loop limit ({SED_MAX_CYCLE_STEPS}) reached \ + on line {}; output may be truncated\n", + self.line_no + )); + } + break; + } + + let cmd = &cmds[pc]; + let (matched, range_ended) = match &cmd.addr { + None => (true, false), + Some(addr) => self.address_matches(pc, addr), + }; + let apply = if cmd.negate { !matched } else { matched }; + + match &cmd.kind { + Kind::Block(end) => { + pc = if apply { pc + 1 } else { *end + 1 }; + continue; + } + Kind::BlockEnd | Kind::Label(_) | Kind::Nop => { + pc += 1; + continue; + } + _ => {} + } + + if !apply { + pc += 1; + continue; + } + + match &cmd.kind { + Kind::Branch(label) => match label { + None => return Cycle::Auto, + Some(name) => { + pc = self.label_pc(name); + continue; + } + }, + Kind::BranchIfSub(label) => { + if self.sub_made { + self.sub_made = false; + match label { + None => return Cycle::Auto, + Some(name) => { + pc = self.label_pc(name); + continue; + } + } + } + } + Kind::BranchIfNoSub(label) => { + if self.sub_made { + self.sub_made = false; + } else { + match label { + None => return Cycle::Auto, + Some(name) => { + pc = self.label_pc(name); + continue; + } + } + } + } + Kind::Substitute(subst) => self.substitute(subst), + Kind::Transliterate(map) => { + self.ps = self + .ps + .chars() + .map(|c| { + map.iter() + .find(|(from, _)| *from == c) + .map_or(c, |(_, to)| *to) + }) + .collect(); + } + Kind::Delete => return Cycle::Silent, + Kind::DeleteFirstLine => { + return match self.ps.find('\n') { + Some(idx) => { + self.ps.drain(..=idx); + Cycle::Restart + } + None => Cycle::Silent, + }; + } + Kind::Print => { + let (ps, nl) = (std::mem::take(&mut self.ps), self.ps_had_newline); + self.out.line(&ps, nl); + self.ps = ps; + } + Kind::PrintFirstLine => { + let ps = std::mem::take(&mut self.ps); + match ps.find('\n') { + Some(idx) => self.out.line(&ps[..idx], true), + None => self.out.line(&ps, self.ps_had_newline), + } + self.ps = ps; + } + Kind::Next => { + if !self.quiet { + let (ps, nl) = (std::mem::take(&mut self.ps), self.ps_had_newline); + self.out.line(&ps, nl); + self.ps = ps; + } + self.flush_appends(); + if !self.read_next() { + self.exit_code = Some(0); + return Cycle::Silent; + } + } + Kind::NextAppend => { + self.flush_appends(); + if self.cursor >= self.lines.len() { + // GNU extension: print what we have and stop. + self.exit_code = Some(0); + return Cycle::Auto; + } + let line = &self.lines[self.cursor]; + self.ps.push('\n'); + self.ps.push_str(&line.text); + self.ps_had_newline = line.had_newline; + self.cur_file = line.file; + self.cursor += 1; + self.line_no += 1; + self.sub_made = false; + } + Kind::Quit(code) => { + self.exit_code = Some(*code); + return Cycle::Auto; + } + Kind::QuitSilent(code) => { + self.exit_code = Some(*code); + self.appends.clear(); + return Cycle::Silent; + } + Kind::Append(text) => self.appends.push(Append::Text(text.clone())), + Kind::Insert(text) => { + let text = format!("{text}\n"); + self.out.raw(&text); + } + Kind::Change(text) => { + let ranged = cmd.addr.as_ref().is_some_and(|a| a.end.is_some()); + if !ranged || cmd.negate || range_ended { + let text = format!("{text}\n"); + self.out.raw(&text); + } + return Cycle::Silent; + } + Kind::HoldCopy => self.hold.clone_from(&self.ps), + Kind::HoldAppend => { + self.hold.push('\n'); + self.hold.push_str(&self.ps); + } + Kind::GetCopy => self.ps.clone_from(&self.hold), + Kind::GetAppend => { + self.ps.push('\n'); + self.ps.push_str(&self.hold); + } + Kind::Exchange => std::mem::swap(&mut self.ps, &mut self.hold), + Kind::LineNumber => { + let text = format!("{}\n", self.line_no); + self.out.raw(&text); + } + Kind::List(width) => { + let width = width.unwrap_or(self.default_line_len); + let text = format!("{}\n", list_format(&self.ps, width)); + self.out.raw(&text); + } + Kind::ReadFile(name) => self.appends.push(Append::File(name.clone())), + Kind::ReadLine(name) => self.appends.push(Append::FileLine(name.clone())), + Kind::WriteFile(name) => { + let (ps, nl) = (std::mem::take(&mut self.ps), self.ps_had_newline); + self.write_out(name, &ps, nl); + self.ps = ps; + } + Kind::WriteFirstLine(name) => { + let ps = std::mem::take(&mut self.ps); + match ps.find('\n') { + Some(idx) => { + let head = ps[..idx].to_string(); + self.write_out(name, &head, true); + } + None => self.write_out(name, &ps, self.ps_had_newline), + } + self.ps = ps; + } + Kind::Zap => self.ps.clear(), + Kind::FileName => { + let name = self + .names + .get(self.cur_file) + .cloned() + .unwrap_or_else(|| "-".to_string()); + let text = format!("{name}\n"); + self.out.raw(&text); + } + Kind::Block(_) | Kind::BlockEnd | Kind::Label(_) | Kind::Nop => {} + } + + pc += 1; + } + + Cycle::Auto + } + + fn read_next(&mut self) -> bool { + if self.cursor >= self.lines.len() { + return false; + } + let line = &self.lines[self.cursor]; + self.ps.clear(); + self.ps.push_str(&line.text); + self.ps_had_newline = line.had_newline; + self.cur_file = line.file; + self.cursor += 1; + self.line_no += 1; + // Reading a line clears the `t`/`T` flag, as GNU's read_pattern_space does. + self.sub_made = false; + true + } + + fn label_pc(&self, target: &str) -> usize { + self.prog + .cmds + .iter() + .position(|c| matches!(&c.kind, Kind::Label(name) if name == target)) + .unwrap_or(self.prog.cmds.len()) + } + + fn write_out(&mut self, name: &str, text: &str, had_newline: bool) { + match name { + "/dev/stdout" => self.out.line(text, had_newline), + "/dev/stderr" => { + self.stderr.push_str(text); + if had_newline { + self.stderr.push('\n'); + } + } + _ => { + let sep = self.sep; + self.write_files + .entry(name.to_string()) + .or_insert_with(|| Sink::with_sep(sep)) + .line(text, had_newline) + } + } + } + + fn flush_appends(&mut self) { + if self.appends.is_empty() { + return; + } + for item in std::mem::take(&mut self.appends) { + match item { + Append::Text(text) => { + let text = format!("{text}\n"); + self.out.raw(&text); + } + Append::File(name) => { + let Some(content) = self.read_files.get(&name) else { + continue; + }; + if content.is_empty() { + continue; + } + match content.strip_suffix('\n') { + Some(body) => self.out.line(body, true), + None => self.out.line(content, false), + } + } + Append::FileLine(name) => { + let Some(content) = self.read_files.get(&name) else { + continue; + }; + let offset = self.read_cursor.entry(name.clone()).or_insert(0); + if *offset >= content.len() { + continue; + } + let rest = &content[*offset..]; + match rest.find('\n') { + Some(idx) => { + let line = rest[..idx].to_string(); + *offset += idx + 1; + self.out.line(&line, true); + } + None => { + let line = rest.to_string(); + *offset = content.len(); + self.out.line(&line, false); + } + } + } + } + } + } + + // ----- addresses ------------------------------------------------------- + + fn resolve( + &mut self, + re: &Option>, + ) -> Option> { + match re { + Some(re) => { + self.last_regex = Some(re.clone()); + Some(re.clone()) + } + None => self.last_regex.clone(), + } + } + + fn is_last_line(&self) -> bool { + self.cursor >= self.lines.len() + } + + /// Returns `(selected, range_just_closed)`. + fn address_matches(&mut self, pc: usize, addr: &Addr) -> (bool, bool) { + let Some(end) = &addr.end else { + return (self.start_matches(&addr.start), false); + }; + + let state = self.range[pc].clone(); + if state.active { + let closed = match end { + EndAddr::Line(n) => self.line_no >= *n, + EndAddr::Last => self.is_last_line(), + EndAddr::Plus(_) | EndAddr::Multiple(_) => { + state.end_line.is_some_and(|e| self.line_no >= e) + } + EndAddr::Regex(re) => { + let re = self.resolve(re); + re.is_some_and(|re| re.is_match(&self.ps)) + } + }; + let closed = closed || self.is_last_line(); + if closed { + self.range[pc] = RangeState::default(); + } + return (true, closed); + } + + let starts = match &addr.start { + StartAddr::Zero => self.line_no == 1, + other => self.start_matches(other), + }; + if !starts { + return (false, false); + } + + let end_line = match end { + EndAddr::Line(n) => Some(*n), + EndAddr::Plus(n) => Some(self.line_no + n), + EndAddr::Multiple(n) => Some(if *n <= 1 { + self.line_no + } else { + (self.line_no / n + 1) * n + }), + EndAddr::Last | EndAddr::Regex(_) => None, + }; + + // A numeric end at or before the start line makes this a one-line range. + if let Some(e) = end_line + && e <= self.line_no + { + return (true, true); + } + if matches!(end, EndAddr::Last) && self.is_last_line() { + return (true, true); + } + // `0,/re/` is the one form whose end regex is tested on the start line. + if let EndAddr::Regex(re) = end + && matches!(addr.start, StartAddr::Zero) + { + let re = self.resolve(re); + if re.is_some_and(|re| re.is_match(&self.ps)) { + return (true, true); + } + } + if self.is_last_line() { + return (true, true); + } + + self.range[pc] = RangeState { + active: true, + end_line, + }; + (true, false) + } + + fn start_matches(&mut self, start: &StartAddr) -> bool { + match start { + StartAddr::Line(n) => self.line_no == *n, + StartAddr::Zero => false, + StartAddr::Last => self.is_last_line(), + StartAddr::Step(first, step) => { + if *step == 0 { + self.line_no == *first + } else if *first == 0 { + self.line_no.is_multiple_of(*step) + } else { + self.line_no >= *first && (self.line_no - *first).is_multiple_of(*step) + } + } + StartAddr::Regex(re) => { + let re = self.resolve(re); + re.is_some_and(|re| re.is_match(&self.ps)) + } + } + } + + // ----- substitution ---------------------------------------------------- + + fn substitute(&mut self, subst: &Subst) { + let Some(re) = self.resolve(&subst.re) else { + return; + }; + let spans = re.matches(&self.ps); + if spans.len() < subst.occurrence { + return; + } + + let mut out = String::with_capacity(self.ps.len()); + let mut cursor = 0usize; + let mut replaced = false; + + for (index, span) in spans.iter().enumerate() { + let nth = index + 1; + if nth < subst.occurrence || (nth > subst.occurrence && !subst.global) { + continue; + } + out.push_str(&self.ps[cursor..span.start]); + expand(&subst.parts, &self.ps, span, &mut out); + cursor = span.end; + replaced = true; + } + + if !replaced { + return; + } + out.push_str(&self.ps[cursor..]); + self.ps = out; + self.sub_made = true; + + if subst.print { + let (ps, nl) = (std::mem::take(&mut self.ps), self.ps_had_newline); + self.out.line(&ps, nl); + self.ps = ps; + } + if let Some(name) = &subst.wfile { + let name = name.clone(); + let (ps, nl) = (std::mem::take(&mut self.ps), self.ps_had_newline); + self.write_out(&name, &ps, nl); + self.ps = ps; + } + } +} + +#[derive(Default)] +struct CaseState { + span: Option, + one: Option, +} + +fn push_cased(out: &mut String, text: &str, state: &mut CaseState) { + for ch in text.chars() { + if let Some(upper) = state.one.take() { + if upper { + out.extend(ch.to_uppercase()); + } else { + out.extend(ch.to_lowercase()); + } + continue; + } + match state.span { + Some(true) => out.extend(ch.to_uppercase()), + Some(false) => out.extend(ch.to_lowercase()), + None => out.push(ch), + } + } +} + +fn expand(parts: &[RepPart], subject: &str, span: &super::pattern::MatchSpan, out: &mut String) { + let mut state = CaseState::default(); + for part in parts { + match part { + RepPart::Lit(text) => push_cased(out, text, &mut state), + RepPart::Group(0) => push_cased(out, &subject[span.start..span.end], &mut state), + RepPart::Group(n) => { + if let Some(Some((s, e))) = span.groups.get(n - 1) { + let text = subject[*s..*e].to_string(); + push_cased(out, &text, &mut state); + } + } + RepPart::Case(op) => match op { + CaseOp::Upper => { + state.span = Some(true); + state.one = None; + } + CaseOp::Lower => { + state.span = Some(false); + state.one = None; + } + CaseOp::UpperOne => state.one = Some(true), + CaseOp::LowerOne => state.one = Some(false), + CaseOp::End => { + state.span = None; + state.one = None; + } + }, + } + } +} + +/// Render the pattern space the way `l` does: unambiguous escapes, `$` +/// terminator, and wrapping with a trailing backslash at `width` columns. +pub(super) fn list_format(text: &str, width: usize) -> String { + let mut out = String::new(); + let mut column = 0usize; + + for ch in text.chars() { + let token = match ch { + '\\' => "\\\\".to_string(), + '\x07' => "\\a".to_string(), + '\x08' => "\\b".to_string(), + '\x0c' => "\\f".to_string(), + '\n' => "\\n".to_string(), + '\r' => "\\r".to_string(), + '\t' => "\\t".to_string(), + '\x0b' => "\\v".to_string(), + c if c.is_ascii_graphic() || c == ' ' => c.to_string(), + c => { + let mut buf = [0u8; 4]; + c.encode_utf8(&mut buf) + .as_bytes() + .iter() + .map(|b| format!("\\{b:03o}")) + .collect() + } + }; + if width > 1 && column + token.len() > width - 1 { + out.push_str("\\\n"); + column = 0; + } + column += token.len(); + out.push_str(&token); + } + + out.push('$'); + out +} diff --git a/crates/bashkit/src/builtins/sed/mod.rs b/crates/bashkit/src/builtins/sed/mod.rs new file mode 100644 index 000000000..6060711e0 --- /dev/null +++ b/crates/bashkit/src/builtins/sed/mod.rs @@ -0,0 +1,487 @@ +//! sed - stream editor builtin. +//! +//! Important decisions (issue #2427): +//! +//! - Option parsing is a real getopt: short options cluster (`-ne`), take +//! attached or separate arguments (`-e`, `-f`, `-l`), `-i` takes an *attached* +//! suffix only, `--` ends option parsing, and the long spellings are +//! accepted. Whether the first operand is the script or a file is decided +//! after the scan, exactly as GNU does. +//! - All script fragments (`-e`, `-f`, the bare operand) are concatenated with +//! newlines and compiled once, so multi-line `a\` text and `#` comments work. +//! - Without `-s`/`-i` every operand is one continuous stream: line numbers keep +//! counting, `$` is the last line of the last file, and `q` stops everything. +//! - `-i` rewrites through [`crate::builtins::atomic_write::atomic_replace`], +//! which preserves the file mode and leaves the original intact on failure +//! (THREAT[TM-FS-016]), and refuses non-UTF-8 input rather than corrupting it. + +mod exec; +mod pattern; +mod script; + +#[cfg(test)] +mod tests; + +use std::collections::HashMap; +use std::path::PathBuf; + +use async_trait::async_trait; + +use exec::{InputLine, Machine}; + +use super::atomic_write::{AtomicFailpoints, atomic_replace}; +use super::{Builtin, Context}; +use crate::error::Result; +use crate::fs::vfs_join; +use crate::interpreter::ExecResult; + +const HELP: &str = "Usage: sed [OPTION]... {script} [FILE]...\n\ +Stream editor for filtering and transforming text.\n\n \ +-n, --quiet, --silent\tsuppress automatic printing of pattern space\n \ +-e script, --expression=script\n\t\tadd the script to the commands to be executed\n \ +-f file, --file=file\tadd the contents of script-file to the commands\n \ +-i[SUFFIX], --in-place[=SUFFIX]\n\t\tedit files in place (makes backup if SUFFIX supplied)\n \ +-E, -r, --regexp-extended\n\t\tuse extended regular expressions\n \ +-s, --separate\tconsider files as separate rather than as a single stream\n \ +-z, --null-data\tseparate lines by NUL characters\n \ +-l N, --line-length=N\tspecify the desired line-wrap length for the `l' command\n \ +-u, --unbuffered\taccepted for compatibility (no effect)\n \ +--help\t\tdisplay this help and exit\n \ +--version\toutput version information and exit\n"; + +/// The `l` command wraps here unless `-l` or an explicit argument says otherwise. +const DEFAULT_LINE_LENGTH: usize = 70; + +/// sed command - stream editor +pub struct Sed; + +enum ScriptSource { + Text(String), + File(String), +} + +struct Options { + sources: Vec, + files: Vec, + /// `Some(suffix)`; an empty suffix means no backup copy. + in_place: Option, + quiet: bool, + extended: bool, + separate: bool, + null_data: bool, + line_length: usize, +} + +fn usage_error(message: impl std::fmt::Display) -> Box { + Box::new(ExecResult::err(format!("sed: {message}\n"), 1)) +} + +impl Options { + fn parse(args: &[String]) -> std::result::Result> { + let mut opts = Options { + sources: Vec::new(), + files: Vec::new(), + in_place: None, + quiet: false, + extended: false, + separate: false, + null_data: false, + line_length: DEFAULT_LINE_LENGTH, + }; + let mut operands: Vec = Vec::new(); + let mut end_of_options = false; + let mut i = 0; + + while i < args.len() { + let arg = &args[i]; + if end_of_options || arg == "-" || !arg.starts_with('-') { + operands.push(arg.clone()); + i += 1; + continue; + } + if arg == "--" { + end_of_options = true; + i += 1; + continue; + } + if let Some(long) = arg.strip_prefix("--") { + let (name, value) = match long.split_once('=') { + Some((n, v)) => (n, Some(v.to_string())), + None => (long, None), + }; + match name { + "quiet" | "silent" => opts.quiet = true, + "regexp-extended" => opts.extended = true, + "separate" => opts.separate = true, + "null-data" | "zero-terminated" => opts.null_data = true, + "unbuffered" | "debug" | "posix" | "sandbox" | "follow-symlinks" => {} + "in-place" => opts.in_place = Some(value.unwrap_or_default()), + "expression" => { + let value = take_value(args, &mut i, value, arg)?; + opts.sources.push(ScriptSource::Text(value)); + } + "file" => { + let value = take_value(args, &mut i, value, arg)?; + opts.sources.push(ScriptSource::File(value)); + } + "line-length" => { + let value = take_value(args, &mut i, value, arg)?; + opts.line_length = parse_length(&value)?; + } + _ => return Err(Box::new(super::invalid_option("sed", arg, 1))), + } + i += 1; + continue; + } + + let chars: Vec = arg.chars().skip(1).collect(); + let mut j = 0; + while j < chars.len() { + match chars[j] { + 'n' => opts.quiet = true, + 'r' | 'E' => opts.extended = true, + 's' => opts.separate = true, + 'z' => opts.null_data = true, + 'u' => {} + 'e' | 'f' | 'l' => { + let flag = chars[j]; + let attached: String = chars[j + 1..].iter().collect(); + let value = if attached.is_empty() { + i += 1; + match args.get(i) { + Some(v) => v.clone(), + None => { + return Err(usage_error(format!( + "option requires an argument -- '{flag}'" + ))); + } + } + } else { + attached + }; + match flag { + 'e' => opts.sources.push(ScriptSource::Text(value)), + 'f' => opts.sources.push(ScriptSource::File(value)), + _ => opts.line_length = parse_length(&value)?, + } + j = chars.len(); + continue; + } + 'i' => { + // GNU only accepts an *attached* suffix, so that + // `sed -i 's/a/b/' f` edits f rather than creating `s/a/b/`. + opts.in_place = Some(chars[j + 1..].iter().collect()); + j = chars.len(); + continue; + } + _ => { + let token = format!("-{}", chars[j]); + return Err(Box::new(super::invalid_option("sed", &token, 1))); + } + } + j += 1; + } + i += 1; + } + + // GNU decides script-vs-file only after the whole scan: any -e/-f makes + // every operand a file name. + let mut operands = operands.into_iter(); + if opts.sources.is_empty() { + match operands.next() { + Some(script) => opts.sources.push(ScriptSource::Text(script)), + None => { + return Err(Box::new(ExecResult::err( + format!("Usage: sed [OPTION]... {{script}} [FILE]...\n{HELP}"), + 1, + ))); + } + } + } + opts.files.extend(operands); + + if opts.in_place.is_some() { + if opts.files.is_empty() { + return Err(Box::new(ExecResult::err( + "sed: no input files\n".to_string(), + 4, + ))); + } + // -i implies -s: each file is its own stream. + opts.separate = true; + } + Ok(opts) + } +} + +fn take_value( + args: &[String], + i: &mut usize, + inline: Option, + arg: &str, +) -> std::result::Result> { + if let Some(v) = inline { + return Ok(v); + } + *i += 1; + args.get(*i) + .cloned() + .ok_or_else(|| usage_error(format!("option '{arg}' requires an argument"))) +} + +fn parse_length(value: &str) -> std::result::Result> { + value + .parse::() + .map_err(|_| usage_error(format!("invalid line length: {value}"))) +} + +/// Split input into lines, remembering whether the final line was terminated. +fn split_lines(content: &str, sep: char, file: usize, out: &mut Vec) { + if content.is_empty() { + return; + } + let mut rest = content; + while let Some(idx) = rest.find(sep) { + out.push(InputLine { + text: rest[..idx].to_string(), + had_newline: true, + file, + }); + rest = &rest[idx + sep.len_utf8()..]; + } + if !rest.is_empty() { + out.push(InputLine { + text: rest.to_string(), + had_newline: false, + file, + }); + } +} + +fn resolve(cwd: &std::path::Path, name: &str) -> PathBuf { + if name.starts_with('/') { + PathBuf::from(name) + } else { + vfs_join(cwd, name) + } +} + +#[async_trait] +impl Builtin for Sed { + async fn execute(&self, ctx: Context<'_>) -> Result { + if let Some(r) = super::check_help_version(ctx.args, HELP, Some("sed (bashkit) 0.1")) { + return Ok(r); + } + + let opts = match Options::parse(ctx.args) { + Ok(o) => o, + Err(e) => return Ok(*e), + }; + + // Assemble the whole script before compiling it: GNU joins every -e and + // -f fragment with newlines and parses the result as one program. + let mut script = String::new(); + for source in &opts.sources { + if !script.is_empty() { + script.push('\n'); + } + match source { + ScriptSource::Text(text) => script.push_str(text), + ScriptSource::File(name) => { + if name == "-" { + script.push_str( + ctx.stdin + .map(ToString::to_string) + .unwrap_or_default() + .as_str(), + ); + continue; + } + let path = resolve(ctx.cwd, name); + match ctx.fs.read_file(&path).await { + Ok(bytes) => { + script.push_str(String::from_utf8_lossy(&bytes).trim_end_matches('\n')) + } + // GNU exits 4 when a script file cannot be opened. + Err(e) => { + return Ok(ExecResult::err( + format!("sed: couldn't open file {name}: {e}\n"), + 4, + )); + } + } + } + } + } + + let program = match script::parse(&script, opts.extended) { + Ok(p) => p, + Err(e) => return Ok(ExecResult::err(format!("sed: {}\n", e.message), e.code)), + }; + + let sep = if opts.null_data { '\0' } else { '\n' }; + + // `r`/`R` operands are read up front; a missing file is silently ignored + // exactly as in GNU sed. + let mut read_files: HashMap = HashMap::new(); + for name in &program.read_files { + if read_files.contains_key(name) { + continue; + } + let text = if name == "/dev/stdin" || name == "-" { + ctx.stdin.map(ToString::to_string).unwrap_or_default() + } else { + let path = resolve(ctx.cwd, name); + match ctx.fs.read_file(&path).await { + Ok(bytes) => String::from_utf8_lossy(&bytes).into_owned(), + Err(_) => continue, + } + }; + read_files.insert(name.clone(), text); + } + + // Gather input. + let names: Vec = if opts.files.is_empty() { + vec!["-".to_string()] + } else { + opts.files.clone() + }; + let mut lines: Vec = Vec::new(); + let mut raw: Vec = Vec::new(); + for (index, name) in names.iter().enumerate() { + let content = if name == "-" || opts.files.is_empty() { + ctx.stdin.map(ToString::to_string).unwrap_or_default() + } else { + let path = resolve(ctx.cwd, name); + match ctx.fs.read_file(&path).await { + Ok(bytes) => match String::from_utf8(bytes) { + Ok(text) => text, + Err(e) => { + if opts.in_place.is_some() { + return Ok(ExecResult::err( + format!( + "sed: couldn't edit {name}: not a valid UTF-8 text file\n" + ), + 1, + )); + } + String::from_utf8_lossy(e.as_bytes()).into_owned() + } + }, + // GNU exits 2 when an input file cannot be read. + Err(e) => { + return Ok(ExecResult::err(format!("sed: can't read {name}: {e}\n"), 2)); + } + } + }; + split_lines(&content, sep, index, &mut lines); + raw.push(content); + } + + // Segment the stream: one segment overall, or one per operand under + // `-s`/`-i`. Built before the machine so it can borrow the slices. + let segments: Vec> = if opts.separate { + (0..names.len()) + .map(|index| { + lines + .iter() + .filter(|l| l.file == index) + .map(|l| InputLine { + text: l.text.clone(), + had_newline: l.had_newline, + file: l.file, + }) + .collect() + }) + .collect() + } else { + vec![lines] + }; + + let mut machine = Machine::new( + &program, + &names, + &read_files, + opts.quiet, + opts.line_length, + sep, + ); + + let mut stdout = String::new(); + let mut edits: Vec<(usize, String, String)> = Vec::new(); + + for (index, segment) in segments.iter().enumerate() { + if machine.finished() { + break; + } + let produced = machine.run_segment(segment); + let name = names.get(index).map(String::as_str).unwrap_or("-"); + if opts.separate && opts.in_place.is_some() && name != "-" { + edits.push((index, name.to_string(), produced)); + } else { + stdout.push_str(&produced); + } + } + + let mut stderr = std::mem::take(&mut machine.stderr); + let exit_code = machine.exit_code.unwrap_or(0); + let write_files = std::mem::take(&mut machine.write_files); + drop(machine); + + // Files named by `w`/`W`/`s///w`. + for (name, sink) in write_files { + let path = resolve(ctx.cwd, &name); + if let Err(e) = ctx.fs.write_file(&path, sink.buf.as_bytes()).await { + return Ok(ExecResult::err( + format!("sed: couldn't open {name}: {e}\n"), + 4, + )); + } + } + + // In-place rewrites. + for (index, name, content) in edits { + let path = resolve(ctx.cwd, &name); + if let Some(suffix) = opts.in_place.as_deref().filter(|s| !s.is_empty()) { + let backup = if suffix.contains('*') { + suffix.replace('*', &name) + } else { + format!("{name}{suffix}") + }; + let backup_path = resolve(ctx.cwd, &backup); + let original = raw.get(index).cloned().unwrap_or_default(); + if let Err(e) = ctx.fs.write_file(&backup_path, original.as_bytes()).await { + return Ok(ExecResult::err( + format!("sed: cannot rename {name}: {e}\n"), + 4, + )); + } + } + if let Err(e) = atomic_replace( + &*ctx.fs, + &path, + content.as_bytes(), + "sed", + AtomicFailpoints { + allocate: "sed::temp_allocate", + chmod: "sed::temp_chmod", + rename: "sed::temp_rename", + }, + ) + .await + { + return Ok(ExecResult::err(format!("sed: {name}: {e}\n"), 4)); + } + } + + if stderr.len() > 1024 { + stderr.truncate(1024); + } + Ok(ExecResult { + stdout: stdout.into(), + stderr: stderr.into(), + exit_code, + ..Default::default() + }) + } +} diff --git a/crates/bashkit/src/builtins/sed/pattern.rs b/crates/bashkit/src/builtins/sed/pattern.rs new file mode 100644 index 000000000..6913e8f40 --- /dev/null +++ b/crates/bashkit/src/builtins/sed/pattern.rs @@ -0,0 +1,369 @@ +//! POSIX regex translation and matching for `sed`. +//! +//! Important decisions (issue #2427, finding B): +//! +//! - BRE and ERE are translated into the `regex` crate dialect by a single +//! [`translate`] pass used for **both** `s///` patterns and address regexes. +//! Previously address regexes skipped conversion entirely and were compiled +//! as ERE, so `/a\+/` never matched while `s/a\+//` did. +//! - Whether `+ ? | * ^ $` are operators or literals is positional in BRE, so +//! the translator tracks "start of a branch" state rather than doing string +//! replacements. `s/a+b/`, `s/a?b/`, `s/a|b/`, `/a^b/` and `s/*a/` are all +//! literal matches in BRE, exactly as GNU sed treats them. +//! - Bracket expressions are copied by a dedicated scanner: inside `[...]` +//! POSIX gives `\` no special meaning, while the `regex` crate treats `\`, +//! `[`, `&&`, `~~` and `--` as syntax, so those are escaped on the way out. + +use regex::{Regex, RegexBuilder}; + +use crate::builtins::search_common::{REGEX_DFA_SIZE_LIMIT, REGEX_SIZE_LIMIT}; + +/// Translate a POSIX BRE (or ERE, when `extended`) into `regex` crate syntax. +pub(super) fn translate(pattern: &str, extended: bool) -> String { + let chars: Vec = pattern.chars().collect(); + let mut out = String::with_capacity(pattern.len() + 8); + let mut i = 0; + // True when the next token begins an alternation branch: `^` anchors there + // and (BRE only) a `*` there is an ordinary character. + let mut branch_start = true; + + while i < chars.len() { + let ch = chars[i]; + match ch { + '\\' if i + 1 < chars.len() => { + let next = chars[i + 1]; + i += 2; + match next { + // In BRE the backslashed forms are the operators. + '(' | ')' | '{' | '}' | '+' | '?' | '|' if !extended => { + out.push(next); + branch_start = matches!(next, '(' | '|'); + } + // Elsewhere a backslashed punctuation character is literal. + '(' | ')' | '{' | '}' | '+' | '?' | '|' | '.' | '*' | '[' | ']' | '^' | '$' + | '\\' | '/' => { + push_literal(&mut out, next); + branch_start = false; + } + 'n' => push_raw(&mut out, '\n', &mut branch_start), + 't' => push_raw(&mut out, '\t', &mut branch_start), + 'r' => push_raw(&mut out, '\r', &mut branch_start), + 'f' => push_raw(&mut out, '\x0c', &mut branch_start), + 'v' => push_raw(&mut out, '\x0b', &mut branch_start), + 'a' => push_raw(&mut out, '\x07', &mut branch_start), + // GNU character-class and word-boundary escapes. + 'w' | 'W' | 's' | 'S' | 'b' | 'B' => { + out.push('\\'); + out.push(next); + branch_start = false; + } + // GNU word-edge operators; the `regex` crate has no + // look-around, so both collapse onto `\b`. + '<' | '>' => { + out.push_str("\\b"); + branch_start = false; + } + '`' => { + out.push_str("\\A"); + branch_start = false; + } + '\'' => { + out.push_str("\\z"); + branch_start = false; + } + // Back-references: passed through for the fancy-regex path. + '1'..='9' => { + out.push('\\'); + out.push(next); + branch_start = false; + } + _ => { + push_literal(&mut out, next); + branch_start = false; + } + } + } + // Trailing lone backslash. + '\\' => { + out.push_str("\\\\"); + i += 1; + branch_start = false; + } + '[' => { + i = copy_bracket(&chars, i, &mut out); + branch_start = false; + } + '^' => { + if extended || branch_start { + out.push('^'); + } else { + out.push_str("\\^"); + } + i += 1; + // `^*` keeps `*` literal in BRE. + branch_start = !extended; + } + '$' => { + if extended || is_bre_dollar_anchor(&chars, i) { + out.push('$'); + } else { + out.push_str("\\$"); + } + i += 1; + branch_start = false; + } + '*' => { + if !extended && branch_start { + out.push_str("\\*"); + } else { + out.push('*'); + } + i += 1; + branch_start = false; + } + '+' | '?' | '|' | '(' | ')' | '{' | '}' if !extended => { + push_literal(&mut out, ch); + i += 1; + branch_start = false; + } + '(' | '|' if extended => { + out.push(ch); + i += 1; + branch_start = true; + } + _ => { + out.push(ch); + i += 1; + branch_start = false; + } + } + } + + out +} + +fn push_raw(out: &mut String, ch: char, branch_start: &mut bool) { + push_literal(out, ch); + *branch_start = false; +} + +/// Emit `ch` so the `regex` crate reads it as an ordinary character. +fn push_literal(out: &mut String, ch: char) { + if ch.is_ascii_alphanumeric() || !ch.is_ascii() { + out.push(ch); + } else { + out.push('\\'); + out.push(ch); + } +} + +/// In BRE, `$` anchors only at the very end of the expression or immediately +/// before `\)` / `\|`; anywhere else it is an ordinary character. +fn is_bre_dollar_anchor(chars: &[char], i: usize) -> bool { + match chars.get(i + 1) { + None => true, + Some('\\') => matches!(chars.get(i + 2), Some(')') | Some('|')), + Some(_) => false, + } +} + +/// Copy a POSIX bracket expression starting at `chars[start] == '['`. +/// Returns the index just past the closing `]`. +fn copy_bracket(chars: &[char], start: usize, out: &mut String) -> usize { + let mut i = start + 1; + let negated = matches!(chars.get(i), Some('^')); + if negated { + i += 1; + } + + let mut content = String::new(); + let mut seen = 0usize; + let mut closed = false; + + while i < chars.len() { + let ch = chars[i]; + if ch == ']' && seen > 0 { + closed = true; + i += 1; + break; + } + if ch == '[' + && let Some(&kind @ (':' | '.' | '=')) = chars.get(i + 1) + && let Some(end) = find_class_end(chars, i + 2, kind) + { + if kind == ':' { + // [:alpha:] and friends are understood by the regex crate. + content.extend(&chars[i..=end + 1]); + } else { + // [.x.] / [=x=] degrade to the literal characters inside. + for &c in &chars[i + 2..end] { + push_class_literal(&mut content, c); + } + } + i = end + 2; + seen += 1; + continue; + } + push_class_literal(&mut content, ch); + i += 1; + seen += 1; + } + + if !closed { + // Unterminated: hand the raw text to the regex crate so it reports the + // error instead of silently matching something else. + out.push('['); + if negated { + out.push('^'); + } + out.push_str(&content); + return chars.len(); + } + + out.push('['); + if negated { + out.push('^'); + } + out.push_str(&content); + out.push(']'); + i +} + +fn find_class_end(chars: &[char], from: usize, kind: char) -> Option { + let mut j = from; + while j + 1 < chars.len() { + if chars[j] == kind && chars[j + 1] == ']' { + return Some(j); + } + j += 1; + } + None +} + +/// Escape the characters the `regex` crate treats as class syntax but POSIX +/// treats as ordinary members of a bracket expression. +fn push_class_literal(out: &mut String, ch: char) { + match ch { + '\\' | '[' | ']' | '&' | '~' => { + out.push('\\'); + out.push(ch); + } + _ => out.push(ch), + } +} + +/// A compiled sed regex. Falls back to `fancy_regex` when the pattern uses +/// back-references, which the default engine rejects. +pub(super) enum SedRegex { + Standard(Regex), + Fancy(fancy_regex::Regex), +} + +/// Backtracking step cap for the fancy-regex fallback. +const FANCY_BACKTRACK_LIMIT: usize = 1_000_000; + +/// One match with its capture group spans (byte offsets into the subject). +pub(super) struct MatchSpan { + pub(super) start: usize, + pub(super) end: usize, + pub(super) groups: Vec>, +} + +impl SedRegex { + pub(super) fn new( + pattern: &str, + extended: bool, + case_insensitive: bool, + multi_line: bool, + ) -> std::result::Result { + let translated = translate(pattern, extended); + match RegexBuilder::new(&translated) + .case_insensitive(case_insensitive) + .multi_line(multi_line) + .size_limit(REGEX_SIZE_LIMIT) + .dfa_size_limit(REGEX_DFA_SIZE_LIMIT) + .build() + { + Ok(re) => Ok(SedRegex::Standard(re)), + Err(e) => { + let message = e.to_string(); + if message.contains("backreference") { + Self::build_fancy(&translated, case_insensitive, multi_line) + .map_err(|_| message) + } else { + Err(message) + } + } + } + } + + pub(super) fn build_fancy( + pattern: &str, + case_insensitive: bool, + multi_line: bool, + ) -> std::result::Result { + Self::build_fancy_with_limit(pattern, case_insensitive, multi_line, FANCY_BACKTRACK_LIMIT) + } + + pub(super) fn build_fancy_with_limit( + pattern: &str, + case_insensitive: bool, + multi_line: bool, + backtrack_limit: usize, + ) -> std::result::Result { + fancy_regex::RegexBuilder::new(pattern) + .case_insensitive(case_insensitive) + .multi_line(multi_line) + .delegate_size_limit(REGEX_SIZE_LIMIT) + .delegate_dfa_size_limit(REGEX_DFA_SIZE_LIMIT) + .backtrack_limit(backtrack_limit) + .build() + .map(SedRegex::Fancy) + .map_err(|e| e.to_string()) + } + + pub(super) fn is_match(&self, text: &str) -> bool { + match self { + SedRegex::Standard(re) => re.is_match(text), + SedRegex::Fancy(re) => re.is_match(text).unwrap_or(false), + } + } + + /// All non-overlapping matches, with capture spans. + pub(super) fn matches(&self, text: &str) -> Vec { + let mut spans = Vec::new(); + match self { + SedRegex::Standard(re) => { + for caps in re.captures_iter(text) { + let whole = match caps.get(0) { + Some(m) => m, + None => continue, + }; + spans.push(MatchSpan { + start: whole.start(), + end: whole.end(), + groups: (1..caps.len()) + .map(|i| caps.get(i).map(|m| (m.start(), m.end()))) + .collect(), + }); + } + } + SedRegex::Fancy(re) => { + for caps in re.captures_iter(text).flatten() { + let whole = match caps.get(0) { + Some(m) => m, + None => continue, + }; + spans.push(MatchSpan { + start: whole.start(), + end: whole.end(), + groups: (1..caps.len()) + .map(|i| caps.get(i).map(|m| (m.start(), m.end()))) + .collect(), + }); + } + } + } + spans + } +} diff --git a/crates/bashkit/src/builtins/sed/script.rs b/crates/bashkit/src/builtins/sed/script.rs new file mode 100644 index 000000000..e66502509 --- /dev/null +++ b/crates/bashkit/src/builtins/sed/script.rs @@ -0,0 +1,869 @@ +//! sed script parser. +//! +//! Important decisions (issue #2427): +//! +//! - The whole script (every `-e`, every `-f`, the bare operand) is joined with +//! newlines and parsed **once**, so multi-line `a\` text, `#` comments and the +//! `#n` first-line directive work the way GNU sed defines them. +//! - The program is a **flat** instruction list: `{` compiles to a [`Kind::Block`] +//! carrying the index of its `}`. Branch targets can therefore sit inside or +//! outside a block, which a tree of nested groups cannot express. +//! - Every regex (substitution or address) goes through +//! [`super::pattern::SedRegex`], so BRE/ERE translation happens in exactly one +//! place. +//! - Parse errors are reported GNU-style (`-e expression #1, char N: ...`) and +//! surface as exit status 1 with a stderr message, never as an interpreter +//! error. + +use std::sync::Arc; + +use super::pattern::SedRegex; +use crate::builtins::limits::SED_MAX_GROUP_NESTING_DEPTH as MAX_GROUP_NESTING_DEPTH; + +/// Longest diagnostic a parse error may produce (TM-INF-022 caps stderr). +const MAX_DIAGNOSTIC: usize = 512; + +/// A script that failed to compile. GNU reports expression errors with the +/// offending character position and exit status 1, and whole-script problems +/// (an unresolvable branch target) with status 4. +pub(super) struct ParseError { + pub(super) message: String, + pub(super) code: i32, +} + +pub(super) type ParseResult = std::result::Result; + +/// One piece of an `s///` replacement. +pub(super) enum RepPart { + Lit(String), + /// Capture group; `0` is the whole match (`&`). + Group(usize), + /// `\U`, `\L`, `\u`, `\l`, `\E` + Case(CaseOp), +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(super) enum CaseOp { + Upper, + Lower, + UpperOne, + LowerOne, + End, +} + +pub(super) struct Subst { + /// `None` means the empty regex `//`: reuse the last regex applied. + pub(super) re: Option>, + pub(super) parts: Vec, + /// 1-based index of the first occurrence to replace. + pub(super) occurrence: usize, + /// Replace `occurrence` and everything after it. + pub(super) global: bool, + pub(super) print: bool, + pub(super) wfile: Option, +} + +pub(super) enum Kind { + /// `{` — index of the matching `}`; skipped past when the address misses. + Block(usize), + BlockEnd, + Label(String), + Branch(Option), + BranchIfSub(Option), + BranchIfNoSub(Option), + Substitute(Box), + Transliterate(Vec<(char, char)>), + Delete, + DeleteFirstLine, + Print, + PrintFirstLine, + Next, + NextAppend, + Quit(i32), + QuitSilent(i32), + Append(String), + Insert(String), + Change(String), + HoldCopy, + HoldAppend, + GetCopy, + GetAppend, + Exchange, + LineNumber, + List(Option), + ReadFile(String), + ReadLine(String), + WriteFile(String), + WriteFirstLine(String), + Zap, + FileName, + Nop, +} + +pub(super) enum StartAddr { + Line(usize), + Last, + Regex(Option>), + Step(usize, usize), + /// `0` — only legal as the start of `0,/re/`. + Zero, +} + +pub(super) enum EndAddr { + Line(usize), + Last, + Regex(Option>), + /// `addr,+N` + Plus(usize), + /// `addr,~N` + Multiple(usize), +} + +pub(super) struct Addr { + pub(super) start: StartAddr, + pub(super) end: Option, +} + +pub(super) struct Cmd { + pub(super) addr: Option, + pub(super) negate: bool, + pub(super) kind: Kind, +} + +pub(super) struct Program { + pub(super) cmds: Vec, + /// `#n` on the very first line implies `-n`. + pub(super) quiet: bool, + /// Files named by `r`/`R`, which are read before execution starts. + pub(super) read_files: Vec, +} + +struct Parser { + chars: Vec, + pos: usize, + extended: bool, + regex_seen: bool, +} + +pub(super) fn parse(script: &str, extended: bool) -> ParseResult { + let mut parser = Parser { + chars: script.chars().collect(), + pos: 0, + extended, + regex_seen: false, + }; + parser.program() +} + +impl Parser { + /// GNU counts the characters consumed so far, so the position is + /// `self.pos` (already one past the offending token), not `pos + 1`. + fn err(&self, msg: impl std::fmt::Display) -> ParseResult { + Err(ParseError { + message: clamp(format!("-e expression #1, char {}: {msg}", self.pos)), + code: 1, + }) + } + + fn peek(&self) -> Option { + self.chars.get(self.pos).copied() + } + + fn at(&self, offset: usize) -> Option { + self.chars.get(self.pos + offset).copied() + } + + fn bump(&mut self) -> Option { + let c = self.peek(); + if c.is_some() { + self.pos += 1; + } + c + } + + fn eat(&mut self, ch: char) -> bool { + if self.peek() == Some(ch) { + self.pos += 1; + true + } else { + false + } + } + + fn skip_blanks(&mut self) { + while matches!(self.peek(), Some(' ') | Some('\t')) { + self.pos += 1; + } + } + + fn skip_separators(&mut self) { + while matches!( + self.peek(), + Some(' ') | Some('\t') | Some('\n') | Some('\r') | Some(';') + ) { + self.pos += 1; + } + } + + fn rest_of_line(&mut self) -> String { + let start = self.pos; + while !matches!(self.peek(), None | Some('\n')) { + self.pos += 1; + } + self.chars[start..self.pos].iter().collect() + } + + fn program(&mut self) -> ParseResult { + let mut quiet = false; + let mut read_files = Vec::new(); + + // `#n` as the first two characters of the script implies -n. + if self.chars.first() == Some(&'#') { + if self.at(1) == Some('n') && matches!(self.at(2), None | Some('\n')) { + quiet = true; + } + self.rest_of_line(); + } + + let mut cmds: Vec = Vec::new(); + let mut open_blocks: Vec = Vec::new(); + + loop { + self.skip_separators(); + let Some(ch) = self.peek() else { break }; + + if ch == '#' { + self.rest_of_line(); + continue; + } + + if ch == '}' { + self.pos += 1; + let Some(open) = open_blocks.pop() else { + return self.err("unexpected `}'"); + }; + let end = cmds.len(); + if let Kind::Block(slot) = &mut cmds[open].kind { + *slot = end; + } + cmds.push(Cmd { + addr: None, + negate: false, + kind: Kind::BlockEnd, + }); + continue; + } + + let addr = self.address()?; + self.skip_blanks(); + let mut negate = false; + while self.eat('!') { + negate = true; + self.skip_blanks(); + } + + let Some(cmd_char) = self.peek() else { + return self.err("missing command"); + }; + + if cmd_char == '{' { + self.pos += 1; + if open_blocks.len() + 1 > MAX_GROUP_NESTING_DEPTH { + return self.err(format!( + "grouped command nesting exceeds max depth {MAX_GROUP_NESTING_DEPTH}" + )); + } + open_blocks.push(cmds.len()); + cmds.push(Cmd { + addr, + negate, + kind: Kind::Block(usize::MAX), + }); + continue; + } + + let kind = self.command(cmd_char, addr.is_some())?; + match &kind { + Kind::ReadFile(f) | Kind::ReadLine(f) => read_files.push(f.clone()), + _ => {} + } + cmds.push(Cmd { addr, negate, kind }); + } + + if !open_blocks.is_empty() { + return self.err("unmatched `{'"); + } + + // GNU resolves branch targets at compile time; an unknown label is an + // error rather than a silent fall-through. + for cmd in &cmds { + let target = match &cmd.kind { + Kind::Branch(Some(l)) + | Kind::BranchIfSub(Some(l)) + | Kind::BranchIfNoSub(Some(l)) => l, + _ => continue, + }; + let known = cmds + .iter() + .any(|c| matches!(&c.kind, Kind::Label(name) if name == target)); + if !known { + return Err(ParseError { + message: clamp(format!("can't find label for jump to `{target}'")), + code: 4, + }); + } + } + + Ok(Program { + cmds, + quiet, + read_files, + }) + } + + // ----- addresses ------------------------------------------------------- + + fn address(&mut self) -> ParseResult> { + self.skip_blanks(); + let Some(ch) = self.peek() else { + return Ok(None); + }; + + let start = match ch { + '$' => { + self.pos += 1; + StartAddr::Last + } + '/' => { + self.pos += 1; + StartAddr::Regex(self.address_regex('/')?) + } + '\\' => { + self.pos += 1; + let Some(delim) = self.bump() else { + return self.err("unterminated address regex"); + }; + StartAddr::Regex(self.address_regex(delim)?) + } + c if c.is_ascii_digit() => { + let n = self.number(); + if self.eat('~') { + let step = self.number(); + StartAddr::Step(n, step) + } else if n == 0 { + StartAddr::Zero + } else { + StartAddr::Line(n) + } + } + _ => return Ok(None), + }; + + self.skip_blanks(); + if !self.eat(',') { + if matches!(start, StartAddr::Zero) { + return self.err("invalid usage of line address 0"); + } + return Ok(Some(Addr { start, end: None })); + } + + self.skip_blanks(); + let end = match self.peek() { + Some('$') => { + self.pos += 1; + EndAddr::Last + } + Some('/') => { + self.pos += 1; + EndAddr::Regex(self.address_regex('/')?) + } + Some('\\') => { + self.pos += 1; + let Some(delim) = self.bump() else { + return self.err("unterminated address regex"); + }; + EndAddr::Regex(self.address_regex(delim)?) + } + Some('+') => { + self.pos += 1; + EndAddr::Plus(self.number()) + } + Some('~') => { + self.pos += 1; + EndAddr::Multiple(self.number()) + } + Some(c) if c.is_ascii_digit() => EndAddr::Line(self.number()), + _ => return self.err("unexpected `,'"), + }; + + if matches!(start, StartAddr::Zero) && !matches!(end, EndAddr::Regex(_)) { + return self.err("invalid usage of line address 0"); + } + + Ok(Some(Addr { + start, + end: Some(end), + })) + } + + fn number(&mut self) -> usize { + let mut n: usize = 0; + while let Some(c) = self.peek() { + let Some(d) = c.to_digit(10) else { break }; + n = n.saturating_mul(10).saturating_add(d as usize); + self.pos += 1; + } + n + } + + /// Read `regex` (the opening delimiter is already consumed) + /// plus the trailing `I`/`M` flags, and compile it. + fn address_regex(&mut self, delim: char) -> ParseResult>> { + let body = self.delimited(delim, "address regex", true)?; + let mut case_insensitive = false; + let mut multi_line = false; + loop { + match self.peek() { + Some('I') => { + case_insensitive = true; + self.pos += 1; + } + Some('M') => { + multi_line = true; + self.pos += 1; + } + _ => break, + } + } + self.compile(&body, case_insensitive, multi_line) + } + + fn compile( + &mut self, + body: &str, + case_insensitive: bool, + multi_line: bool, + ) -> ParseResult>> { + if body.is_empty() { + if !self.regex_seen { + return self.err("no previous regular expression"); + } + return Ok(None); + } + self.regex_seen = true; + match SedRegex::new(body, self.extended, case_insensitive, multi_line) { + Ok(re) => Ok(Some(Arc::new(re))), + Err(e) => self.err(format!("invalid pattern: {}", flatten(&e))), + } + } + + /// Consume characters up to the next unescaped `delim`, keeping backslash + /// escapes intact except `\`, which becomes a literal delimiter. + /// + /// `bracket_aware` is set while scanning a regex: inside a POSIX bracket + /// expression the delimiter is an ordinary character, so `s/[/]/X/` matches + /// a slash and `s/[//` is an unterminated command rather than an empty + /// bracket expression. + fn delimited(&mut self, delim: char, what: &str, bracket_aware: bool) -> ParseResult { + let mut out = String::new(); + while let Some(c) = self.bump() { + if bracket_aware && c == '[' { + out.push('['); + self.copy_bracket(&mut out, what)?; + continue; + } + if c == '\\' { + match self.bump() { + // `\` is the delimiter as an ordinary character. + Some(n) if n == delim => { + if n.is_ascii_alphanumeric() { + out.push(n); + } else { + out.push('\\'); + out.push(n); + } + } + Some('\n') => out.push('\n'), + Some(n) => { + out.push('\\'); + out.push(n); + } + None => return self.err(format!("unterminated {what}")), + } + continue; + } + if c == delim { + return Ok(out); + } + if c == '\n' && delim != '\n' { + break; + } + out.push(c); + } + self.err(format!("unterminated {what}")) + } + + /// Copy a bracket expression verbatim; the opening `[` is already emitted. + fn copy_bracket(&mut self, out: &mut String, what: &str) -> ParseResult<()> { + if self.peek() == Some('^') { + out.push('^'); + self.pos += 1; + } + if self.peek() == Some(']') { + out.push(']'); + self.pos += 1; + } + while let Some(c) = self.bump() { + out.push(c); + if c == ']' { + return Ok(()); + } + if c == '[' + && let Some(kind @ (':' | '.' | '=')) = self.peek() + { + out.push(kind); + self.pos += 1; + while let Some(inner) = self.bump() { + out.push(inner); + if inner == kind && self.peek() == Some(']') { + out.push(']'); + self.pos += 1; + break; + } + } + } + } + self.err(format!("unterminated {what}")) + } + + // ----- commands -------------------------------------------------------- + + fn command(&mut self, ch: char, has_addr: bool) -> ParseResult { + self.pos += 1; + match ch { + 's' => self.substitute(), + 'y' => self.transliterate(), + 'd' => Ok(Kind::Delete), + 'D' => Ok(Kind::DeleteFirstLine), + 'p' => Ok(Kind::Print), + 'P' => Ok(Kind::PrintFirstLine), + 'n' => Ok(Kind::Next), + 'N' => Ok(Kind::NextAppend), + 'h' => Ok(Kind::HoldCopy), + 'H' => Ok(Kind::HoldAppend), + 'g' => Ok(Kind::GetCopy), + 'G' => Ok(Kind::GetAppend), + 'x' => Ok(Kind::Exchange), + 'z' => Ok(Kind::Zap), + 'F' => Ok(Kind::FileName), + '=' => Ok(Kind::LineNumber), + 'q' => Ok(Kind::Quit(self.exit_code())), + 'Q' => Ok(Kind::QuitSilent(self.exit_code())), + 'l' => { + self.skip_blanks(); + let width = if matches!(self.peek(), Some(c) if c.is_ascii_digit()) { + Some(self.number()) + } else { + None + }; + Ok(Kind::List(width)) + } + 'a' => Ok(Kind::Append(self.text())), + 'i' => Ok(Kind::Insert(self.text())), + 'c' => Ok(Kind::Change(self.text())), + 'r' => Ok(Kind::ReadFile(self.filename())), + 'R' => Ok(Kind::ReadLine(self.filename())), + 'w' => Ok(Kind::WriteFile(self.filename())), + 'W' => Ok(Kind::WriteFirstLine(self.filename())), + ':' => { + if has_addr { + return self.err(": doesn't want any addresses"); + } + let label = self.label(); + if label.is_empty() { + return self.err("\":\" lacks a label"); + } + Ok(Kind::Label(label)) + } + 'b' => Ok(Kind::Branch(self.optional_label())), + 't' => Ok(Kind::BranchIfSub(self.optional_label())), + 'T' => Ok(Kind::BranchIfNoSub(self.optional_label())), + 'v' => { + self.rest_of_line(); + Ok(Kind::Nop) + } + 'e' => self.err( + "the `e' command is not available: Bashkit never executes host \ + commands from a sed script", + ), + _ => self.err(format!("unknown command: `{ch}'")), + } + } + + fn exit_code(&mut self) -> i32 { + self.skip_blanks(); + if matches!(self.peek(), Some(c) if c.is_ascii_digit()) { + (self.number() & 0xff) as i32 + } else { + 0 + } + } + + fn label(&mut self) -> String { + self.skip_blanks(); + let start = self.pos; + while !matches!(self.peek(), None | Some('\n') | Some(';') | Some('}')) { + self.pos += 1; + } + self.chars[start..self.pos] + .iter() + .collect::() + .trim() + .to_string() + } + + fn optional_label(&mut self) -> Option { + let label = self.label(); + if label.is_empty() { None } else { Some(label) } + } + + /// `r`/`w` take the rest of the line, `;` included, as the file name. + fn filename(&mut self) -> String { + self.skip_blanks(); + self.rest_of_line().trim_end().to_string() + } + + /// Text argument of `a`, `i` and `c`, in both the POSIX `a\` + newline form + /// and the GNU one-liner form. + fn text(&mut self) -> String { + self.skip_blanks(); + if self.eat('\\') { + self.eat('\n'); + } + let mut out = String::new(); + while let Some(c) = self.bump() { + if c == '\\' { + match self.bump() { + Some('\n') => out.push('\n'), + Some(n) => out.push(n), + None => break, + } + continue; + } + if c == '\n' { + break; + } + out.push(c); + } + out + } + + fn transliterate(&mut self) -> ParseResult { + let Some(delim) = self.bump() else { + return self.err("unterminated `y' command"); + }; + let src = unescape_y(&self.delimited(delim, "`y' command", false)?, delim); + let dst = unescape_y(&self.delimited(delim, "`y' command", false)?, delim); + if src.len() != dst.len() { + return self.err("strings for `y' command are different lengths"); + } + Ok(Kind::Transliterate( + src.into_iter().zip(dst).collect::>(), + )) + } + + fn substitute(&mut self) -> ParseResult { + let Some(delim) = self.bump() else { + return self.err("unterminated `s' command"); + }; + if delim == '\\' || delim == '\n' { + return self.err("unknown option to `s'"); + } + let pattern = self.delimited(delim, "`s' command", true)?; + let replacement = self.delimited(delim, "`s' command", false)?; + + let mut global = false; + let mut print = false; + let mut case_insensitive = false; + let mut multi_line = false; + let mut occurrence: Option = None; + let mut wfile = None; + + loop { + match self.peek() { + Some('g') => { + global = true; + self.pos += 1; + } + Some('p') => { + print = true; + self.pos += 1; + } + Some('i') | Some('I') => { + case_insensitive = true; + self.pos += 1; + } + Some('m') | Some('M') => { + multi_line = true; + self.pos += 1; + } + Some('e') => { + return self.err( + "the `e' option to `s' is not available: Bashkit never \ + executes host commands from a sed script", + ); + } + Some(c) if c.is_ascii_digit() => { + if occurrence.is_some() { + return self.err("multiple number options to `s' command"); + } + let n = self.number(); + if n == 0 { + return self.err("number option to `s' command may not be zero"); + } + occurrence = Some(n); + } + Some('w') => { + self.pos += 1; + wfile = Some(self.filename()); + break; + } + Some(' ') | Some('\t') => { + self.pos += 1; + } + None | Some('\n') | Some(';') | Some('}') | Some('#') | Some('\r') => break, + Some(c) => return self.err(format!("unknown option to `s': `{c}'")), + } + } + + let re = self.compile(&pattern, case_insensitive, multi_line)?; + Ok(Kind::Substitute(Box::new(Subst { + re, + parts: parse_replacement(&replacement), + occurrence: occurrence.unwrap_or(1), + global, + print, + wfile, + }))) + } +} + +fn clamp(mut text: String) -> String { + if text.len() > MAX_DIAGNOSTIC { + text.truncate(MAX_DIAGNOSTIC); + text.push('…'); + } + text +} + +/// Collapse a multi-line engine diagnostic (the `regex` crate draws a caret +/// diagram) into one line, keeping stderr small per TM-INF-022. +fn flatten(message: &str) -> String { + let compact = message + .lines() + .map(str::trim) + .filter(|line| !line.is_empty() && !line.chars().all(|c| c == '^')) + .collect::>() + .join(" "); + clamp(compact) +} + +fn unescape_y(s: &str, delim: char) -> Vec { + let chars: Vec = s.chars().collect(); + let mut out = Vec::new(); + let mut i = 0; + while i < chars.len() { + if chars[i] == '\\' && i + 1 < chars.len() { + let n = chars[i + 1]; + out.push(match n { + 'n' => '\n', + 't' => '\t', + 'r' => '\r', + '\\' => '\\', + c if c == delim => c, + c => c, + }); + i += 2; + } else { + out.push(chars[i]); + i += 1; + } + } + out +} + +/// Parse an `s///` replacement into literal / capture / case-fold parts. +/// +/// Doing this ourselves (instead of translating to `regex`-crate replacement +/// syntax) is what keeps a literal `$` literal: `sed 's/a/$x/'` must emit `$x`, +/// not expand `$x` as a named capture group (issue #2427, finding A). +pub(super) fn parse_replacement(text: &str) -> Vec { + let chars: Vec = text.chars().collect(); + let mut parts = Vec::new(); + let mut lit = String::new(); + let mut i = 0; + + macro_rules! flush { + () => { + if !lit.is_empty() { + parts.push(RepPart::Lit(std::mem::take(&mut lit))); + } + }; + } + + while i < chars.len() { + let c = chars[i]; + if c == '&' { + flush!(); + parts.push(RepPart::Group(0)); + i += 1; + continue; + } + if c == '\\' && i + 1 < chars.len() { + let n = chars[i + 1]; + i += 2; + match n { + '0'..='9' => { + flush!(); + parts.push(RepPart::Group(n as usize - '0' as usize)); + } + 'n' => lit.push('\n'), + 't' => lit.push('\t'), + 'r' => lit.push('\r'), + 'f' => lit.push('\x0c'), + 'v' => lit.push('\x0b'), + 'a' => lit.push('\x07'), + '\n' => lit.push('\n'), + 'L' => { + flush!(); + parts.push(RepPart::Case(CaseOp::Lower)); + } + 'U' => { + flush!(); + parts.push(RepPart::Case(CaseOp::Upper)); + } + 'l' => { + flush!(); + parts.push(RepPart::Case(CaseOp::LowerOne)); + } + 'u' => { + flush!(); + parts.push(RepPart::Case(CaseOp::UpperOne)); + } + 'E' => { + flush!(); + parts.push(RepPart::Case(CaseOp::End)); + } + // GNU drops the backslash before an ordinary character, so + // `\$` is a literal `$` and `\q` is a literal `q`. + other => lit.push(other), + } + continue; + } + lit.push(c); + i += 1; + } + + flush!(); + parts +} diff --git a/crates/bashkit/src/builtins/sed/tests.rs b/crates/bashkit/src/builtins/sed/tests.rs new file mode 100644 index 000000000..4261328ca --- /dev/null +++ b/crates/bashkit/src/builtins/sed/tests.rs @@ -0,0 +1,648 @@ +use super::*; +use crate::fs::{FileSystem, InMemoryFs}; +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::Arc; + +async fn run_with( + fs: Arc, + args: &[&str], + stdin: Option<&str>, +) -> crate::interpreter::ExecResult { + let sed = Sed; + let mut vars = HashMap::new(); + let mut cwd = PathBuf::from("/"); + let args: Vec = args.iter().map(|s| s.to_string()).collect(); + let ctx = Context { + args: &args, + env: &HashMap::new(), + variables: &mut vars, + cwd: &mut cwd, + fs, + stdin: crate::builtins::test_stream_opt(stdin), + #[cfg(feature = "http_client")] + http_client: None, + #[cfg(feature = "git")] + git_client: None, + #[cfg(feature = "ssh")] + ssh_client: None, + shell: None, + }; + sed.execute(ctx) + .await + .expect("sed must not fail the interpreter") +} + +async fn run_sed(args: &[&str], stdin: Option<&str>) -> crate::interpreter::ExecResult { + run_with(Arc::new(InMemoryFs::new()), args, stdin).await +} + +async fn out(args: &[&str], stdin: &str) -> String { + run_sed(args, Some(stdin)).await.stdout.to_string() +} + +async fn stdout_of(args: &[&str], stdin: &str) -> String { + out(args, stdin).await +} + +// === regression coverage for the original builtin ======================= + +#[tokio::test] +async fn substitute_basics() { + assert_eq!( + out(&["s/hello/goodbye/"], "hello world\nhello again\n").await, + "goodbye world\ngoodbye again\n" + ); + assert_eq!(out(&["s/o/0/g"], "hello world\n").await, "hell0 w0rld\n"); + assert_eq!(out(&["s/o/0/"], "hello world\n").await, "hell0 world\n"); + assert_eq!( + out(&["s/world/[&]/"], "hello world\n").await, + "hello [world]\n" + ); + assert_eq!(out(&["s/hello/hi/i"], "Hello World\n").await, "hi World\n"); + assert_eq!( + out(&["s/hello/hi/; s/world/there/"], "hello world\n").await, + "hi there\n" + ); +} + +#[tokio::test] +async fn addresses_and_hold_space() { + assert_eq!( + out(&["2d"], "line1\nline2\nline3\n").await, + "line1\nline3\n" + ); + assert_eq!(out(&["-n", "2p"], "line1\nline2\nline3\n").await, "line2\n"); + assert_eq!( + out(&["2,3d"], "line1\nline2\nline3\nline4\n").await, + "line1\nline4\n" + ); + assert_eq!( + out(&["$d"], "line1\nline2\nline3\n").await, + "line1\nline2\n" + ); + assert_eq!( + out(&["2c\\replaced"], "line1\nline2\nline3\n").await, + "line1\nreplaced\nline3\n" + ); + assert_eq!( + out(&["/one/a\\inserted"], "one\ntwo\n").await, + "one\ninserted\ntwo\n" + ); + assert_eq!( + out(&["/two/i\\inserted"], "one\ntwo\n").await, + "one\ninserted\ntwo\n" + ); + assert_eq!( + out(&["-e", "1h", "-e", "2g"], "first\nsecond\nthird\n").await, + "first\nfirst\nthird\n" + ); + assert_eq!( + out(&["-e", "1h", "-e", "2H", "-e", "3G"], "a\nb\nc\n").await, + "a\nb\nc\na\nb\n" + ); + assert_eq!( + out(&["-e", "1h", "-e", "2x"], "first\nsecond\nthird\n").await, + "first\nfirst\nthird\n" + ); + assert_eq!( + out(&["/start/,/end/d"], "before\nstart\nmiddle\nend\nafter\n").await, + "before\nafter\n" + ); + assert_eq!( + out(&["/begin/,/end/s/x/y/g"], "ax\nbeginx\nmiddlex\nendx\nax\n").await, + "ax\nbeginy\nmiddley\nendy\nax\n" + ); +} + +#[tokio::test] +async fn back_references_use_the_fancy_engine() { + assert_eq!( + out(&["s/\\(hello\\) \\(world\\)/\\2 \\1/"], "hello world\n").await, + "world hello\n" + ); + assert_eq!(out(&["s/\\(.\\)\\1/X/g"], "aabbc\n").await, "XXc\n"); + assert_eq!(out(&["s/\\(hel\\)lo/\\1p/"], "hello\n").await, "help\n"); + assert_eq!( + out( + &[r#"s|\1|\1|g"#], + "hello\n" + ) + .await, + "hello\n" + ); +} + +#[test] +fn fancy_regex_backtrack_limit_is_enforced() { + let Ok(pattern::SedRegex::Fancy(re)) = + pattern::SedRegex::build_fancy_with_limit(r"(a+)+\1b", false, false, 100) + else { + panic!("expected fancy-regex fallback"); + }; + let err = re + .is_match("a".repeat(5_000).as_str()) + .expect_err("expected backtrack limit error"); + assert!(!err.to_string().is_empty()); +} + +#[tokio::test] +async fn grouped_command_nesting_depth_is_capped() { + let depth = crate::builtins::limits::SED_MAX_GROUP_NESTING_DEPTH + 1; + let script = format!("{}p{}", "{".repeat(depth), "}".repeat(depth)); + let result = run_sed(&[&script], Some("a\n")).await; + assert_eq!(result.exit_code, 1); + assert!( + result.stderr.contains("nesting exceeds max depth"), + "{}", + result.stderr + ); +} + +#[tokio::test] +async fn branch_loop_limit_emits_a_warning() { + let result = run_sed(&[":loop; s/a/aa/; /a\\{2000\\}/!b loop"], Some("a\n")).await; + assert!(result.stderr.contains("loop limit"), "{}", result.stderr); + assert_eq!(result.exit_code, 0); +} + +#[tokio::test] +async fn ordinary_scripts_write_nothing_to_stderr() { + let result = run_sed(&["s/hello/world/"], Some("hello\n")).await; + assert!(result.stderr.is_empty()); + assert_eq!(result.stdout, "world\n"); +} + +// === finding A: `$` in the replacement is literal ======================= + +#[tokio::test] +async fn dollar_in_replacement_is_literal() { + assert_eq!(out(&["s/a/$x/"], "ab\n").await, "$xb\n"); + assert_eq!(out(&["s/\\(a\\)/[$1]/"], "ab\n").await, "[$1]b\n"); + assert_eq!(out(&["s/a/$$/"], "ab\n").await, "$$b\n"); + assert_eq!(out(&["s/a/${x}/"], "ab\n").await, "${x}b\n"); + // GNU drops the backslash before an ordinary character. + assert_eq!(out(&["s/a/\\$/"], "ab\n").await, "$b\n"); + assert_eq!(out(&["s/a/\\q/"], "ab\n").await, "qb\n"); + assert_eq!(out(&["s/^/$PREFIX-/"], "x\n").await, "$PREFIX-x\n"); +} + +#[tokio::test] +async fn replacement_case_conversion() { + assert_eq!( + out(&["s/a\\(b\\)c/\\U\\1x\\E-\\1/"], "abc\n").await, + "BX-b\n" + ); + assert_eq!(out(&["s/.*/\\u&/"], "abc\n").await, "Abc\n"); + assert_eq!(out(&["s/.*/\\L&/"], "ABC\n").await, "abc\n"); +} + +// === finding B: BRE literals vs ERE operators =========================== + +#[tokio::test] +async fn bre_treats_plus_question_pipe_as_literals() { + assert_eq!(out(&["s/a+b/X/"], "a+b\n").await, "X\n"); + assert_eq!(out(&["s/a?b/X/"], "a?b\n").await, "X\n"); + assert_eq!(out(&["s/a|b/X/"], "a|b\n").await, "X\n"); + assert_eq!(out(&["s/a{b/X/"], "a{b\n").await, "X\n"); + assert_eq!(out(&["s/a(b)/X/"], "a(b)\n").await, "X\n"); + // A leading `*` is an ordinary character, not a dangling quantifier. + assert_eq!(out(&["s/*a/X/"], "aaa\n").await, "aaa\n"); + // `^` and `$` only anchor at the edges. + assert_eq!(out(&["-n", "/a^b/p"], "a^b\n").await, "a^b\n"); + assert_eq!(out(&["-n", "/a$b/p"], "a$b\n").await, "a$b\n"); +} + +#[tokio::test] +async fn address_regexes_go_through_the_same_bre_translation() { + assert_eq!(out(&["-n", "/a\\+/p"], "aaa\n").await, "aaa\n"); + assert_eq!(out(&["-n", "/a\\|b/p"], "b\n").await, "b\n"); + assert_eq!(out(&["-n", "/a+b/p"], "a+b\n").await, "a+b\n"); + assert_eq!(out(&["/x\\?/d"], "x?\n").await, ""); +} + +#[tokio::test] +async fn extended_mode_keeps_ere_operators() { + assert_eq!(out(&["-E", "s/a+/X/"], "aaa\n").await, "X\n"); + assert_eq!(out(&["-E", "s/a(X)b/[\\1]/"], "aXb\n").await, "[X]\n"); + assert_eq!(out(&["-E", "s/a\\+b/X/"], "a+b\n").await, "X\n"); +} + +#[tokio::test] +async fn bracket_expressions_keep_posix_semantics() { + assert_eq!(out(&["s/[]]/X/"], "a]b\n").await, "aXb\n"); + assert_eq!(out(&["s/[/]/X/"], "a/b\n").await, "aXb\n"); + assert_eq!(out(&["s/[a-c]/X/g"], "a-b\n").await, "X-X\n"); + assert_eq!(out(&["s/[[:alpha:]]/X/2"], "abc\n").await, "aXc\n"); + // POSIX gives `\` no special meaning inside a bracket expression. + assert_eq!(out(&["s/[\\]/X/g"], "a\\b\n").await, "aXb\n"); +} + +#[test] +fn translate_is_positional() { + assert_eq!(pattern::translate("a+b", false), "a\\+b"); + assert_eq!(pattern::translate("a\\+b", false), "a+b"); + assert_eq!(pattern::translate("^a$", false), "^a$"); + assert_eq!(pattern::translate("a^b$c", false), "a\\^b\\$c"); + assert_eq!(pattern::translate("*a", false), "\\*a"); + assert_eq!(pattern::translate("a*", false), "a*"); + assert_eq!(pattern::translate("a+b", true), "a+b"); +} + +// === finding C: range addresses ========================================= + +#[tokio::test] +async fn regex_to_line_ranges_keep_the_regex() { + assert_eq!(out(&["/b/,3d"], "a\nb\nc\nd\n").await, "a\nd\n"); + // A numeric end at or before the start line makes a one-line range. + assert_eq!(out(&["-n", "/b/,1p"], "a\nb\nc\nd\n").await, "b\n"); +} + +#[tokio::test] +async fn a_closed_range_stays_closed() { + assert_eq!(out(&["-n", "1,/b/p"], "a\nb\nc\n").await, "a\nb\n"); + // A line-number start tests the end regex from the *next* line ... + assert_eq!(out(&["-n", "1,/b/p"], "b\nb\nc\n").await, "b\nb\n"); + // ... while `0,/re/` tests it on the first line. + assert_eq!(out(&["-n", "0,/b/p"], "b\nb\nc\n").await, "b\n"); + // A regex start may re-open later. + assert_eq!( + out(&["-n", "/a/,/b/p"], "a\nb\na\nb\nc\n").await, + "a\nb\na\nb\n" + ); +} + +#[tokio::test] +async fn relative_and_multiple_end_addresses() { + let input = (1..=12).map(|n| format!("{n}\n")).collect::(); + assert_eq!(out(&["-n", "2,~4p"], &input).await, "2\n3\n4\n"); + assert_eq!(out(&["-n", "4,~4p"], &input).await, "4\n5\n6\n7\n8\n"); + assert_eq!(out(&["-n", "3,+2p"], &input).await, "3\n4\n5\n"); + assert_eq!(out(&["-n", "$,$p"], "a\nb\nc\n").await, "c\n"); +} + +#[tokio::test] +async fn change_on_a_range_emits_its_text_once() { + assert_eq!(out(&["1,2c\\Z"], "a\nb\nc\n").await, "Z\nc\n"); + assert_eq!(out(&["1,2!c\\Z"], "a\nb\nc\n").await, "a\nb\nZ\n"); + // A range that never matches emits nothing. + assert_eq!(out(&["/zz/,/b/c\\Z"], "a\nb\nc\n").await, "a\nb\nc\n"); + assert_eq!(out(&["2c\\Z"], "a\nb\nc\n").await, "a\nZ\nc\n"); +} + +// === finding D: operands are one stream ================================= + +async fn fs_with_two_files() -> Arc { + let fs = Arc::new(InMemoryFs::new()); + fs.write_file(std::path::Path::new("/f1"), b"a\nb\nc\n") + .await + .unwrap(); + fs.write_file(std::path::Path::new("/f2"), b"d\ne\n") + .await + .unwrap(); + fs +} + +#[tokio::test] +async fn multiple_files_are_one_continuous_stream() { + let fs = fs_with_two_files().await; + let last = run_with(fs.clone(), &["-n", "$p", "/f1", "/f2"], None).await; + assert_eq!(last.stdout, "e\n"); + let quit = run_with(fs.clone(), &["2q", "/f1", "/f2"], None).await; + assert_eq!(quit.stdout, "a\nb\n"); + let first = run_with(fs.clone(), &["-n", "1p", "/f1", "/f2"], None).await; + assert_eq!(first.stdout, "a\n"); + let count = run_with(fs.clone(), &["-n", "$=", "/f1", "/f2"], None).await; + assert_eq!(count.stdout, "5\n"); +} + +#[tokio::test] +async fn separate_restores_per_file_streams() { + let fs = fs_with_two_files().await; + let last = run_with(fs.clone(), &["-s", "-n", "$p", "/f1", "/f2"], None).await; + assert_eq!(last.stdout, "c\ne\n"); + let names = run_with(fs.clone(), &["-n", "F", "/f1", "/f2"], None).await; + assert_eq!(names.stdout, "/f1\n/f1\n/f1\n/f2\n/f2\n"); +} + +// === finding E: trailing newline and in-place writes ==================== + +#[tokio::test] +async fn a_missing_final_newline_is_not_invented() { + assert_eq!(out(&["s/a/b/"], "a").await, "b"); + assert_eq!(out(&["p"], "a").await, "a\na"); + assert_eq!(out(&["-n", "p"], "a\nb").await, "a\nb"); + assert_eq!(out(&["s/a/b/"], "a\n").await, "b\n"); + assert_eq!(out(&[""], "").await, ""); +} + +#[tokio::test] +async fn in_place_preserves_bytes_and_mode() { + let fs = Arc::new(InMemoryFs::new()); + let path = std::path::Path::new("/k.txt"); + fs.write_file(path, b"abc").await.unwrap(); + fs.chmod(path, 0o600).await.unwrap(); + + let result = run_with(fs.clone(), &["-i", "s/b/X/", "/k.txt"], None).await; + assert_eq!(result.exit_code, 0); + assert!(result.stdout.is_empty()); + assert_eq!(fs.read_file(path).await.unwrap(), b"aXc"); + assert_eq!(fs.stat(path).await.unwrap().mode, 0o600); + // No temporary left behind. + assert!( + fs.read_dir(std::path::Path::new("/")) + .await + .unwrap() + .iter() + .all(|e| !e.name.starts_with(".bashkit-sed-")) + ); +} + +#[tokio::test] +async fn in_place_with_suffix_keeps_a_backup() { + let fs = Arc::new(InMemoryFs::new()); + fs.write_file(std::path::Path::new("/k.txt"), b"a\nb\n") + .await + .unwrap(); + let result = run_with(fs.clone(), &["-i.bak", "s/a/A/", "/k.txt"], None).await; + assert_eq!(result.exit_code, 0); + assert_eq!( + fs.read_file(std::path::Path::new("/k.txt")).await.unwrap(), + b"A\nb\n" + ); + assert_eq!( + fs.read_file(std::path::Path::new("/k.txt.bak")) + .await + .unwrap(), + b"a\nb\n" + ); +} + +#[tokio::test] +async fn in_place_refuses_non_utf8_rather_than_corrupting() { + let fs = Arc::new(InMemoryFs::new()); + let path = std::path::Path::new("/bin.dat"); + fs.write_file(path, &[0x61, 0xff, 0x62]).await.unwrap(); + let result = run_with(fs.clone(), &["-i", "s/a/X/", "/bin.dat"], None).await; + assert_eq!(result.exit_code, 1); + assert!( + result.stderr.contains("not a valid UTF-8 text file"), + "{}", + result.stderr + ); + assert_eq!(fs.read_file(path).await.unwrap(), vec![0x61, 0xff, 0x62]); +} + +#[tokio::test] +async fn in_place_without_operands_is_an_error() { + let result = run_sed(&["-i", "s/a/b/"], Some("a\n")).await; + assert_eq!(result.exit_code, 4); + assert!( + result.stderr.contains("no input files"), + "{}", + result.stderr + ); +} + +// === finding F: `s///Ng` ================================================ + +#[tokio::test] +async fn occurrence_and_global_compose() { + assert_eq!(out(&["s/L/x/2g"], "heLLo\n").await, "heLxo\n"); + assert_eq!(out(&["s/a/X/3g"], "aaa\n").await, "aaX\n"); + assert_eq!(out(&["s/a/X/3"], "aaaa\n").await, "aaXa\n"); + assert_eq!(out(&["s/a/X/4"], "aaa\n").await, "aaa\n"); + assert_eq!(out(&["s/o/0/3"], "hello world\n").await, "hello world\n"); +} + +// === finding G: the missing commands ==================================== + +#[tokio::test] +async fn previously_missing_commands_work() { + assert_eq!(out(&["-n", "$="], "a\nb\n").await, "2\n"); + assert_eq!(out(&["N;s/\\n/ /"], "a\nb\nc\n").await, "a b\nc\n"); + assert_eq!(out(&["y/abc/xyz/"], "abc\n").await, "xyz\n"); + assert_eq!(out(&["-n", "/a/{n;p}"], "a\nb\nc\n").await, "b\n"); + // `T` branches when *no* substitution has happened yet. + assert_eq!(out(&["T end;s/a/X/;:end"], "abc\n").await, "abc\n"); + assert_eq!(out(&["s/a/X/;T end;s/b/Y/;:end"], "abc\n").await, "XYc\n"); + assert_eq!(out(&["# comment"], "a\nb\n").await, "a\nb\n"); + assert_eq!(out(&["-n", "N;P"], "a\nb\n").await, "a\n"); + assert_eq!(out(&["z;s/^$/EMPTY/"], "abc\n").await, "EMPTY\n"); + assert_eq!(out(&["-n", "l"], "a\tb\\c\n").await, "a\\tb\\\\c$\n"); + // `#n` on the first line implies -n. + assert_eq!(out(&["#n\np"], "a\n").await, "a\n"); +} + +#[tokio::test] +async fn delete_first_line_restarts_the_cycle() { + // Classic "squeeze repeated blank lines" idiom. + assert_eq!(out(&["/^$/{N;/^\\n$/D}"], "a\n\n\n\nb\n").await, "a\n\nb\n"); +} + +#[tokio::test] +async fn transliterate_rejects_unequal_lengths() { + let result = run_sed(&["y/ab/x/"], Some("a\n")).await; + assert_eq!(result.exit_code, 1); + assert!( + result.stderr.contains("different lengths"), + "{}", + result.stderr + ); +} + +#[tokio::test] +async fn read_and_write_commands_use_the_vfs() { + let fs = Arc::new(InMemoryFs::new()); + fs.write_file(std::path::Path::new("/r.txt"), b"R1\nR2\n") + .await + .unwrap(); + + let appended = run_with(fs.clone(), &["1r /r.txt"], Some("a\nb\n")).await; + assert_eq!(appended.stdout, "a\nR1\nR2\nb\n"); + + let per_line = run_with(fs.clone(), &["R /r.txt"], Some("a\nb\n")).await; + assert_eq!(per_line.stdout, "a\nR1\nb\nR2\n"); + + let written = run_with(fs.clone(), &["-n", "1w /out.txt"], Some("a\nb\n")).await; + assert_eq!(written.exit_code, 0); + assert_eq!( + fs.read_file(std::path::Path::new("/out.txt")) + .await + .unwrap(), + b"a\n" + ); + + let via_subst = run_with(fs.clone(), &["-n", "s/a/X/w /out2.txt"], Some("a\nb\n")).await; + assert_eq!(via_subst.exit_code, 0); + assert_eq!( + fs.read_file(std::path::Path::new("/out2.txt")) + .await + .unwrap(), + b"X\n" + ); +} + +#[tokio::test] +async fn the_e_command_is_refused_not_silently_ignored() { + for script in ["1e echo hi", "s/a/b/e"] { + let result = run_sed(&[script], Some("a\n")).await; + assert_eq!(result.exit_code, 1, "{script}"); + assert!( + result.stderr.contains("not available"), + "{script}: {}", + result.stderr + ); + } +} + +#[tokio::test] +async fn quit_carries_an_exit_status() { + let quit = run_sed(&["1q5"], Some("a\nb\n")).await; + assert_eq!( + (quit.stdout.to_string(), quit.exit_code), + ("a\n".to_string(), 5) + ); + let silent = run_sed(&["1Q3"], Some("a\nb\n")).await; + assert_eq!( + (silent.stdout.to_string(), silent.exit_code), + (String::new(), 3) + ); +} + +#[test] +fn list_format_wraps_and_escapes() { + assert_eq!(exec::list_format("a\tb", 70), "a\\tb$"); + assert_eq!(exec::list_format("é", 70), "\\303\\251$"); + assert_eq!( + exec::list_format(&"x".repeat(80), 70), + format!("{}\\\n{}$", "x".repeat(69), "x".repeat(11)) + ); + assert_eq!(exec::list_format(&"x".repeat(3), 0), "xxx$"); +} + +// === finding H: multi-byte `s` delimiter (TM-UNI-002) =================== + +#[tokio::test] +async fn multibyte_delimiters_do_not_panic() { + // Documented superset over GNU, which rejects a multi-byte delimiter. + assert_eq!(out(&["s≠a≠X≠"], "a\n").await, "X\n"); + assert_eq!(out(&["s😀a😀X😀g"], "aa\n").await, "XX\n"); + let truncated = run_sed(&["s≠a"], Some("a\n")).await; + assert_eq!(truncated.exit_code, 1); + assert!( + truncated.stderr.contains("unterminated"), + "{}", + truncated.stderr + ); +} + +// === finding I: option parsing ========================================== + +#[tokio::test] +async fn short_options_cluster_and_take_arguments() { + assert_eq!(out(&["-ne", "p"], "a\n").await, "a\n"); + assert_eq!( + out(&["-n", "-e", "1p", "-e", "2p"], "a\nb\n").await, + "a\nb\n" + ); + assert_eq!(out(&["-nE", "s/a+/X/p"], "aa\n").await, "X\n"); + assert_eq!(out(&["--quiet", "-e", "p"], "x\n").await, "x\n"); + assert_eq!(out(&["--expression=s/x/y/"], "x\n").await, "y\n"); + assert_eq!(out(&["-n", "--expression", "p"], "x\n").await, "x\n"); +} + +#[tokio::test] +async fn an_empty_script_is_a_passthrough() { + let result = run_sed(&[""], Some("a\n")).await; + assert_eq!( + (result.stdout.to_string(), result.exit_code), + ("a\n".to_string(), 0) + ); +} + +#[tokio::test] +async fn double_dash_ends_option_parsing() { + // `-e` is then the script, and `-` is not a valid command. + let result = run_sed(&["--", "-e", "p"], Some("a\n")).await; + assert_eq!(result.exit_code, 1); + assert!( + result.stderr.contains("unknown command: `-'"), + "{}", + result.stderr + ); +} + +#[tokio::test] +async fn script_files_are_read_from_the_vfs() { + let fs = Arc::new(InMemoryFs::new()); + fs.write_file(std::path::Path::new("/sc.sed"), b"s/a/X/\ns/X/Y/\n") + .await + .unwrap(); + assert_eq!( + run_with(fs.clone(), &["-f", "/sc.sed"], Some("a\n")) + .await + .stdout, + "Y\n" + ); + let missing = run_with(fs, &["-f", "/nope.sed"], Some("a\n")).await; + assert_eq!(missing.exit_code, 4); + assert!( + missing.stderr.contains("couldn't open file"), + "{}", + missing.stderr + ); +} + +#[tokio::test] +async fn unreadable_input_exits_two() { + let fs = Arc::new(InMemoryFs::new()); + let result = run_with(fs, &["s/a/b/", "/nope.txt"], None).await; + assert_eq!(result.exit_code, 2); + assert!(result.stderr.contains("can't read"), "{}", result.stderr); +} + +#[tokio::test] +async fn unknown_options_are_rejected() { + let result = run_sed(&["-Q", "s/a/b/"], Some("a\n")).await; + assert_eq!(result.exit_code, 1); + assert!( + result.stderr.contains("invalid option -- 'Q'"), + "{}", + result.stderr + ); + let long = run_sed(&["--bogus", "p"], Some("a\n")).await; + assert_eq!(long.exit_code, 1); + assert!( + long.stderr.contains("unrecognized option '--bogus'"), + "{}", + long.stderr + ); +} + +#[tokio::test] +async fn null_data_splits_on_nul() { + assert_eq!(stdout_of(&["-z", "s/a/X/g"], "a\0a\0").await, "X\0X\0"); +} + +#[tokio::test] +async fn unresolvable_branch_targets_exit_four() { + let result = run_sed(&["b nowhere"], Some("a\n")).await; + assert_eq!(result.exit_code, 4); + assert!( + result.stderr.contains("can't find label"), + "{}", + result.stderr + ); +} + +// TM-INF-022: malformed-regex stderr must not leak `regex` crate Debug shapes. +#[tokio::test] +async fn no_leak_invalid_regex() { + let r = crate::builtins::debug_leak_check::run(r"echo 1 | sed -E 's/[[:bogus:]]/x/'").await; + crate::builtins::debug_leak_check::assert_no_leak( + &r, + "sed_invalid_regex", + &["regex::Error", "ParseError {"], + ); +} diff --git a/crates/bashkit/src/builtins/yq.rs b/crates/bashkit/src/builtins/yq.rs index 0d1a22edc..b25602365 100644 --- a/crates/bashkit/src/builtins/yq.rs +++ b/crates/bashkit/src/builtins/yq.rs @@ -4,15 +4,13 @@ //! evaluated by the existing jq/jaq builtin, so Bashkit has one query language, //! one evaluator, and one set of work/output/deadline mitigations. -use std::path::Path; - use async_trait::async_trait; use serde::Deserialize; use super::{Builtin, Context, Jq, read_text_file, resolve_path}; use crate::StreamData; +use crate::builtins::atomic_write::{AtomicFailpoints, atomic_replace}; use crate::error::Result; -use crate::fs::vfs_join; use crate::interpreter::ExecResult; use crate::limits::ExecutionLimits; @@ -203,7 +201,19 @@ impl Builtin for Yq { } if let Some(path) = inplace_path { - if let Err(message) = atomic_replace(fs.as_ref(), &path, rendered.as_bytes()).await { + if let Err(message) = atomic_replace( + fs.as_ref(), + &path, + rendered.as_bytes(), + "yq", + AtomicFailpoints { + allocate: "yq::temp_allocate", + chmod: "yq::temp_chmod", + rename: "yq::temp_rename", + }, + ) + .await + { return Ok(ExecResult::err(format!("yq: {message}\n"), 1)); } return Ok(ExecResult::ok(String::new())); @@ -661,73 +671,6 @@ fn render_results(values: &[serde_json::Value], args: &Args) -> DataResult std::result::Result<(), String> { - let parent = target.parent().unwrap_or_else(|| Path::new("/")); - #[cfg(feature = "failpoints")] - if injected_failure("yq::temp_allocate", "exhausted") { - return Err("cannot allocate temporary file".to_string()); - } - let mut temp = None; - for _ in 0..16 { - let mut suffix = [0u8; 8]; - getrandom::fill(&mut suffix) - .map_err(|_| "cannot generate temporary filename".to_string())?; - let suffix = suffix - .iter() - .map(|byte| format!("{byte:02x}")) - .collect::(); - let candidate = vfs_join(parent, format!(".bashkit-yq-{suffix}.tmp")); - if !fs - .exists(&candidate) - .await - .map_err(|error| error.to_string())? - { - temp = Some(candidate); - break; - } - } - let temp = temp.ok_or_else(|| "cannot allocate temporary file".to_string())?; - if let Err(error) = fs.write_file(&temp, content).await { - let _ = fs.remove(&temp, false).await; - return Err(format!("cannot write temporary file: {error}")); - } - #[cfg(feature = "failpoints")] - if injected_failure("yq::temp_chmod", "error") { - let _ = fs.remove(&temp, false).await; - return Err("cannot preserve file mode: injected failure".to_string()); - } - if let Ok(metadata) = fs.stat(target).await - && let Err(error) = fs.chmod(&temp, metadata.mode).await - { - let _ = fs.remove(&temp, false).await; - return Err(format!("cannot preserve file mode: {error}")); - } - #[cfg(feature = "failpoints")] - if injected_failure("yq::temp_rename", "error") { - let _ = fs.remove(&temp, false).await; - return Err(format!( - "cannot replace '{}': injected failure", - target.display() - )); - } - if let Err(error) = fs.rename(&temp, target).await { - let _ = fs.remove(&temp, false).await; - return Err(format!("cannot replace '{}': {error}", target.display())); - } - Ok(()) -} - -#[cfg(feature = "failpoints")] -fn injected_failure(name: &str, expected: &str) -> bool { - fail::eval(name, |action| action.as_deref() == Some(expected)).unwrap_or(false) -} - fn relabel_jq_error(mut result: ExecResult) -> ExecResult { let stderr = result.stderr.to_string().replace("jq:", "yq:"); result.stderr = StreamData::from(stderr); diff --git a/crates/bashkit/src/testing.rs b/crates/bashkit/src/testing.rs index 84b31d24e..2ffccd8bf 100644 --- a/crates/bashkit/src/testing.rs +++ b/crates/bashkit/src/testing.rs @@ -34,6 +34,12 @@ pub const UNIVERSAL_BANNED: &[&str] = &[ "Some([", "Span {", "Range {", + // -- Caught panics (TM-INT-001, TM-UNI-002) -- + // The interpreter catches builtin panics so the host stays up, but a + // panic reachable from untrusted script text is still a bug; without + // these the fuzz targets pass on every caught panic. Issue #2427. + "internal error:", + "builtin failed unexpectedly", // -- Host paths (TM-INF-016) -- // Rust compiler internals leaked via panic backtraces. "/rustc/", diff --git a/crates/bashkit/tests/integration/main.rs b/crates/bashkit/tests/integration/main.rs index 8034ae78a..26bb2ccc6 100644 --- a/crates/bashkit/tests/integration/main.rs +++ b/crates/bashkit/tests/integration/main.rs @@ -102,6 +102,7 @@ pub mod runtime_env_tests; pub mod script_analysis; pub mod script_execution_tests; pub mod security_audit_pocs; +mod sed_fuzz_scaffold_tests; pub mod set_e_and_or_tests; pub mod shuf_resource_tests; pub mod skills_tests; diff --git a/crates/bashkit/tests/integration/sed_fuzz_scaffold_tests.rs b/crates/bashkit/tests/integration/sed_fuzz_scaffold_tests.rs new file mode 100644 index 000000000..691d62a2e --- /dev/null +++ b/crates/bashkit/tests/integration/sed_fuzz_scaffold_tests.rs @@ -0,0 +1,114 @@ +// Scaffold tests for the sed_fuzz target. +// +// Validates that the sed builtin survives arbitrary scripts and input without +// panicking AND without leaking Debug shapes, host paths, or the host-env +// canary into stderr/stdout. +// +// `fuzz_exec` also rejects stderr that reports a *caught* panic (`internal +// error:` / `builtin failed unexpectedly`). Before issue #2427 those passed +// silently, so `sed 's≠a≠X≠'` — a byte-index slice through a multi-byte +// delimiter, THREAT[TM-UNI-002] — looked clean to the fuzzer. + +use bashkit::testing::{fuzz_exec, fuzz_init}; +use bashkit::{Bash, ExecutionLimits}; + +fn fuzz_bash() -> Bash { + fuzz_init(); + Bash::builder() + .limits( + ExecutionLimits::new() + .max_commands(50) + .max_subst_depth(3) + .max_stdout_bytes(4096) + .max_stderr_bytes(4096) + .timeout(std::time::Duration::from_secs(2)), + ) + .build() +} + +/// Every seed must leave stderr free of Debug shapes and caught panics. +async fn seed(name: &str, script: &str) { + let mut bash = fuzz_bash(); + fuzz_exec(&mut bash, script, name, &[]).await; +} + +#[tokio::test] +async fn sed_multibyte_delimiters() { + // THREAT[TM-UNI-002]: the delimiter is read as a char but the body used to + // be sliced at byte offset 2, panicking inside `≠`. + for script in [ + r#"printf 'a\n' | sed 's≠a≠X≠'"#, + r#"printf 'a\n' | sed 's≠a'"#, + r#"printf 'a\n' | sed 'sé'"#, + r#"printf 'a\n' | sed 'y≠a≠b≠'"#, + r#"printf 'a\n' | sed '\≠a≠p'"#, + r#"printf 'a\n' | sed 's😀a😀X😀g'"#, + ] { + seed("sed_multibyte_delimiters", script).await; + } +} + +#[tokio::test] +async fn sed_truncated_scripts() { + for script in [ + r#"printf 'a\n' | sed 's/a'"#, + r#"printf 'a\n' | sed 's/'"#, + r#"printf 'a\n' | sed 's'"#, + r#"printf 'a\n' | sed '/a'"#, + r#"printf 'a\n' | sed '\'"#, + r#"printf 'a\n' | sed 'y/ab/x/'"#, + r#"printf 'a\n' | sed '1,'"#, + r#"printf 'a\n' | sed '{'"#, + r#"printf 'a\n' | sed '}'"#, + r#"printf 'a\n' | sed 's/a/b/zz'"#, + r#"printf 'a\n' | sed 's/[//'"#, + r#"printf 'a\n' | sed 's/a/b/0'"#, + ] { + seed("sed_truncated_scripts", script).await; + } +} + +#[tokio::test] +async fn sed_pathological_regexes() { + for script in [ + r#"printf 'aaaa\n' | sed 's/\(a*\)*b/X/'"#, + r#"printf 'aaaa\n' | sed -E 's/(a+)+\1b/X/'"#, + r#"printf 'a\n' | sed 's/[[:bogus:]]/X/'"#, + r#"printf 'a\n' | sed 's/a\{1,999999\}/X/'"#, + r#"printf 'a\n' | sed -n '/\(/p'"#, + r#"printf 'a\n' | sed 's/*a/X/'"#, + ] { + seed("sed_pathological_regexes", script).await; + } +} + +#[tokio::test] +async fn sed_flow_control_seeds() { + for script in [ + r#"printf 'a\n' | sed ':a;ba'"#, + r#"printf 'a\n' | sed ':a;s/a/aa/;ta'"#, + r#"printf 'a\n' | sed 'b missing'"#, + r#"printf 'a\nb\n' | sed 'N;N;N;P;D'"#, + r#"printf 'a\n' | sed -n 'l 1'"#, + r#"printf 'a\n' | sed -n 'l 0'"#, + ] { + seed("sed_flow_control_seeds", script).await; + } +} + +#[tokio::test] +async fn sed_option_seeds() { + for script in [ + "printf 'a\\n' | sed -ne p", + "printf 'a\\n' | sed ''", + "printf 'a\\n' | sed -e", + "printf 'a\\n' | sed -f", + "printf 'a\\n' | sed -i", + "printf 'a\\n' | sed -l x -n l", + "printf 'a\\n' | sed -- -e p", + "printf 'a\\n' | sed --in-place", + "printf 'a\\n' | sed --bogus p", + ] { + seed("sed_option_seeds", script).await; + } +} diff --git a/crates/bashkit/tests/integration/spec_tests.rs b/crates/bashkit/tests/integration/spec_tests.rs index 23bc4c64f..c75fbfc91 100644 --- a/crates/bashkit/tests/integration/spec_tests.rs +++ b/crates/bashkit/tests/integration/spec_tests.rs @@ -425,10 +425,23 @@ async fn run_category_tests( #[tokio::test] #[ignore = "strict host-bash parity gate; run explicitly in CI or via just check-bash-compat"] async fn bash_comparison_tests() { - let dir = spec_cases_dir().join("bash"); + compare_against_real_bash("bash").await; +} + +/// The sed spec suite is a differential suite: every expectation was taken from +/// GNU sed, so it must keep matching the host tool (issue #2427). Divergences we +/// keep on purpose carry `### bash_diff:` and are excluded, like the bash gate. +#[tokio::test] +#[ignore = "strict host-sed parity gate; run explicitly in CI or via just check-bash-compat"] +async fn sed_comparison_tests() { + compare_against_real_bash("sed").await; +} + +async fn compare_against_real_bash(category: &str) { + let dir = spec_cases_dir().join(category); let all_tests = load_spec_tests(&dir); - println!("\n=== Bash Comparison Tests ==="); + println!("\n=== {} Comparison Tests ===", category.to_uppercase()); println!("Comparing Bashkit output against real bash\n"); let mut total = 0; diff --git a/crates/bashkit/tests/security_failpoint_tests.rs b/crates/bashkit/tests/security_failpoint_tests.rs index c8fc5c95e..1c84456fe 100644 --- a/crates/bashkit/tests/security_failpoint_tests.rs +++ b/crates/bashkit/tests/security_failpoint_tests.rs @@ -304,6 +304,87 @@ async fn security_yq_inplace_stage_failures_are_atomic() { } } +/// THREAT[TM-FS-016]: `sed -i` shares yq's atomic-replace helper, so every +/// injectable stage must keep the original bytes and clean up the temporary. +#[tokio::test] +#[serial] +async fn security_sed_inplace_stage_failures_are_atomic() { + for (failpoint, action) in [ + ("sed::temp_allocate", "return(exhausted)"), + ("sed::temp_chmod", "return(error)"), + ("sed::temp_rename", "return(error)"), + ] { + let fs = Arc::new(InMemoryFs::new()); + fs.write_file(Path::new("/tmp/data.txt"), b"name: original\n") + .await + .unwrap(); + let mut bash = Bash::builder().fs(fs.clone()).build(); + + fail::cfg(failpoint, action).unwrap(); + let result = bash + .exec("sed -i 's/original/changed/' /tmp/data.txt") + .await + .unwrap(); + fail::cfg(failpoint, "off").unwrap(); + + assert_ne!(result.exit_code, 0, "{failpoint}"); + assert_eq!( + fs.read_file(Path::new("/tmp/data.txt")).await.unwrap(), + b"name: original\n", + "{failpoint}" + ); + assert!( + fs.read_dir(Path::new("/tmp")) + .await + .unwrap() + .iter() + .all(|entry| !entry.name.starts_with(".bashkit-sed-")), + "temporary leaked after {failpoint}" + ); + } +} + +/// THREAT[TM-FS-016]: backend write failures cannot damage the file `sed -i` +/// is editing or leave a sed temporary behind. +#[tokio::test] +#[serial] +async fn security_sed_inplace_write_failures_are_atomic() { + for action in [ + "io_error", + "disk_full", + "permission_denied", + "partial_write", + ] { + let fs = Arc::new(InMemoryFs::new()); + fs.write_file(Path::new("/tmp/data.txt"), b"original\n") + .await + .unwrap(); + let mut bash = Bash::builder().fs(fs.clone()).build(); + + fail::cfg("fs::write_file", &format!("return({action})")).unwrap(); + let result = bash + .exec("sed -i 's/original/changed/' /tmp/data.txt") + .await + .unwrap(); + fail::cfg("fs::write_file", "off").unwrap(); + + assert_ne!(result.exit_code, 0, "{action}"); + assert_eq!( + fs.read_file(Path::new("/tmp/data.txt")).await.unwrap(), + b"original\n", + "{action}" + ); + assert!( + fs.read_dir(Path::new("/tmp")) + .await + .unwrap() + .iter() + .all(|entry| !entry.name.starts_with(".bashkit-sed-")), + "temporary leaked after {action}" + ); + } +} + /// THREAT[TM-FS-016]: backend write failures, including partial-write /// simulation, cannot damage the source or leave a yq temporary behind. #[tokio::test] diff --git a/crates/bashkit/tests/spec_cases/sed/sed.test.sh b/crates/bashkit/tests/spec_cases/sed/sed.test.sh index f300b5881..93bda90e7 100644 --- a/crates/bashkit/tests/spec_cases/sed/sed.test.sh +++ b/crates/bashkit/tests/spec_cases/sed/sed.test.sh @@ -587,3 +587,316 @@ echo "abc" | sed 's/b/\\/' ### expect a\c ### end + +### sed_replacement_dollar_is_literal +# Issue #2427 A: `$` in the RHS is text, not a capture reference +printf 'ab\n' | sed 's/a/$x/' +printf 'ab\n' | sed 's/\(a\)/[$1]/' +printf 'ab\n' | sed 's/a/$$/' +### expect +$xb +[$1]b +$$b +### end + +### sed_replacement_backslash_drops +# Issue #2427 A: GNU drops the backslash before an ordinary character +printf 'ab\n' | sed 's/a/\$/' +printf 'ab\n' | sed 's/a/\q/' +### expect +$b +qb +### end + +### sed_replacement_case_conversion +# GNU \U \L \u \l \E in the replacement +printf 'abc\n' | sed 's/a\(b\)c/\U\1x\E-\1/' +printf 'abc\n' | sed 's/.*/\u&/' +printf 'ABC\n' | sed 's/.*/\L&/' +### expect +BX-b +Abc +abc +### end + +### sed_bre_literals +# Issue #2427 B: + ? | are ordinary characters in BRE +printf 'a+b\n' | sed 's/a+b/X/' +printf 'a?b\n' | sed 's/a?b/X/' +printf 'a|b\n' | sed 's/a|b/X/' +printf 'aaa\n' | sed 's/*a/X/' +### expect +X +X +X +aaa +### end + +### sed_bre_anchors_are_positional +# Issue #2427 B: ^ and $ only anchor at the edges of a BRE +printf 'a^b\n' | sed -n '/a^b/p' +printf 'a$b\n' | sed -n '/a$b/p' +### expect +a^b +a$b +### end + +### sed_address_regex_uses_bre +# Issue #2427 B: address regexes take the same BRE path as s/// +printf 'aaa\n' | sed -n '/a\+/p' +printf 'a+b\n' | sed -n '/a+b/p' +### expect +aaa +a+b +### end + +### sed_bracket_expression_holds_delimiter +# Inside [...] the s/// delimiter is an ordinary character +printf 'a/b\n' | sed 's/[/]/X/' +printf 'a]b\n' | sed 's/[]]/X/' +### expect +aXb +aXb +### end + +### sed_regex_to_line_range +# Issue #2427 C: /re/,N must keep the regex +printf 'a\nb\nc\nd\n' | sed '/b/,3d' +### expect +a +d +### end + +### sed_closed_range_stays_closed +# Issue #2427 C: N,/re/ closes once and does not re-open +printf 'a\nb\nc\n' | sed -n '1,/b/p' +### expect +a +b +### end + +### sed_zero_range_tests_first_line +# 0,/re/ is the form whose end regex is tested on line 1 +printf 'b\nb\nc\n' | sed -n '0,/b/p' +### expect +b +### end + +### sed_relative_end_addresses +# addr,+N and addr,~N +printf '1\n2\n3\n4\n5\n6\n7\n8\n9\n' | sed -n '3,+2p' +printf '1\n2\n3\n4\n5\n6\n7\n8\n9\n' | sed -n '2,~4p' +### expect +3 +4 +5 +2 +3 +4 +### end + +### sed_change_on_range_prints_once +# Issue #2427 C: c on a range emits its text once +printf 'a\nb\nc\n' | sed '1,2c\Z' +### expect +Z +c +### end + +### sed_multiple_files_are_one_stream +# Issue #2427 D: line numbers, $ and q span all operands +printf 'a\nb\nc\n' > /tmp/sed_f1.txt +printf 'd\ne\n' > /tmp/sed_f2.txt +sed -n '$p' /tmp/sed_f1.txt /tmp/sed_f2.txt +sed '2q' /tmp/sed_f1.txt /tmp/sed_f2.txt +sed -n '$=' /tmp/sed_f1.txt /tmp/sed_f2.txt +### expect +e +a +b +5 +### end + +### sed_separate_restores_per_file_streams +# -s puts each operand back in its own stream +printf 'a\nb\nc\n' > /tmp/sed_s1.txt +printf 'd\ne\n' > /tmp/sed_s2.txt +sed -s -n '$p' /tmp/sed_s1.txt /tmp/sed_s2.txt +### expect +c +e +### end + +### sed_missing_final_newline_is_preserved +# Issue #2427 E: sed must not invent a trailing newline +printf 'a' | sed 's/a/b/' > /tmp/sed_nl.txt +wc -c < /tmp/sed_nl.txt +printf 'abc' > /tmp/sed_inplace.txt +sed -i 's/b/X/' /tmp/sed_inplace.txt +wc -c < /tmp/sed_inplace.txt +cat /tmp/sed_inplace.txt +echo +### expect +1 +3 +aXc +### end + +### sed_in_place_preserves_mode +# Issue #2427 E: -i keeps the original file mode +printf 'x\n' > /tmp/sed_mode.txt +chmod 600 /tmp/sed_mode.txt +sed -i 's/x/y/' /tmp/sed_mode.txt +stat -c %a /tmp/sed_mode.txt +cat /tmp/sed_mode.txt +### expect +600 +y +### end + +### sed_in_place_backup_suffix +# -i.bak keeps the original alongside the edit +printf 'a\nb\n' > /tmp/sed_bak.txt +sed -i.bak 's/a/A/' /tmp/sed_bak.txt +cat /tmp/sed_bak.txt +cat /tmp/sed_bak.txt.bak +### expect +A +b +a +b +### end + +### sed_occurrence_with_global +# Issue #2427 F: s///Ng replaces the Nth match and everything after it +printf 'heLLo\n' | sed 's/L/x/2g' +printf 'aaa\n' | sed 's/a/X/3g' +printf 'aaaa\n' | sed 's/a/X/3' +### expect +heLxo +aaX +aaXa +### end + +### sed_line_number_command +# Issue #2427 G: `=` +printf 'a\nb\n' | sed -n '$=' +### expect +2 +### end + +### sed_next_commands +# Issue #2427 G: n and N +printf 'a\nb\nc\n' | sed 'N;s/\n/ /' +printf 'a\nb\nc\n' | sed -n '/a/{n;p}' +### expect +a b +c +b +### end + +### sed_transliterate +# Issue #2427 G: y +printf 'abc\n' | sed 'y/abc/xyz/' +### expect +xyz +### end + +### sed_comments_and_hash_n +# Issue #2427 G: # comments, and #n as the first line implying -n +printf 'a\nb\n' | sed '# just a comment' +printf 'a\n' | sed '#n +p' +### expect +a +b +a +### end + +### sed_list_command +# Issue #2427 G: l renders the pattern space unambiguously +printf 'a\tb\\c\n' | sed -n 'l' +### expect +a\tb\\c$ +### end + +### sed_branch_if_no_substitution +# Issue #2427 G: T +printf 'abc\n' | sed 's/a/X/;T end;s/b/Y/;:end' +printf 'abc\n' | sed 'T end;s/a/X/;:end' +### expect +XYc +abc +### end + +### sed_read_and_write_files +# Issue #2427 G: r, R and w against the VFS +printf 'R1\nR2\n' > /tmp/sed_r.txt +printf 'a\nb\n' | sed '1r /tmp/sed_r.txt' +printf 'a\nb\n' | sed 'R /tmp/sed_r.txt' +printf 'a\nb\n' | sed -n '1w /tmp/sed_w.txt' +cat /tmp/sed_w.txt +### expect +a +R1 +R2 +b +a +R1 +b +R2 +a +### end + +### sed_quit_exit_status +# q and Q carry an exit status +printf 'a\nb\n' | sed '1q5' +echo "rc=$?" +printf 'a\nb\n' | sed '1Q3' +echo "rc=$?" +### expect +a +rc=5 +rc=3 +### end + +### sed_multibyte_delimiter_does_not_crash +# Issue #2427 H (TM-UNI-002): a multi-byte s delimiter is parsed, not sliced +### bash_diff: L-SED-001 - GNU rejects a multi-byte delimiter; Bashkit accepts it +printf 'a\n' | sed 's≠a≠X≠' +### expect +X +### end + +### sed_clustered_short_options +# Issue #2427 I: -ne must behave like -n -e +printf 'a\n' | sed -ne p +### expect +a +### end + +### sed_empty_script_is_passthrough +# Issue #2427 I: an empty script copies input through +printf 'a\n' | sed '' +echo "rc=$?" +### expect +a +rc=0 +### end + +### sed_script_file +# Issue #2427 I: -f reads the script from a file +printf 's/a/X/\ns/X/Y/\n' > /tmp/sed_script.sed +printf 'a\n' | sed -f /tmp/sed_script.sed +### expect +Y +### end + +### sed_long_options +# Issue #2427 I: long spellings +printf 'x\n' | sed --expression='s/x/y/' +printf 'x\n' | sed --quiet -e p +### expect +y +x +### end diff --git a/knowledge/operations/limitations.md b/knowledge/operations/limitations.md index 5ab62f512..67a75a04f 100644 --- a/knowledge/operations/limitations.md +++ b/knowledge/operations/limitations.md @@ -128,6 +128,10 @@ pass in CI); only divergences and boundaries are recorded here. | L-YQ-002 | yq | YAML conversion follows YAML 1.1, deterministically sorts mapping keys at the JSON-value boundary, drops comments/style/anchors, and rejects aliases, custom tags, non-string mapping keys, and non-finite numbers rather than expanding graphs or silently corrupting data | `yaml_aliases_are_rejected_before_expansion`, `yaml_tags_and_non_string_keys_fail_closed`, `yaml_duplicate_keys_and_lossy_numbers_fail_closed`, `inplace_update_is_atomic_and_suppresses_stdout` | | L-YQ-003 | yq | Input/output conversion supports YAML and JSON only; mikefarah/yq's XML, CSV, TOML, properties, HCL, Lua, and INI formats are not exposed through yq | stance | | L-GREP-001 | grep | `--color`/`--colour`, `--line-buffered` accepted as no-ops | `l_grep_001_noop_flags` | +| L-SED-001 | sed | A multi-byte `s`/`y`/address delimiter is accepted (`s≠a≠X≠`); GNU rejects it with "delimiter character is not a single-byte character". Deliberate superset: accepting more never turns a working GNU script into a silently wrong one | `multibyte_delimiters_do_not_panic`, 1 `bash_diff` spec test | +| L-SED-002 | sed | The `e` command and the `s///e` flag (run the pattern space as a shell command) are refused at compile time, not silently ignored | `the_e_command_is_refused_not_silently_ignored` | +| L-SED-003 | sed | `-i` refuses a file that is not valid UTF-8 instead of rewriting it through a lossy decode; without `-i`, non-UTF-8 input is decoded with replacement like every other text builtin (L-STREAM-001) | `in_place_refuses_non_utf8_rather_than_corrupting` | +| L-SED-004 | sed | `\<` and `\>` (GNU word-edge operators) both compile to `\b`; the linear-time `regex` engine has no look-around, so a word start and a word end are not distinguished | `translate_is_positional` | | L-CURL-001 | curl | Spec-test coverage for methods/headers/auth/redirects not ported (needs `http_client` + allowlist in harness); payload behavior has integration and real-curl differential coverage | stance | | L-CURL-002 | curl/wget | Unknown options are ignored for compatibility, not rejected (real curl/wget error); deliberate leniency | `curl.rs` | | L-STR-001 | strings | Accepts dash-prefixed filenames (e.g. `-data.bin`), so only a lone unknown short option (`-Q`) is rejected as invalid; GNU rejects `-data.bin` too | `strings.rs` | diff --git a/knowledge/security/security-testing.md b/knowledge/security/security-testing.md index accf806b9..ff6b15610 100644 --- a/knowledge/security/security-testing.md +++ b/knowledge/security/security-testing.md @@ -53,6 +53,9 @@ cargo test --features failpoints security_ -- --test-threads=1 | `yq::temp_allocate` | `exhausted` | Source preservation when no sibling temporary can be allocated | | `yq::temp_chmod` | `error` | Temporary cleanup and source preservation on mode-copy failure | | `yq::temp_rename` | `error` | Temporary cleanup and source preservation before atomic replacement | +| `sed::temp_allocate` | `exhausted` | `sed -i` source preservation when no sibling temporary can be allocated | +| `sed::temp_chmod` | `error` | `sed -i` temporary cleanup and source preservation on mode-copy failure | +| `sed::temp_rename` | `error` | `sed -i` temporary cleanup and source preservation before atomic replacement | The existing `fs::write_file` actions cover yq temporary-write failures, including the partial-write class. `security_yq_inplace_*` asserts original diff --git a/knowledge/security/threat-model.md b/knowledge/security/threat-model.md index 075a46a19..40415bfc7 100644 --- a/knowledge/security/threat-model.md +++ b/knowledge/security/threat-model.md @@ -1236,7 +1236,7 @@ This section maps former vulnerability IDs to the new threat ID scheme and track | ~~TM-PY-027~~ | ~~py_to_json unbounded recursion~~ | ~~Stack overflow~~ | `MAX_NESTING_DEPTH = 64` in `json_to_py_inner` / `py_to_json_inner` / MontyObject converters (**FIXED**) | | ~~TM-DOS-040~~ | ~~Integer truncation on 32-bit~~ | ~~Size check bypass~~ | `usize::try_from(...).unwrap_or(usize::MAX)` (**FIXED**) | | TM-UNI-001 | Awk parser byte-boundary panic on Unicode | Silent builtin failure on valid input | Fix awk parser to use char-boundary-safe indexing | -| TM-UNI-002 | Sed parser byte-boundary issues | Silent builtin failure on valid input | Audit and fix sed byte-indexing | +| ~~TM-UNI-002~~ | ~~Sed parser byte-boundary issues~~ | ~~Silent builtin failure on valid input~~ | ~~Audit and fix sed byte-indexing~~ The sed script parser is char-based end to end (`builtins/sed/script.rs`), so a multi-byte `s`/`y`/address delimiter is parsed rather than sliced at a byte offset. `fuzz_exec` now also rejects stderr reporting a *caught* panic (`internal error:`, `builtin failed unexpectedly`), which previously let these pass; seeds live in `sed_fuzz_scaffold_tests` and the `sed_fuzz` target (**FIXED**, issue #2427) | | ~~TM-UNI-003~~ | ~~Zero-width chars in filenames~~ | ~~Invisible/confusable filenames~~ | ~~Extend `find_unsafe_path_char()`~~ (**FIXED**) | | ~~TM-UNI-011~~ | ~~Tag characters in filenames~~ | ~~Invisible content in filenames~~ | ~~Extend `find_unsafe_path_char()`~~ (**FIXED**) | | TM-UNI-015 | Expr `substr` byte-boundary panic | Silent failure on multi-byte substr | Fix to use char-boundary-safe indexing (issue #434) | diff --git a/scripts/update-spec-expected.sh b/scripts/update-spec-expected.sh index 9d1d47621..9e1db4064 100755 --- a/scripts/update-spec-expected.sh +++ b/scripts/update-spec-expected.sh @@ -15,7 +15,7 @@ # 3. Or add ### bash_diff marker if the difference is intentional # # This is a wrapper around: -# cargo test --test integration -- spec_tests::bash_comparison_tests --ignored +# cargo test --test integration -- spec_tests:: --ignored set -euo pipefail @@ -62,9 +62,9 @@ echo "Checking spec tests against real bash..." echo "" if $VERBOSE; then - cargo test --test integration -- spec_tests::bash_comparison_tests --ignored --nocapture 2>&1 + cargo test --test integration -- spec_tests:: --ignored --nocapture 2>&1 else - cargo test --test integration -- spec_tests::bash_comparison_tests --ignored 2>&1 + cargo test --test integration -- spec_tests:: --ignored 2>&1 fi exit_code=$? From 2b2640730a03c79d1a7d4db1b883cacf7b1196f3 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Mon, 21 Sep 2026 22:08:35 +0000 Subject: [PATCH 2/3] fix(sed): close six more GNU divergences found by differential fuzzing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A randomized differential sweep (18k generated scripts through `bash -c` and `bashkit -c`, compared against GNU sed 4.9) turned up six classes the hand-written audit in #2427 missed. All six are now fixed and pinned: - The hold space carries its own "was this line terminated" flag, which `g`, `G` and `x` transfer onto the pattern space. It starts terminated, so `printf 'ab' | sed G` emits the trailing newline GNU emits. - `q` flushes a withheld newline before exiting; falling off the end of input still does not, and `Q` prints nothing to flush. - A range whose start is a line number opens on the first line at or past it — `n`, `N`, `D` and `b` can carry the stream beyond the start before the range is ever evaluated — and is then spent. - A numeric end excludes the line once the stream is past it, instead of selecting it one last time on the way out. - A *relative* end (`+N`, `~N`) is recomputed on each activation, so skipping past it re-arms the start; an absolute `,N` end does not. - `q`/`Q` reject a second address, and `\N` in an `s///` replacement must name a group the pattern actually has, both GNU compile errors. THREAT[TM-DOS]: `D` reruns the script without reading input, so `sed 'G;D'` never terminates — GNU hangs on it outright. Restarts now share the `b`/`t` loop budget and stop with the same warning (L-SED-005); a `D` loop that makes progress is unaffected and now matches GNU exactly. The sweep is down from 35 divergences per 3000 cases to 3 per 18,000, all of them multi-line-pattern-space commands crossing range addresses (L-SED-006). --- crates/bashkit/src/builtins/sed/exec.rs | 134 ++++++++++++++---- crates/bashkit/src/builtins/sed/pattern.rs | 8 ++ crates/bashkit/src/builtins/sed/script.rs | 21 ++- crates/bashkit/src/builtins/sed/tests.rs | 107 ++++++++++++++ .../bashkit/tests/spec_cases/sed/sed.test.sh | 27 ++++ knowledge/operations/limitations.md | 2 + 6 files changed, 268 insertions(+), 31 deletions(-) diff --git a/crates/bashkit/src/builtins/sed/exec.rs b/crates/bashkit/src/builtins/sed/exec.rs index da634c6ed..8cba60552 100644 --- a/crates/bashkit/src/builtins/sed/exec.rs +++ b/crates/bashkit/src/builtins/sed/exec.rs @@ -52,7 +52,7 @@ impl Sink { impl Sink { /// Write one line, adding a terminator only when the source had one. fn line(&mut self, text: &str, had_newline: bool) { - self.pending(); + self.flush_pending(); self.buf.push_str(text); if had_newline { self.buf.push(self.sep); @@ -63,11 +63,12 @@ impl Sink { /// Write text that already carries its own terminator. fn raw(&mut self, text: &str) { - self.pending(); + self.flush_pending(); self.buf.push_str(text); } - fn pending(&mut self) { + /// Emit a newline that was withheld because the source line had none. + fn flush_pending(&mut self) { if self.missing_newline { self.buf.push(self.sep); self.missing_newline = false; @@ -80,6 +81,11 @@ struct RangeState { active: bool, /// Resolved last line of the range for numeric / `+N` / `~N` ends. end_line: Option, + /// A line-number range start fires at most once. It opens on the first + /// line at or past it — `n`, `N` and `b` can carry the stream past the + /// line before the range is ever evaluated — and must not re-open + /// afterwards, when `line_no >= start` is still trivially true. + start_spent: bool, } enum Append { @@ -111,11 +117,20 @@ pub(super) struct Machine<'a> { range: Vec, hold: String, + /// GNU tracks "was this line terminated" on the hold space too, and `g`, + /// `G` and `x` carry it onto the pattern space. The hold space starts out + /// terminated, which is why `printf 'ab' | sed G` emits a trailing newline + /// even though the input had none. + hold_had_newline: bool, appends: Vec, read_cursor: HashMap, last_regex: Option>, pub(super) exit_code: Option, warned: bool, + /// `q` terminates the line it printed even when the input line had no + /// terminator, unlike falling off the end of input. `Q`, which prints + /// nothing, does not. + flush_newline_on_quit: bool, // Per-segment state. lines: &'a [InputLine], @@ -148,11 +163,13 @@ impl<'a> Machine<'a> { write_files: HashMap::new(), range: Vec::new(), hold: String::new(), + hold_had_newline: true, appends: Vec::new(), read_cursor: HashMap::new(), last_regex: None, exit_code: None, warned: false, + flush_newline_on_quit: false, lines: &[], cursor: 0, line_no: 0, @@ -176,6 +193,11 @@ impl<'a> Machine<'a> { self.out = Sink::with_sep(self.sep); let mut restart = false; + // THREAT[TM-DOS]: `D` restarts the script without reading input, so a + // script like `G;D` never terminates and grows the pattern space + // without bound. The per-cycle step budget cannot see across restarts, + // so they get their own budget, reset whenever a line is read. + let mut restarts = 0usize; loop { if self.exit_code.is_some() { break; @@ -183,8 +205,17 @@ impl<'a> Machine<'a> { // `D` restarts the script on what is left of the pattern space // without reading input, and GNU only clears the `t` flag when a // line is actually read, so the reset lives in `read_next`. - if !restart && !self.read_next() { - break; + if restart { + restarts += 1; + if restarts > SED_MAX_CYCLE_STEPS { + self.warn_loop_limit(); + break; + } + } else { + if !self.read_next() { + break; + } + restarts = 0; } restart = false; @@ -203,6 +234,10 @@ impl<'a> Machine<'a> { restart = true; } } + + if self.flush_newline_on_quit { + self.out.flush_pending(); + } } std::mem::replace(&mut self.out, Sink::with_sep(self.sep)).buf @@ -216,14 +251,7 @@ impl<'a> Machine<'a> { while pc < cmds.len() { steps += 1; if steps > SED_MAX_CYCLE_STEPS { - if !self.warned { - self.warned = true; - self.stderr.push_str(&format!( - "sed: warning: branch/label loop limit ({SED_MAX_CYCLE_STEPS}) reached \ - on line {}; output may be truncated\n", - self.line_no - )); - } + self.warn_loop_limit(); break; } @@ -349,6 +377,7 @@ impl<'a> Machine<'a> { } Kind::Quit(code) => { self.exit_code = Some(*code); + self.flush_newline_on_quit = true; return Cycle::Auto; } Kind::QuitSilent(code) => { @@ -369,17 +398,28 @@ impl<'a> Machine<'a> { } return Cycle::Silent; } - Kind::HoldCopy => self.hold.clone_from(&self.ps), + Kind::HoldCopy => { + self.hold.clone_from(&self.ps); + self.hold_had_newline = self.ps_had_newline; + } Kind::HoldAppend => { self.hold.push('\n'); self.hold.push_str(&self.ps); + self.hold_had_newline = self.ps_had_newline; + } + Kind::GetCopy => { + self.ps.clone_from(&self.hold); + self.ps_had_newline = self.hold_had_newline; } - Kind::GetCopy => self.ps.clone_from(&self.hold), Kind::GetAppend => { self.ps.push('\n'); self.ps.push_str(&self.hold); + self.ps_had_newline = self.hold_had_newline; + } + Kind::Exchange => { + std::mem::swap(&mut self.ps, &mut self.hold); + std::mem::swap(&mut self.ps_had_newline, &mut self.hold_had_newline); } - Kind::Exchange => std::mem::swap(&mut self.ps, &mut self.hold), Kind::LineNumber => { let text = format!("{}\n", self.line_no); self.out.raw(&text); @@ -426,6 +466,18 @@ impl<'a> Machine<'a> { Cycle::Auto } + fn warn_loop_limit(&mut self) { + if self.warned { + return; + } + self.warned = true; + self.stderr.push_str(&format!( + "sed: warning: branch/label loop limit ({SED_MAX_CYCLE_STEPS}) reached on line {}; \ + output may be truncated\n", + self.line_no + )); + } + fn read_next(&mut self) -> bool { if self.cursor >= self.lines.len() { return false; @@ -544,31 +596,52 @@ impl<'a> Machine<'a> { let state = self.range[pc].clone(); if state.active { - let closed = match end { - EndAddr::Line(n) => self.line_no >= *n, - EndAddr::Last => self.is_last_line(), - EndAddr::Plus(_) | EndAddr::Multiple(_) => { - state.end_line.is_some_and(|e| self.line_no >= e) - } + // A numeric end both closes the range and *excludes* the line once + // the stream is past it: `n`/`N`/`b` can carry the stream beyond + // the end line without the range ever being evaluated there. + let (closed, selected) = match end { + EndAddr::Line(n) => (self.line_no >= *n, self.line_no <= *n), + EndAddr::Plus(_) | EndAddr::Multiple(_) => match state.end_line { + Some(e) => (self.line_no >= e, self.line_no <= e), + None => (false, true), + }, + EndAddr::Last => (self.is_last_line(), true), EndAddr::Regex(re) => { let re = self.resolve(re); - re.is_some_and(|re| re.is_match(&self.ps)) + (re.is_some_and(|re| re.is_match(&self.ps)), true) } }; - let closed = closed || self.is_last_line(); if closed { - self.range[pc] = RangeState::default(); + // A relative end (`+N`, `~N`) is recomputed every time the + // range opens, so skipping past it (via `n`, `N`, `D` or a + // branch) re-arms the start address; an absolute `,N` end is + // spent for good once the stream is past it. + let relative = matches!(end, EndAddr::Plus(_) | EndAddr::Multiple(_)); + self.range[pc] = RangeState { + start_spent: state.start_spent && (selected || !relative), + ..RangeState::default() + }; } - return (true, closed); + if selected || !matches!(end, EndAddr::Plus(_) | EndAddr::Multiple(_)) { + return (selected, selected && closed); + } + // Fall through and retest the start address on this line. } + let state = self.range[pc].clone(); + let one_shot = matches!(addr.start, StartAddr::Line(_) | StartAddr::Zero); let starts = match &addr.start { - StartAddr::Zero => self.line_no == 1, + // A numeric start opens on the first line at or past it, once. + StartAddr::Line(n) => !state.start_spent && self.line_no >= *n, + StartAddr::Zero => !state.start_spent, other => self.start_matches(other), }; if !starts { return (false, false); } + if one_shot { + self.range[pc].start_spent = true; + } let end_line = match end { EndAddr::Line(n) => Some(*n), @@ -590,6 +663,9 @@ impl<'a> Machine<'a> { if matches!(end, EndAddr::Last) && self.is_last_line() { return (true, true); } + // Running off the end of input does NOT close a range: GNU only fires + // `c` when the end address actually matched, so `sed '1,5c\Z'` over two + // lines prints nothing. // `0,/re/` is the one form whose end regex is tested on the start line. if let EndAddr::Regex(re) = end && matches!(addr.start, StartAddr::Zero) @@ -599,13 +675,11 @@ impl<'a> Machine<'a> { return (true, true); } } - if self.is_last_line() { - return (true, true); - } self.range[pc] = RangeState { active: true, end_line, + start_spent: one_shot, }; (true, false) } diff --git a/crates/bashkit/src/builtins/sed/pattern.rs b/crates/bashkit/src/builtins/sed/pattern.rs index 6913e8f40..743e1d9e9 100644 --- a/crates/bashkit/src/builtins/sed/pattern.rs +++ b/crates/bashkit/src/builtins/sed/pattern.rs @@ -322,6 +322,14 @@ impl SedRegex { .map_err(|e| e.to_string()) } + /// Number of capture groups, excluding the whole match. + pub(super) fn group_count(&self) -> usize { + match self { + SedRegex::Standard(re) => re.captures_len().saturating_sub(1), + SedRegex::Fancy(re) => re.captures_len().saturating_sub(1), + } + } + pub(super) fn is_match(&self, text: &str) -> bool { match self { SedRegex::Standard(re) => re.is_match(text), diff --git a/crates/bashkit/src/builtins/sed/script.rs b/crates/bashkit/src/builtins/sed/script.rs index e66502509..d8490f1f6 100644 --- a/crates/bashkit/src/builtins/sed/script.rs +++ b/crates/bashkit/src/builtins/sed/script.rs @@ -283,6 +283,12 @@ impl Parser { } let kind = self.command(cmd_char, addr.is_some())?; + // `q` and `Q` stop the stream, so a range makes no sense for them. + if matches!(kind, Kind::Quit(_) | Kind::QuitSilent(_)) + && addr.as_ref().is_some_and(|a| a.end.is_some()) + { + return self.err("command only uses one address"); + } match &kind { Kind::ReadFile(f) | Kind::ReadLine(f) => read_files.push(f.clone()), _ => {} @@ -737,9 +743,22 @@ impl Parser { } let re = self.compile(&pattern, case_insensitive, multi_line)?; + let parts = parse_replacement(&replacement); + // GNU rejects `s/a/\1/` at compile time rather than substituting an + // empty string, so a typo in a back-reference fails loudly. + if let Some(re) = &re { + let groups = re.group_count(); + for part in &parts { + if let RepPart::Group(n) = part + && *n > groups + { + return self.err(format!("invalid reference \\{n} on `s' command's RHS")); + } + } + } Ok(Kind::Substitute(Box::new(Subst { re, - parts: parse_replacement(&replacement), + parts, occurrence: occurrence.unwrap_or(1), global, print, diff --git a/crates/bashkit/src/builtins/sed/tests.rs b/crates/bashkit/src/builtins/sed/tests.rs index 4261328ca..14c15838e 100644 --- a/crates/bashkit/src/builtins/sed/tests.rs +++ b/crates/bashkit/src/builtins/sed/tests.rs @@ -272,6 +272,55 @@ async fn a_closed_range_stays_closed() { ); } +/// A relative end (`+N`, `~N`) is recomputed on each activation, so a stream +/// carried past it by `n`/`N`/`D`/`b` re-arms the start; an absolute `,N` end +/// is spent for good. Both shapes are GNU behaviour. +#[tokio::test] +async fn ranges_that_the_stream_skipped_past() { + // `1~2N` advances the line counter mid-cycle, so `2,+1` is evaluated at + // lines 2 and 4 and opens both times. + assert_eq!( + out(&["-n", "1~2N;2,+1a\\APP"], "a\nb\nc\nd\ne\n").await, + "APP\nAPP\n" + ); + // An absolute end does not re-arm: `1,2` is spent once line 3 is reached. + assert_eq!( + out(&["-s", "1,2n;1,/b/s/^/>/"], "a\nb\nc\n").await, + "a\n>b\n>c\n" + ); + // A range is only re-tested from its start once the end has been passed. + assert_eq!(out(&["-n", "1b;1,/zz/p"], "a\nb\nc\n").await, "b\nc\n"); + assert_eq!(out(&["-n", "1b;1,2p"], "a\nb\nc\n").await, "b\n"); + assert_eq!(out(&["-n", "1,2b;1,3p"], "a\nb\nc\n").await, "c\n"); + assert_eq!( + out(&["2,~3b;2,~3p"], "a\nb\nc\nd\ne\n").await, + "a\nb\nc\nd\nd\ne\ne\n" + ); +} + +#[tokio::test] +async fn hold_space_carries_its_own_newline_flag() { + // The hold space starts out terminated, so `G` re-terminates a final line + // that arrived without a newline. + assert_eq!(out(&["G"], "ab").await, "ab\n\n"); + assert_eq!(out(&["x"], "a\nb\n").await, "\na\n"); + assert_eq!( + out(&["-n", "1!G;h;$p"], "a\nb\nc\nd\n").await, + "d\nc\nb\na\n" + ); +} + +#[tokio::test] +async fn quit_terminates_the_line_it_printed() { + // Falling off the end of input keeps a missing newline missing, but `q` + // flushes it; `Q`, which prints nothing, does not. + assert_eq!(out(&["q"], "ab").await, "ab\n"); + assert_eq!(out(&["-n", "p;q"], "ab").await, "ab\n"); + assert_eq!(out(&["-n", "p"], "ab").await, "ab"); + assert_eq!(out(&["Q"], "ab").await, ""); + assert_eq!(out(&["2q"], "a\nb").await, "a\nb\n"); +} + #[tokio::test] async fn relative_and_multiple_end_addresses() { let input = (1..=12).map(|n| format!("{n}\n")).collect::(); @@ -288,6 +337,64 @@ async fn change_on_a_range_emits_its_text_once() { // A range that never matches emits nothing. assert_eq!(out(&["/zz/,/b/c\\Z"], "a\nb\nc\n").await, "a\nb\nc\n"); assert_eq!(out(&["2c\\Z"], "a\nb\nc\n").await, "a\nZ\nc\n"); + // `c` ignores -n. + assert_eq!(out(&["-n", "1,2c\\Z"], "a\nb\nc\n").await, "Z\n"); + // Running off the end of input does not close a range, so no text fires. + assert_eq!(out(&["1,5c\\Z"], "a\nb\n").await, ""); + assert_eq!(out(&["2,5c\\Z"], "a\nb\nc\n").await, "a\n"); + assert_eq!(out(&["/a/,/zz/c\\Z"], "a\nb\nc\n").await, ""); + // Inside a block the `c` carries no address of its own, so it fires per line. + assert_eq!(out(&["1,2{c\\Z\n}"], "a\nb\nc\n").await, "Z\nZ\nc\n"); +} + +#[tokio::test] +async fn quit_rejects_a_second_address() { + for script in ["1,2q", "2,+1q", "1,2Q", "/a/,/b/q"] { + let result = run_sed(&[script], Some("a\nb\n")).await; + assert_eq!(result.exit_code, 1, "{script}"); + assert!( + result.stderr.contains("command only uses one address"), + "{script}: {}", + result.stderr + ); + } + // Commands that GNU does allow two addresses on still work. + assert_eq!(out(&["-n", "1,2="], "a\nb\nc\n").await, "1\n2\n"); + assert_eq!(out(&["1,2a\\X"], "a\nb\n").await, "a\nX\nb\nX\n"); +} + +#[tokio::test] +async fn back_references_beyond_the_group_count_are_rejected() { + for script in [r"s/a/\1/", r"s/\(a\)/\2/", r"s/a/[\9]/"] { + let result = run_sed(&[script], Some("a\n")).await; + assert_eq!(result.exit_code, 1, "{script}"); + assert!( + result.stderr.contains("invalid reference"), + "{script}: {}", + result.stderr + ); + } + // In ERE mode `\(` is a literal paren, so there is no group 1 to reference. + let ere = run_sed(&["-E", r"s/\(a\)b/[\1]/"], Some("a\n")).await; + assert_eq!(ere.exit_code, 1); + assert!(ere.stderr.contains("invalid reference"), "{}", ere.stderr); + // A real group is fine. + assert_eq!(out(&["s/\\(a\\)/[\\1]/"], "ab\n").await, "[a]b\n"); +} + +/// THREAT[TM-DOS]: `D` reruns the script without reading input, so `G;D` never +/// terminates and grows the pattern space without bound. GNU hangs here; the +/// sandbox must not. +#[tokio::test] +async fn a_non_progressing_delete_restart_loop_terminates() { + let result = run_sed(&["G;D"], Some("a\nb\n")).await; + assert_eq!(result.exit_code, 0); + assert!(result.stderr.contains("loop limit"), "{}", result.stderr); + // A `D` loop that does make progress still terminates on its own. + let progressing = run_sed(&["-s", "1,2!h;2,+1G;2,$D"], Some("abc\nABC\n")).await; + assert_eq!(progressing.exit_code, 0); + assert_eq!(progressing.stdout, "abc\n\n\n"); + assert!(progressing.stderr.is_empty(), "{}", progressing.stderr); } // === finding D: operands are one stream ================================= diff --git a/crates/bashkit/tests/spec_cases/sed/sed.test.sh b/crates/bashkit/tests/spec_cases/sed/sed.test.sh index 93bda90e7..166808f6b 100644 --- a/crates/bashkit/tests/spec_cases/sed/sed.test.sh +++ b/crates/bashkit/tests/spec_cases/sed/sed.test.sh @@ -900,3 +900,30 @@ printf 'x\n' | sed --quiet -e p y x ### end + +### sed_change_range_needs_a_real_end +# A range that runs off the end of input never "ends", so c emits nothing +printf 'a\nb\n' | sed '1,5c\Z' +printf 'a\nb\nc\n' | sed '2,5c\Z' +printf 'a\nb\nc\n' | sed '/a/,/zz/c\Z' +### expect +a +### end + +### sed_quit_takes_one_address +# q and Q stop the stream, so a range is a compile error +printf 'a\nb\n' | sed '1,2q' 2>&1 +echo "rc=$?" +### expect +sed: -e expression #1, char 4: command only uses one address +rc=1 +### end + +### sed_invalid_backreference_is_rejected +# GNU rejects \1 with no group instead of substituting an empty string +printf 'a\n' | sed 's/a/\1/' 2>&1 +echo "rc=$?" +### expect +sed: -e expression #1, char 7: invalid reference \1 on `s' command's RHS +rc=1 +### end diff --git a/knowledge/operations/limitations.md b/knowledge/operations/limitations.md index 67a75a04f..8374cbbb8 100644 --- a/knowledge/operations/limitations.md +++ b/knowledge/operations/limitations.md @@ -132,6 +132,8 @@ pass in CI); only divergences and boundaries are recorded here. | L-SED-002 | sed | The `e` command and the `s///e` flag (run the pattern space as a shell command) are refused at compile time, not silently ignored | `the_e_command_is_refused_not_silently_ignored` | | L-SED-003 | sed | `-i` refuses a file that is not valid UTF-8 instead of rewriting it through a lossy decode; without `-i`, non-UTF-8 input is decoded with replacement like every other text builtin (L-STREAM-001) | `in_place_refuses_non_utf8_rather_than_corrupting` | | L-SED-004 | sed | `\<` and `\>` (GNU word-edge operators) both compile to `\b`; the linear-time `regex` engine has no look-around, so a word start and a word end are not distinguished | `translate_is_positional` | +| L-SED-005 | sed | `D` restarts the script without reading input, so a non-progressing loop such as `sed 'G;D'` never terminates — GNU hangs on it. Restarts are capped at `SED_MAX_CYCLE_STEPS`, after which sed stops with the same `loop limit` warning `b`/`t` loops get, and exits 0. A `D` loop that does make progress is unaffected | `a_non_progressing_delete_restart_loop_terminates` | +| L-SED-006 | sed | Scripts that carry the stream past a range's end with `n`, `N` or `D` can still diverge from GNU's range bookkeeping in rare shapes (e.g. `sed '$!D;1,2!x'`). A randomized differential sweep of 18,000 generated scripts against GNU sed 4.9 finds 3 such cases; all combine a multi-line pattern space with a range address, a shape absent from ordinary scripts | `ranges_that_the_stream_skipped_past` | | L-CURL-001 | curl | Spec-test coverage for methods/headers/auth/redirects not ported (needs `http_client` + allowlist in harness); payload behavior has integration and real-curl differential coverage | stance | | L-CURL-002 | curl/wget | Unknown options are ignored for compatibility, not rejected (real curl/wget error); deliberate leniency | `curl.rs` | | L-STR-001 | strings | Accepts dash-prefixed filenames (e.g. `-data.bin`), so only a lone unknown short option (`-Q`) is rejected as invalid; GNU rejects `-data.bin` too | `strings.rs` | From 50b6411202e5040da7ac260c9cdd1f6fc0aa8745 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Mon, 21 Sep 2026 23:25:27 +0000 Subject: [PATCH 3/3] test(sed): cover every long option, and name two unreachable match arms A coverage audit over the parser's own dispatch tables found the gap: every sed command and short option is exercised by a test, but eight long spellings (`--silent`, `--regexp-extended`, `--separate`, `--file`, `--line-length`, `--zero-terminated`, `--follow-symlinks`) and the inert `-u`/`--unbuffered`, `--posix`, `--debug`, `--sandbox` were only reachable through code, so any of them could rot into "silently ignored" unnoticed. Adds two tests driven off that dispatch table, so the audit now comes back clean, and comments the two match arms that exist only for exhaustiveness. --- crates/bashkit/src/builtins/sed/exec.rs | 4 +++ crates/bashkit/src/builtins/sed/tests.rs | 46 ++++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/crates/bashkit/src/builtins/sed/exec.rs b/crates/bashkit/src/builtins/sed/exec.rs index 8cba60552..729adedd5 100644 --- a/crates/bashkit/src/builtins/sed/exec.rs +++ b/crates/bashkit/src/builtins/sed/exec.rs @@ -457,6 +457,7 @@ impl<'a> Machine<'a> { let text = format!("{name}\n"); self.out.raw(&text); } + // Handled above, before the address gate; listed for exhaustiveness. Kind::Block(_) | Kind::BlockEnd | Kind::Label(_) | Kind::Nop => {} } @@ -687,6 +688,9 @@ impl<'a> Machine<'a> { fn start_matches(&mut self, start: &StartAddr) -> bool { match start { StartAddr::Line(n) => self.line_no == *n, + // `0` is only legal as the start of `0,/re/`, which + // `address_matches` handles before reaching here; the parser + // rejects a bare `0` address. StartAddr::Zero => false, StartAddr::Last => self.is_last_line(), StartAddr::Step(first, step) => { diff --git a/crates/bashkit/src/builtins/sed/tests.rs b/crates/bashkit/src/builtins/sed/tests.rs index 14c15838e..d7b1175d6 100644 --- a/crates/bashkit/src/builtins/sed/tests.rs +++ b/crates/bashkit/src/builtins/sed/tests.rs @@ -659,6 +659,52 @@ async fn short_options_cluster_and_take_arguments() { assert_eq!(out(&["-n", "--expression", "p"], "x\n").await, "x\n"); } +/// Every long option and every accepted-but-inert short option, so none of +/// them can rot into "silently ignored" without a test noticing. +#[tokio::test] +async fn every_long_option_is_exercised() { + assert_eq!(out(&["--silent", "-e", "p"], "x\n").await, "x\n"); + assert_eq!(out(&["--quiet", "-e", "p"], "x\n").await, "x\n"); + assert_eq!(out(&["--regexp-extended", "s/a+/X/"], "aa\n").await, "X\n"); + assert_eq!(out(&["--expression=s/x/y/"], "x\n").await, "y\n"); + assert_eq!( + out(&["--line-length=5", "-n", "l"], "abcdefgh\n").await, + "abcd\\\nefgh$\n" + ); + // -u / --unbuffered and --follow-symlinks are accepted and inert. + assert_eq!(out(&["-u", "s/a/b/"], "a\n").await, "b\n"); + assert_eq!(out(&["--unbuffered", "s/a/b/"], "a\n").await, "b\n"); + assert_eq!(out(&["--follow-symlinks", "s/a/b/"], "a\n").await, "b\n"); + assert_eq!(out(&["--posix", "s/a/b/"], "a\n").await, "b\n"); + assert_eq!(out(&["--debug", "s/a/b/"], "a\n").await, "b\n"); + assert_eq!(out(&["--sandbox", "s/a/b/"], "a\n").await, "b\n"); + // --null-data has two spellings. + assert_eq!(out(&["--null-data", "s/a/X/g"], "a\0a\0").await, "X\0X\0"); + assert_eq!( + out(&["--zero-terminated", "s/a/X/g"], "a\0a\0").await, + "X\0X\0" + ); +} + +#[tokio::test] +async fn long_option_separate_and_file() { + let fs = Arc::new(InMemoryFs::new()); + fs.write_file(std::path::Path::new("/f1"), b"a\nb\n") + .await + .unwrap(); + fs.write_file(std::path::Path::new("/f2"), b"c\n") + .await + .unwrap(); + let sep = run_with(fs.clone(), &["--separate", "-n", "$p", "/f1", "/f2"], None).await; + assert_eq!(sep.stdout, "b\nc\n"); + + fs.write_file(std::path::Path::new("/sc.sed"), b"s/a/X/\n") + .await + .unwrap(); + let from_file = run_with(fs, &["--file=/sc.sed"], Some("a\n")).await; + assert_eq!(from_file.stdout, "X\n"); +} + #[tokio::test] async fn an_empty_script_is_a_passthrough() { let result = run_sed(&[""], Some("a\n")).await;