diff --git a/crates/bashkit/src/builtins/mapfile.rs b/crates/bashkit/src/builtins/mapfile.rs index 35c32ea2d..994638967 100644 --- a/crates/bashkit/src/builtins/mapfile.rs +++ b/crates/bashkit/src/builtins/mapfile.rs @@ -1,6 +1,10 @@ -//! mapfile/readarray builtin — read lines from stdin into an array. +//! mapfile/readarray builtin — read records from stdin into an array. //! //! Mutates arrays via [`BuiltinSideEffect::SetIndexedArray`](super::BuiltinSideEffect). +//! +//! Decision: `-u FD` other than 0 and the `-C`/`-c` callback are rejected with +//! status 2 instead of being ignored; reading the wrong input silently would +//! be worse than a clear error (see L-MAPFILE-001 in operations/limitations). use async_trait::async_trait; @@ -8,65 +12,128 @@ use super::{Builtin, BuiltinSideEffect, Context}; use crate::error::Result; use crate::interpreter::ExecResult; -/// `mapfile`/`readarray` builtin — read lines from stdin into an indexed array. +/// `mapfile`/`readarray` builtin — read records from stdin into an indexed array. /// -/// Usage: mapfile [-t] [ARRAY] +/// Usage: mapfile [-d DELIM] [-n COUNT] [-O ORIGIN] [-s COUNT] [-t] [ARRAY] /// -/// - `-t` — strip trailing newlines from each line +/// - `-d DELIM` — end records at the first char of DELIM (`''` means NUL) +/// - `-n COUNT` — copy at most COUNT records (0 means all) +/// - `-O ORIGIN` — start at index ORIGIN and keep existing elements +/// - `-s COUNT` — skip the first COUNT records +/// - `-t` — strip the delimiter from each record /// - Default array name is `MAPFILE` pub struct Mapfile; -#[async_trait] -impl Builtin for Mapfile { - async fn execute(&self, ctx: Context<'_>) -> Result { - let mut trim_trailing = false; // -t: strip trailing newlines - let mut array_name = "MAPFILE".to_string(); - let mut positional = Vec::new(); +struct Opts { + delim: char, + count: usize, + origin: Option, + skip: usize, + trim: bool, + name: String, +} - for arg in ctx.args { - match arg.as_str() { - "-t" => trim_trailing = true, - a if a.starts_with('-') => {} // skip unknown flags - _ => positional.push(arg.clone()), - } - } +fn parse_count(opt: char, v: &str) -> std::result::Result { + v.trim().parse::().map_err(|_| { + let what = if opt == 'O' { "origin" } else { "line count" }; + format!("{v}: invalid {what}") + }) +} - if let Some(name) = positional.first() { - array_name = name.clone(); +fn parse_opts(args: &[String]) -> std::result::Result { + let mut o = Opts { + delim: '\n', + count: 0, + origin: None, + skip: 0, + trim: false, + name: "MAPFILE".to_string(), + }; + let mut i = 0; + while i < args.len() { + let arg = &args[i]; + i += 1; + if arg == "--" { + break; + } + if !arg.starts_with('-') || arg == "-" { + i -= 1; + break; + } + let mut chars = arg[1..].chars(); + while let Some(c) = chars.next() { + match c { + 't' => o.trim = true, + 'd' | 'n' | 'O' | 's' | 'u' | 'C' | 'c' => { + let rest: String = chars.by_ref().collect(); + let value = if !rest.is_empty() { + rest + } else if let Some(v) = args.get(i) { + i += 1; + v.clone() + } else { + return Err(format!("-{c}: option requires an argument")); + }; + match c { + 'd' => o.delim = value.chars().next().unwrap_or('\0'), + 'n' => o.count = parse_count(c, &value)?, + 'O' => o.origin = Some(parse_count(c, &value)?), + 's' => o.skip = parse_count(c, &value)?, + 'u' if value.trim() == "0" => {} + 'u' => return Err(format!("-u {value}: only fd 0 is supported")), + _ => return Err(format!("-{c}: callbacks are not supported")), + } + } + _ => return Err(format!("-{c}: invalid option")), + } } + } + if let Some(name) = args.get(i) { + o.name = name.clone(); + } + Ok(o) +} +#[async_trait] +impl Builtin for Mapfile { + async fn execute(&self, ctx: Context<'_>) -> Result { + let o = match parse_opts(ctx.args) { + Ok(o) => o, + Err(msg) => return Ok(ExecResult::err(format!("mapfile: {msg}\n"), 2)), + }; let input = ctx.stdin.map(|stdin| &**stdin).unwrap_or(""); let mut result = ExecResult::ok(String::new()); + // Without -O the array is cleared first. + if o.origin.is_none() { + result + .side_effects + .push(BuiltinSideEffect::RemoveArray(o.name.clone())); + } - // Always remove existing array first - result - .side_effects - .push(BuiltinSideEffect::RemoveArray(array_name.clone())); - - // Split into lines and populate array - if !input.is_empty() { - let entries: Vec<(usize, String)> = input - .lines() - .enumerate() - .map(|(idx, line)| { - let value = if trim_trailing { - line.to_string() - } else { - format!("{}\n", line) - }; - (idx, value) - }) - .collect(); + let start = o.origin.unwrap_or(0); + let entries: Vec<(usize, String)> = input + .split_inclusive(o.delim) + .skip(o.skip) + .take(if o.count == 0 { usize::MAX } else { o.count }) + .enumerate() + .map(|(idx, rec)| { + let value = if o.trim { + rec.strip_suffix(o.delim).unwrap_or(rec) + } else { + rec + }; + (start + idx, value.to_string()) + }) + .collect(); - if !entries.is_empty() { - result - .side_effects - .push(BuiltinSideEffect::SetIndexedArray { - name: array_name, - entries, - }); - } + if !entries.is_empty() { + result + .side_effects + .push(BuiltinSideEffect::SetIndexedArray { + name: o.name, + entries, + }); } Ok(result) diff --git a/crates/bashkit/src/interpreter/expansion.rs b/crates/bashkit/src/interpreter/expansion.rs index 2f3a8e4f9..e7a6b6068 100644 --- a/crates/bashkit/src/interpreter/expansion.rs +++ b/crates/bashkit/src/interpreter/expansion.rs @@ -8,6 +8,14 @@ use super::*; +/// Case change for `${v^}`, `${v,}` and `${v~}`. +#[derive(Clone, Copy)] +enum CaseChange { + Upper, + Lower, + Toggle, +} + impl Interpreter { /// Expand an array access expression (`${arr[index]}`). pub(super) fn expand_array_access_part(&self, name: &str, index: &str) -> String { @@ -698,14 +706,7 @@ impl Interpreter { .unwrap_or_default(); for (idx, part) in word.parts.iter().enumerate() { let part_is_quoted = word.part_quoted.get(idx).copied().unwrap_or(word.quoted); - let part_has_expansion = matches!( - part, - WordPart::Variable(_) - | WordPart::CommandSubstitution(_) - | WordPart::ArithmeticExpansion(_) - | WordPart::ParameterExpansion { .. } - | WordPart::ArrayAccess { .. } - ); + let part_has_expansion = Self::is_field_split_expansion(part); let value = if idx > 0 && let WordPart::Literal(s) = part { @@ -792,16 +793,7 @@ impl Interpreter { matches!(word.parts.first(), Some(WordPart::Literal(s)) if s.contains('=')); let has_expansion = !word.quoted && !is_assignment_word - && word.parts.iter().any(|p| { - matches!( - p, - WordPart::Variable(_) - | WordPart::CommandSubstitution(_) - | WordPart::ArithmeticExpansion(_) - | WordPart::ParameterExpansion { .. } - | WordPart::ArrayAccess { .. } - ) - }); + && word.parts.iter().any(Self::is_field_split_expansion); if has_expansion { self.ifs_split(&expanded) @@ -811,6 +803,23 @@ impl Interpreter { }) } + /// Expansion parts whose unquoted result undergoes IFS field splitting. + fn is_field_split_expansion(part: &WordPart) -> bool { + matches!( + part, + WordPart::Variable(_) + | WordPart::CommandSubstitution(_) + | WordPart::ArithmeticExpansion(_) + | WordPart::ParameterExpansion { .. } + | WordPart::ArrayAccess { .. } + | WordPart::IndirectExpansion { .. } + | WordPart::PrefixMatch(_) + | WordPart::Substring { .. } + | WordPart::ArraySlice { .. } + | WordPart::Transformation { .. } + ) + } + /// Resolve name for parameter expansion, handling array subscripts and special params. /// Returns (is_set, expanded_value). pub(super) fn resolve_param_expansion_name(&self, name: &str) -> (bool, String) { @@ -1457,6 +1466,8 @@ impl Interpreter { | ParameterOp::UpperAll | ParameterOp::LowerFirst | ParameterOp::LowerAll + | ParameterOp::ToggleFirst + | ParameterOp::ToggleAll ) } @@ -1596,17 +1607,19 @@ impl Interpreter { let expanded_pat = self.expand_replace_pattern(pattern); self.replace_pattern(value, &expanded_pat, &expanded_rep, true) } - ParameterOp::UpperFirst => self.change_case(value, operand, true, false), - ParameterOp::UpperAll => self.change_case(value, operand, true, true), - ParameterOp::LowerFirst => self.change_case(value, operand, false, false), - ParameterOp::LowerAll => self.change_case(value, operand, false, true), + ParameterOp::UpperFirst => self.change_case(value, operand, CaseChange::Upper, false), + ParameterOp::UpperAll => self.change_case(value, operand, CaseChange::Upper, true), + ParameterOp::LowerFirst => self.change_case(value, operand, CaseChange::Lower, false), + ParameterOp::LowerAll => self.change_case(value, operand, CaseChange::Lower, true), + ParameterOp::ToggleFirst => self.change_case(value, operand, CaseChange::Toggle, false), + ParameterOp::ToggleAll => self.change_case(value, operand, CaseChange::Toggle, true), } } /// `${v^pat}`, `${v^^pat}`, `${v,pat}`, `${v,,pat}`: change the case of /// the first (or every) character that matches `pat` on its own. An empty /// pattern matches any character, as in bash. - fn change_case(&mut self, value: &str, operand: &str, upper: bool, all: bool) -> String { + fn change_case(&mut self, value: &str, operand: &str, mode: CaseChange, all: bool) -> String { let pattern = if operand.is_empty() { String::new() } else { @@ -1619,7 +1632,14 @@ impl Interpreter { && (pattern.is_empty() || self.pattern_matches(ch.encode_utf8(&mut buf), &pattern)); if !selected { out.push(ch); - } else if upper { + continue; + } + let upper = match mode { + CaseChange::Upper => true, + CaseChange::Lower => false, + CaseChange::Toggle => !ch.is_uppercase(), + }; + if upper { out.extend(ch.to_uppercase()); } else { out.extend(ch.to_lowercase()); diff --git a/crates/bashkit/src/interpreter/mod.rs b/crates/bashkit/src/interpreter/mod.rs index 891dea7cc..80b22d9e2 100644 --- a/crates/bashkit/src/interpreter/mod.rs +++ b/crates/bashkit/src/interpreter/mod.rs @@ -8812,6 +8812,8 @@ impl Interpreter { ParameterOp::UpperAll => out.push_str(&format!("${{{}^^{}}}", name, operand)), ParameterOp::LowerFirst => out.push_str(&format!("${{{},{}}}", name, operand)), ParameterOp::LowerAll => out.push_str(&format!("${{{},,{}}}", name, operand)), + ParameterOp::ToggleFirst => out.push_str(&format!("${{{}~{}}}", name, operand)), + ParameterOp::ToggleAll => out.push_str(&format!("${{{}~~{}}}", name, operand)), }, WordPart::Length(name) => out.push_str(&format!("${{#{}}}", name)), WordPart::ArrayAccess { name, index } => { @@ -10400,9 +10402,10 @@ impl Interpreter { self.insert_array_checked(name.clone(), arr); } builtins::BuiltinSideEffect::SetIndexedArray { name, entries } => { - let arr: HashMap = entries.iter().cloned().collect(); - // Remove existing array first (mirrors mapfile behavior) - self.arrays_mut().remove(name); + // Merges into the existing array; mapfile sends RemoveArray + // first unless `-O` asked to keep the other elements. + let mut arr = self.arrays_mut().remove(name).unwrap_or_default(); + arr.extend(entries.iter().cloned()); if !arr.is_empty() { self.insert_array_checked(name.clone(), arr); } diff --git a/crates/bashkit/src/interpreter/state.rs b/crates/bashkit/src/interpreter/state.rs index c46d41ed9..639e2db8e 100644 --- a/crates/bashkit/src/interpreter/state.rs +++ b/crates/bashkit/src/interpreter/state.rs @@ -33,7 +33,8 @@ pub enum BuiltinSideEffect { SetPositional(Vec), /// Populate an indexed array variable (replaces `_ARRAY_READ_*`). SetArray { name: String, elements: Vec }, - /// Populate an indexed array with index->value pairs (for mapfile). + /// Set index->value pairs in an indexed array, keeping its other elements + /// (for mapfile, which sends `RemoveArray` first unless `-O` is given). SetIndexedArray { name: String, entries: Vec<(usize, String)>, diff --git a/crates/bashkit/src/parser/ast.rs b/crates/bashkit/src/parser/ast.rs index ff7189f99..406ca4ce8 100644 --- a/crates/bashkit/src/parser/ast.rs +++ b/crates/bashkit/src/parser/ast.rs @@ -353,6 +353,8 @@ impl fmt::Display for Word { ParameterOp::UpperAll => write!(f, "${{{}^^{}}}", name, operand)?, ParameterOp::LowerFirst => write!(f, "${{{},{}}}", name, operand)?, ParameterOp::LowerAll => write!(f, "${{{},,{}}}", name, operand)?, + ParameterOp::ToggleFirst => write!(f, "${{{}~{}}}", name, operand)?, + ParameterOp::ToggleAll => write!(f, "${{{}~~{}}}", name, operand)?, }, WordPart::Length(name) => write!(f, "${{#{}}}", name)?, WordPart::ArrayAccess { name, index } => write!(f, "${{{}[{}]}}", name, index)?, @@ -511,6 +513,10 @@ pub enum ParameterOp { LowerFirst, /// ,, lowercase all chars LowerAll, + /// ~ toggle case of first char + ToggleFirst, + /// ~~ toggle case of all chars + ToggleAll, } /// I/O redirection. diff --git a/crates/bashkit/src/parser/mod.rs b/crates/bashkit/src/parser/mod.rs index 768ed8107..28b025df0 100644 --- a/crates/bashkit/src/parser/mod.rs +++ b/crates/bashkit/src/parser/mod.rs @@ -3526,7 +3526,8 @@ impl<'a> Parser<'a> { operand, colon_variant: false, }); - } else if matches!(next_c, '#' | '%' | '/' | '^' | ',' | '@') { + } else if matches!(next_c, '#' | '%' | '/' | '^' | ',' | '~' | '@') + { var_name = format!("{}[{}]", var_name, index); subscript_op = true; } else { @@ -3720,6 +3721,11 @@ impl<'a> Parser<'a> { } else if ch == '\\' { repl.push(chars.next().unwrap()); if let Some(n) = chars.next() { + // `\/` is a literal slash, as in the + // pattern (bash removes the backslash). + if n == '/' { + repl.pop(); + } repl.push(n); } continue; @@ -3770,6 +3776,24 @@ impl<'a> Parser<'a> { colon_variant: false, }); } + '~' => { + // `${v~}` / `${v~~}`: toggle case (undocumented + // in the bash manual, supported by bash 4+). + chars.next(); + let op = if chars.peek() == Some(&'~') { + chars.next(); + ParameterOp::ToggleAll + } else { + ParameterOp::ToggleFirst + }; + let operand = self.read_brace_operand(&mut chars); + push_part!(WordPart::ParameterExpansion { + name: var_name, + operator: op, + operand, + colon_variant: false, + }); + } ',' => { chars.next(); let op = if chars.peek() == Some(&',') { diff --git a/crates/bashkit/tests/spec_cases/bash/bashbox-interpreter-core.test.sh b/crates/bashkit/tests/spec_cases/bash/bashbox-interpreter-core.test.sh index cfb628064..ff6fa9635 100644 --- a/crates/bashkit/tests/spec_cases/bash/bashbox-interpreter-core.test.sh +++ b/crates/bashkit/tests/spec_cases/bash/bashbox-interpreter-core.test.sh @@ -874,7 +874,6 @@ printf "a\0b\0" | while read -r -d "" v; do echo "[$v]"; done ### bashbox_interpreter_core_mapfile_on_nul_delimiters # mapfile on NUL delimiters -### skip: TODO bashbox corpus gap, bashkit output differs from real bash printf "a\0b\0" | { mapfile -d "" arr; echo ${#arr[@]}; } ### expect 2 diff --git a/crates/bashkit/tests/spec_cases/bash/param-expansion-gaps.test.sh b/crates/bashkit/tests/spec_cases/bash/param-expansion-gaps.test.sh new file mode 100644 index 000000000..ed875c418 --- /dev/null +++ b/crates/bashkit/tests/spec_cases/bash/param-expansion-gaps.test.sh @@ -0,0 +1,65 @@ +### toggle_case +s="hello World" +echo "${s~~}|${s~}" +a=(abC Def); echo "${a[@]~~}" +### expect +HELLO wORLD|Hello World +ABc dEF +### end + +### replace_escaped_slash +p="a.b.c" +echo "${p//./\/}" ${p//./\/} "${p/./\/}" +### expect +a/b/c a/b/c a/b.c +### end + +### prefix_names_split_in_for +cfg_host=localhost cfg_port=80 other=1 +for v in ${!cfg_@}; do echo "$v=${!v}"; done +n=0; for v in ${!cfg_*}; do n=$((n+1)); done; echo $n +set -- ${!cfg_@}; echo $# +### expect +cfg_host=localhost +cfg_port=80 +2 +2 +### end + +### mapfile_delim +mapfile -t -d , campos <<< "a,b,c" +printf '[%s]' "${campos[@]}"; echo +mapfile -d , keep <<< "x,y" +printf '<%s>' "${keep[@]}"; echo +### expect +[a][b][c +] + +### end + +### mapfile_skip_count_origin +printf 'l0\nl1\nl2\nl3\n' > /tmp/mf.txt +readarray -t -s 1 -n 2 parte < /tmp/mf.txt +echo "${parte[*]}" +arr=(keep) +mapfile -t -O 2 arr < /tmp/mf.txt +declare -p arr +mapfile -tn1 one < /tmp/mf.txt; echo "${#one[@]} ${one[0]}" +### expect +l1 l2 +declare -a arr=([0]="keep" [2]="l0" [3]="l1" [4]="l2" [5]="l3") +1 l0 +### end + +### mapfile_nul_delim +printf 'a\0b\0' | { mapfile -d '' arr; echo ${#arr[@]}; } +### expect +2 +### end + +### mapfile_bad_option +mapfile -x arr; echo "rc=$?" +### expect +rc=2 +### end diff --git a/knowledge/log.md b/knowledge/log.md index 2a0aa2eaa..fc6b8fbc7 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -10,6 +10,7 @@ * **Fix**: array literals expand like command words. `arr=(*.txt x{1,2})` now runs brace and pathname expansion (honoring `nullglob`; quoted words stay literal), and `arr=([2]=x y)` sets indexed elements instead of storing `[2]=x`. Under `failglob` an unmatched pattern is stored as-is rather than aborting (WTF in code). `command -v a b ...` reports every name found and fails only if none was. `declare -a d=(...)` is unchanged here (declare/local work is in a separate branch). * **Fix**: the `bashkit` CLI exits 2 (was 1) when a script or `-c` string has a syntax error before any runnable command, printing `bash: syntax error: ...` instead of the anyhow `Error: Failed to execute script` chain. Matches the partial-run path, which already exited 2. See foundations/parser "Partial execution before a syntax error". * **Fix**: `eval` with a syntax error exits 2 like bash and reports `bash: eval: line N: syntax error: ...` plus the offending line, each newline-terminated, with the eval's own redirects applied (`$(eval bad 2>&1)` captures it). `$LINENO` inside `$( )` now counts from the outer line instead of restarting at 1. The parser's message text still differs from bash's `near unexpected token` wording. +* **Fix**: parameter expansion and `mapfile` gaps from the Debian oracle. `${v~}`/`${v~~}` toggle case (also per array element). `\/` in a `${v/pat/rep}` replacement is a plain `/`. Unquoted `${!prefix@}`, `${!ref}`, `${v:off:len}`, `${a[@]:o:n}` and `${v@op}` now undergo field splitting like other expansions, so `for v in ${!cfg_@}` loops per name. `mapfile` gained `-d DELIM` (`''` is NUL), `-n`, `-s`, `-O` (keeps other elements) and grouped options; `-u` other than 0 and `-C`/`-c` exit 2 (L-MAPFILE-001). * **Decision**: CPython's workload priority is agents running `python3` for scripted logic (processing a file, transforming data). Low-use stdlib modules are no longer shipped: `unittest`, `doctest`, profilers, `compileall`/`zipapp`, `dbm`/`shelve`, `plistlib`, `wave`, `netrc`, `cmd`, `tty`/`pty`, and `bz2`/`lzma`/`compression.zstd` (no C codec in the guest anyway). Stdlib zip 3.58 to 3.16 MB. L-CPY-009 updated. See [CPython WebAssembly Runtime](runtimes/cpython-wasm.md#workload-priority). * **Feature**: `yes` and `seq` stream into pipelines. As a producer stage they write chunks into the pipe as they go (`Context::stdout_stream`), so `yes | head -1` and `seq 1000000 | head -1` stop after the first chunk with `PIPESTATUS` `141 0` instead of running to the output cap and printing a truncation notice. Step 2 of the roadmap (streaming builtin I/O), starting with the generators. * **Feature**: `sudo` and `busybox`, the top missing commands in the gap replay (`sudo tee /etc/x <