diff --git a/crates/bashkit/docs/compatibility.md b/crates/bashkit/docs/compatibility.md index 9653e404b..d8ff763e0 100644 --- a/crates/bashkit/docs/compatibility.md +++ b/crates/bashkit/docs/compatibility.md @@ -282,6 +282,9 @@ Chrono's validated strftime implementation, plus GNU `%N`, `%3N`, `%6N`, and | `${var^^}` | ✅ | `${s^^}` | Uppercase all | | `${var,}` | ✅ | `${s,}` | Lowercase first | | `${var,,}` | ✅ | `${s,,}` | Lowercase all | +| `${var^^pat}` | ✅ | `${s^^[aeiou]}` | Case change only for characters matching `pat` (also `^`, `,`, `,,`) | +| `${var@op}` | ✅ | `${s@Q}` | Transform: `Q` quote, `E` escapes, `U`/`u`/`L` case, `A` assignment, `a` attributes | +| `${arr[@]op}` | ✅ | `${a[@]/x/y}`, `${a[@]^^}`, `${a[@]@Q}` | Pattern, case and transform operators apply per element (also `${a[i]op}`, `$@`, `$*`) | ### Prefix Environment Assignments diff --git a/crates/bashkit/src/interpreter/expansion.rs b/crates/bashkit/src/interpreter/expansion.rs index 6eb8dc095..b790337b2 100644 --- a/crates/bashkit/src/interpreter/expansion.rs +++ b/crates/bashkit/src/interpreter/expansion.rs @@ -66,16 +66,31 @@ impl Interpreter { /// Apply a `${var@operator}` transformation. pub(super) fn apply_transformation(&self, name: &str, operator: char) -> String { - let value = self.expand_variable(name); + // `${a[@]@Q}`, `${@@U}`: value transforms apply to each element. + if matches!(operator, 'Q' | 'E' | 'P' | 'u' | 'U' | 'L') + && let Some(elems) = self.resolve_param_expansion_elements(name) + { + // THREAT[TM-DOS]: same output cap as per-element pattern ops. + let mut out = String::new(); + for v in &elems { + let t = Self::transform_value(v, operator); + if out.len().saturating_add(t.len() + 1) > Self::MAX_EXPANSION_RESULT_BYTES { + return elems.join(" "); + } + if !out.is_empty() { + out.push(' '); + } + out.push_str(&t); + } + return out; + } + let value = if name.contains('[') { + self.resolve_param_expansion_name(name).1 + } else { + self.expand_variable(name) + }; match operator { - 'Q' => format!("'{}'", value.replace('\'', "'\\''")), - 'E' => value - .replace("\\n", "\n") - .replace("\\t", "\t") - .replace("\\\\", "\\"), - 'P' => value.clone(), 'A' => format!("{}='{}'", name, value.replace('\'', "'\\''")), - 'K' => value.clone(), 'a' => { let mut attrs = String::new(); if self.is_var_readonly(name) { @@ -86,19 +101,28 @@ impl Interpreter { } attrs } - 'u' | 'U' => { - if operator == 'U' { - value.to_uppercase() - } else { - let mut chars = value.chars(); - match chars.next() { - Some(first) => first.to_uppercase().collect::() + chars.as_str(), - None => String::new(), - } + _ => Self::transform_value(&value, operator), + } + } + + /// The `${v@op}` transforms that depend only on the value. + fn transform_value(value: &str, operator: char) -> String { + match operator { + 'Q' => format!("'{}'", value.replace('\'', "'\\''")), + 'E' => value + .replace("\\n", "\n") + .replace("\\t", "\t") + .replace("\\\\", "\\"), + 'U' => value.to_uppercase(), + 'u' => { + let mut chars = value.chars(); + match chars.next() { + Some(first) => first.to_uppercase().collect::() + chars.as_str(), + None => String::new(), } } 'L' => value.to_lowercase(), - _ => value.clone(), + _ => value.to_string(), } } @@ -596,6 +620,20 @@ impl Interpreter { } return Ok(Vec::new()); } + // "${a[@]/x/y}", "${@^}", "${a[@]@Q}": one field per element. + if let Some((elems, star)) = self.elementwise_fields(&word.parts[0]) { + if word.quoted { + if star { + return Ok(vec![elems.join(&self.get_ifs_separator())]); + } + return Ok(elems); + } + let mut fields = Vec::new(); + for e in &elems { + fields.extend(self.ifs_split(e)?); + } + return Ok(fields); + } // "${!arr[@]}" - array keys/indices as separate fields if let WordPart::ArrayIndices(name) = &word.parts[0] { let resolved = self.resolve_nameref(name); @@ -1280,16 +1318,62 @@ impl Interpreter { /// /// Extracted from the async `expand_word_inner` path to keep `Vec` /// locals off the async state machine (prevents stack overflow at depth 32). - pub(super) fn apply_param_op_maybe_per_element( - &mut self, - value: &str, - name: &str, - operator: &ParameterOp, - operand: &str, - colon_variant: bool, - is_set: bool, - ) -> String { - let needs_per_element = matches!( + /// Per-element results of a pattern, case or transform operator applied + /// to `@`, `*`, `a[@]` or `a[*]`, plus whether the subscript was `*`. + /// Sync so the async field expansion keeps its small state machine. + fn elementwise_fields(&mut self, part: &WordPart) -> Option<(Vec, bool)> { + let (name, results) = match part { + WordPart::ParameterExpansion { + name, + operator, + operand, + colon_variant, + } if Self::is_elementwise_op(operator) => { + let elems = self.resolve_param_expansion_elements(name)?; + let (is_set, _) = self.resolve_param_expansion_name(name); + let mut out = Vec::with_capacity(elems.len()); + let mut total = 0usize; + for elem in &elems { + let r = self.apply_parameter_op( + elem, + name, + operator, + operand, + *colon_variant, + is_set, + ); + // THREAT[TM-DOS]: same cap as the joined per-element path. + total = total.saturating_add(r.len() + 1); + if total > Self::MAX_EXPANSION_RESULT_BYTES { + return None; + } + out.push(r); + } + (name, out) + } + WordPart::Transformation { name, operator } + if matches!(operator, 'Q' | 'E' | 'P' | 'u' | 'U' | 'L') => + { + let elems = self.resolve_param_expansion_elements(name)?; + let mut out = Vec::with_capacity(elems.len()); + let mut total = 0usize; + for elem in &elems { + let r = Self::transform_value(elem, *operator); + total = total.saturating_add(r.len() + 1); + if total > Self::MAX_EXPANSION_RESULT_BYTES { + return None; + } + out.push(r); + } + (name, out) + } + _ => return None, + }; + Some((results, name == "*" || name.ends_with("[*]"))) + } + + fn is_elementwise_op(operator: &ParameterOp) -> bool { + matches!( operator, ParameterOp::RemovePrefixShort | ParameterOp::RemovePrefixLong @@ -1301,7 +1385,19 @@ impl Interpreter { | ParameterOp::UpperAll | ParameterOp::LowerFirst | ParameterOp::LowerAll - ); + ) + } + + pub(super) fn apply_param_op_maybe_per_element( + &mut self, + value: &str, + name: &str, + operator: &ParameterOp, + operand: &str, + colon_variant: bool, + is_set: bool, + ) -> String { + let needs_per_element = Self::is_elementwise_op(operator); if needs_per_element && let Some(elems) = self.resolve_param_expansion_elements(name) { let mut result = String::new(); for elem in &elems { @@ -1428,31 +1524,36 @@ impl Interpreter { let expanded_pat = self.expand_replace_pattern(pattern); self.replace_pattern(value, &expanded_pat, &expanded_rep, true) } - ParameterOp::UpperFirst => { - // ${var^} - uppercase first character - let mut chars = value.chars(); - match chars.next() { - Some(first) => first.to_uppercase().collect::() + chars.as_str(), - None => String::new(), - } - } - ParameterOp::UpperAll => { - // ${var^^} - uppercase all characters - value.to_uppercase() - } - ParameterOp::LowerFirst => { - // ${var,} - lowercase first character - let mut chars = value.chars(); - match chars.next() { - Some(first) => first.to_lowercase().collect::() + chars.as_str(), - None => String::new(), - } - } - ParameterOp::LowerAll => { - // ${var,,} - lowercase all characters - value.to_lowercase() + ParameterOp::UpperFirst => self.change_case(value, operand, true, false), + ParameterOp::UpperAll => self.change_case(value, operand, true, true), + ParameterOp::LowerFirst => self.change_case(value, operand, false, false), + ParameterOp::LowerAll => self.change_case(value, operand, false, true), + } + } + + /// `${v^pat}`, `${v^^pat}`, `${v,pat}`, `${v,,pat}`: change the case of + /// the first (or every) character that matches `pat` on its own. An empty + /// pattern matches any character, as in bash. + fn change_case(&mut self, value: &str, operand: &str, upper: bool, all: bool) -> String { + let pattern = if operand.is_empty() { + String::new() + } else { + self.expand_pattern_operand(operand) + }; + let mut out = String::with_capacity(value.len()); + let mut buf = [0u8; 4]; + for (i, ch) in value.chars().enumerate() { + let selected = (all || i == 0) + && (pattern.is_empty() || self.pattern_matches(ch.encode_utf8(&mut buf), &pattern)); + if !selected { + out.push(ch); + } else if upper { + out.extend(ch.to_uppercase()); + } else { + out.extend(ch.to_lowercase()); } } + out } /// Replace pattern in value diff --git a/crates/bashkit/src/interpreter/mod.rs b/crates/bashkit/src/interpreter/mod.rs index 279ee296a..4db2e026c 100644 --- a/crates/bashkit/src/interpreter/mod.rs +++ b/crates/bashkit/src/interpreter/mod.rs @@ -8445,10 +8445,10 @@ impl Interpreter { pattern, replacement, } => out.push_str(&format!("${{{}//{}/{}}}", name, pattern, replacement)), - ParameterOp::UpperFirst => out.push_str(&format!("${{{}^}}", name)), - ParameterOp::UpperAll => out.push_str(&format!("${{{}^^}}", name)), - ParameterOp::LowerFirst => out.push_str(&format!("${{{},}}", name)), - ParameterOp::LowerAll => out.push_str(&format!("${{{},,}}", name)), + ParameterOp::UpperFirst => out.push_str(&format!("${{{}^{}}}", name, operand)), + ParameterOp::UpperAll => out.push_str(&format!("${{{}^^{}}}", name, operand)), + ParameterOp::LowerFirst => out.push_str(&format!("${{{},{}}}", name, operand)), + ParameterOp::LowerAll => out.push_str(&format!("${{{},,{}}}", name, operand)), }, WordPart::Length(name) => out.push_str(&format!("${{#{}}}", name)), WordPart::ArrayAccess { name, index } => { diff --git a/crates/bashkit/src/parser/ast.rs b/crates/bashkit/src/parser/ast.rs index 6d2accb66..ff7189f99 100644 --- a/crates/bashkit/src/parser/ast.rs +++ b/crates/bashkit/src/parser/ast.rs @@ -349,10 +349,10 @@ impl fmt::Display for Word { pattern, replacement, } => write!(f, "${{{}///{}/{}}}", name, pattern, replacement)?, - ParameterOp::UpperFirst => write!(f, "${{{}^}}", name)?, - ParameterOp::UpperAll => write!(f, "${{{}^^}}", name)?, - ParameterOp::LowerFirst => write!(f, "{}{{,}}", name)?, - ParameterOp::LowerAll => write!(f, "${{{},,}}", name)?, + ParameterOp::UpperFirst => write!(f, "${{{}^{}}}", name, operand)?, + ParameterOp::UpperAll => write!(f, "${{{}^^{}}}", name, operand)?, + ParameterOp::LowerFirst => write!(f, "${{{},{}}}", name, operand)?, + ParameterOp::LowerAll => write!(f, "${{{},,{}}}", name, operand)?, }, WordPart::Length(name) => write!(f, "${{#{}}}", name)?, WordPart::ArrayAccess { name, index } => write!(f, "${{{}[{}]}}", name, index)?, @@ -982,6 +982,7 @@ mod tests { }; check(ParameterOp::UpperFirst, "${var^}"); check(ParameterOp::UpperAll, "${var^^}"); + check(ParameterOp::LowerFirst, "${var,}"); check(ParameterOp::LowerAll, "${var,,}"); } diff --git a/crates/bashkit/src/parser/mod.rs b/crates/bashkit/src/parser/mod.rs index c7c339b64..0b3022573 100644 --- a/crates/bashkit/src/parser/mod.rs +++ b/crates/bashkit/src/parser/mod.rs @@ -3478,6 +3478,10 @@ impl<'a> Parser<'a> { } // Check for array access ${arr[index]} or ${arr[@]:offset:length} + // `${arr[i]OP...}` with a pattern/case/transform OP reuses the + // scalar operator parsing below on the name `arr[i]`. + let mut subscript_op = false; + let had_subscript = chars.peek() == Some(&'['); if chars.peek() == Some(&'[') { chars.next(); // consume '[' let mut index = String::new(); @@ -3572,7 +3576,7 @@ impl<'a> Parser<'a> { chars.next(); } push_part!(WordPart::ArraySlice { - name: var_name, + name: std::mem::take(&mut var_name), offset, length, }); @@ -3595,23 +3599,29 @@ impl<'a> Parser<'a> { operand, colon_variant: false, }); + } else if matches!(next_c, '#' | '%' | '/' | '^' | ',' | '@') { + var_name = format!("{}[{}]", var_name, index); + subscript_op = true; } else { // Plain array access ${arr[index]} if chars.peek() == Some(&'}') { chars.next(); } push_part!(WordPart::ArrayAccess { - name: var_name, + name: std::mem::take(&mut var_name), index, }); } } else { push_part!(WordPart::ArrayAccess { - name: var_name, + name: std::mem::take(&mut var_name), index, }); } - } else if let Some(&c) = chars.peek() { + } + if (!had_subscript || subscript_op) + && let Some(&c) = chars.peek() + { // Check for operator match c { ':' => { @@ -3824,13 +3834,12 @@ impl<'a> Parser<'a> { } else { ParameterOp::UpperFirst }; - if chars.peek() == Some(&'}') { - chars.next(); - } + // `${v^^pat}`: only characters matching `pat`. + let operand = self.read_brace_operand(&mut chars); push_part!(WordPart::ParameterExpansion { name: var_name, operator: op, - operand: String::new(), + operand, colon_variant: false, }); } @@ -3842,13 +3851,12 @@ impl<'a> Parser<'a> { } else { ParameterOp::LowerFirst }; - if chars.peek() == Some(&'}') { - chars.next(); - } + // `${v^^pat}`: only characters matching `pat`. + let operand = self.read_brace_operand(&mut chars); push_part!(WordPart::ParameterExpansion { name: var_name, operator: op, - operand: String::new(), + operand, colon_variant: false, }); } @@ -3889,7 +3897,7 @@ impl<'a> Parser<'a> { } } } - } else if !var_name.is_empty() { + } else if !had_subscript && !var_name.is_empty() { push_part!(WordPart::Variable(var_name)); } } diff --git a/crates/bashkit/tests/spec_cases/bash/param-array-ops.test.sh b/crates/bashkit/tests/spec_cases/bash/param-array-ops.test.sh new file mode 100644 index 000000000..14d42ec4e --- /dev/null +++ b/crates/bashkit/tests/spec_cases/bash/param-array-ops.test.sh @@ -0,0 +1,62 @@ +### array_element_replace_and_trim +a=(xa ya) +echo "${a[@]/a/b}" "${a[1]#y}" "${a[@]%a}" "${a[@]##?}" "${a[0]//a/-}" +### expect +xb yb a x y a a x- +### end + +### array_element_case +a=(xa ya); b=(AB CD) +echo "${a[0]^}" "${a[@]^^}" "${b[@],}" "${b[*],,}" +### expect +Xa XA YA aB cD ab cd +### end + +### array_quoted_at_keeps_fields +a=('a b' c) +printf '<%s>' "${a[@]/b/x}" "${a[@]^}"; echo +printf '<%s>' "${a[*]/b/x}"; echo +### expect + + +### end + +### assoc_element_ops +declare -A m=([k]=val) +echo "${m[k]^}" "${m[@]%l}" "${m[k]/a/A}" +### expect +Val va vAl +### end + +### array_transform_ops +a=(x 'y z') +echo ${a[@]@Q} +echo "${a[*]@Q}" "${a[0]@U}" "${a[1]@u}" +### expect +'x' 'y z' +'x' 'y z' X Y z +### end + +### positional_transform_per_element +set -- ab 'c d' +printf '<%s>' "${@@Q}"; echo +### expect +<'ab'><'c d'> +### end + +### case_mod_with_pattern +v="hello world" +echo "${v^^[lo]}" "${v^[hw]}" "${v^[a-g]}" "${v^^?}" +V=HELLO +echo "${V,,[L]}" "${V,[H]}" "${V,[X]}" "${V,,}" +### expect +heLLO wOrLd Hello world hello world HELLO WORLD +HEllO hELLO HELLO hello +### end + +### case_mod_pattern_on_array +a=(hello yellow) +echo "${a[@]^^l}" "${a[1]^y}" +### expect +heLLo yeLLow Yellow +### end diff --git a/knowledge/log.md b/knowledge/log.md index 196d2b0cc..7635180e2 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -16,6 +16,7 @@ * **Fix**: `tar -P`/`--absolute-names` keeps the leading `/` (no notice), `-T FILE`/`--files-from` reads member names (blank lines skipped, `-` is stdin), and `--` ends options. `ls` implements the quoting styles: `-b`/`--escape`, `-N`, `-Q`, `--quoting-style=literal|escape|c|shell|shell-always|shell-escape|shell-escape-always` (GNU rules, including `"it's"` and headers left unescaped under `-b`); `locale`/`clocale` still report not implemented. Found by gap telemetry. * **Fix**: Arithmetic assigns to array elements. `((a[i]++))`, `((c[$k]+=3))`, `((a[2]=5))`, `((--a[0]))`, `let a[1]++` and `$((c[x]--))` read and write the element (indexed subscripts are arithmetic and accept negative indexes; associative keys are literal, quotes dropped as in `((c["x y"]++))`). Before, the update was lost or written to a scalar. Writes go through the budgeted `${a[i]:=v}` writer, so array memory limits still apply. Found by gap telemetry. * **Fix**: Gaps from the replayed agent calls: `set -n` / `set -o noexec` now stops running the rest of the script, as in bash (it was accepted and ignored); `find -ls` prints GNU's `ls -dils` layout; `tar -c` stores absolute paths without the leading `/` (with GNU's `Removing leading` notice) and `tar -cv` lists members on stdout unless the archive goes to stdout. Specs: `set-noexec.test.sh`, `find.test.sh`, `tar.test.sh`. +* **Fix**: Pattern, case and transform operators work on array elements and whole arrays. `${a[@]/x/y}`, `${a[1]#p}`, `${a[@]%s}`, `${a[@]^^}`, `${m[k],}`, `${a[@]@Q}` and `${a[0]@U}` printed the array followed by the operator text (`xa ya/a/b}`); they now apply per element, and quoted `"${a[@]op}"`/`"${@op}"` keep one field per element (`"${@@Q}"` was one joined field). `${v^^pat}`, `${v^pat}`, `${v,,pat}` and `${v,pat}` change only characters matching `pat` (the pattern was printed as text). Found by the pseudo-linus Debian comparison. * **Fix**: Quotes and patterns inside `${...}` operands. Unquoted `${x:-'d'}` keeps no quotes and `'$y'` stays literal; `"${x:-'d'}"` keeps them, as in bash; a `}` or `/` inside quotes no longer ends the operand. `${x/pattern/rep}` now expands `$var`, quotes and `\c` in the pattern (it used them verbatim) and matches globs leftmost-longest with `?`, `[...]` and any number of `*`; extglob patterns follow bash's empty-match scan. Globs compile to a size-capped regex, extglob search is budgeted (TM-DOS-127). Known gap: a quoted default with spaces (`${x:-'a b'}`) is still word-split. Spec: `param-operand-quotes.test.sh`. * **Feature**: Gap telemetry (`bashkit-replay`, `just gaps`). Replays the 5,206 recorded eval tool calls (1,493 sessions) on the current bashkit and ranks missing commands, unknown options, unsupported features and parse errors; `/benches` shows the top 20 and the history. First report: calls hitting a gap fell from 319 recorded to 125, non-zero exits from 723 to 231. Top remaining: heredoc `expected 'done'` and `unterminated command substitution` parse errors, `tar --strip-components`, `grep -I`, `join -o`, `find -ls`, plus probes for host tools (`busybox`, `gawk`, `sudo`, `python3`). * **Fix**: `#` mid-word is literal (`echo a#b` printed `a`; it starts a comment only at word start), and the `[[ =~ ]]` operand is read from source as one regex word as bash does: `#`, `|` and spaces inside groups are pattern text (`^(#+)[[:space:]](.+)$` was a parse error), quoted text and `\c` match literally (`"a.b"` no longer matches `axb`), and a bare trailing `$` is an anchor. Found by gap telemetry: `expected 'done'` / `unexpected end of input in [[ ]]` were the top parse errors in replayed agent calls.