From 3ff18b05f6d074c1c564b85006c4f2ee4750fff9 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Wed, 7 Oct 2026 17:22:29 +0000 Subject: [PATCH] fix(interpreter): brace expansion before other expansions Brace expansion now runs on the parsed word, before parameter and command expansion, treating everything but unquoted literal text as opaque. Expansion results no longer brace-expand, quoted and escaped braces and commas stay literal, invalid groups keep later groups expanding, empty items are dropped and ranges zero-pad. The lexer treats braces as ordinary word characters. Claude-Session: https://claude.ai/code/session_019aFikmptPc91Fj4N2iDXQA --- .../src/interpreter/brace_expansion.rs | 166 ++++++++++++++- crates/bashkit/src/interpreter/expansion.rs | 2 +- crates/bashkit/src/interpreter/glob.rs | 14 +- crates/bashkit/src/interpreter/mod.rs | 147 ++++++------- crates/bashkit/src/parser/lexer.rs | 193 ++---------------- crates/bashkit/src/parser/mod.rs | 13 +- .../bash/brace-expansion-order.test.sh | 68 ++++++ .../spec_cases/bash/brace-expansion.test.sh | 5 +- knowledge/foundations/parser.md | 11 + knowledge/log.md | 1 + knowledge/operations/limitations.md | 1 + 11 files changed, 358 insertions(+), 263 deletions(-) create mode 100644 crates/bashkit/tests/spec_cases/bash/brace-expansion-order.test.sh diff --git a/crates/bashkit/src/interpreter/brace_expansion.rs b/crates/bashkit/src/interpreter/brace_expansion.rs index e3b51c199..70a0e75dc 100644 --- a/crates/bashkit/src/interpreter/brace_expansion.rs +++ b/crates/bashkit/src/interpreter/brace_expansion.rs @@ -90,7 +90,14 @@ impl Interpreter { // Brace content with leading/trailing space is not expanded if brace_content.starts_with(' ') || brace_content.ends_with(' ') { - return vec![s.to_string()]; + return self.expand_after_literal_brace( + &chars, + start, + count, + bytes, + max, + recursion_depth, + ); } // Check for range expansion like {1..5} or {a..z} @@ -113,9 +120,17 @@ impl Interpreter { // List expansion like {a,b,c} // Need to split by comma, but respect nested braces let items = self.split_brace_items(&brace_content); - if items.len() <= 1 && !brace_content.contains(',') { - // Not a valid brace expansion (e.g., just {foo}) - return vec![s.to_string()]; + if items.len() <= 1 { + // Not a valid brace expansion (e.g., just {foo}): the `{` stays + // literal and later groups still expand (`{x}{a,b}` -> `{x}a {x}b`). + return self.expand_after_literal_brace( + &chars, + start, + count, + bytes, + max, + recursion_depth, + ); } let mut results = Vec::new(); @@ -133,6 +148,104 @@ impl Interpreter { results } + /// The group opening at `start` is not a brace expansion: keep text up to + /// and including that `{` and expand whatever follows it. + fn expand_after_literal_brace( + &self, + chars: &[char], + start: usize, + count: &mut usize, + bytes: &mut usize, + max: usize, + recursion_depth: usize, + ) -> Vec { + let head: String = chars[..=start].iter().collect(); + let rest: String = chars[start + 1..].iter().collect(); + self.expand_braces_capped(&rest, count, bytes, max, recursion_depth + 1) + .into_iter() + .map(|tail| format!("{head}{tail}")) + .collect() + } + + /// Brace-expand a word before any other expansion, as bash does. + /// + /// Only unquoted literal text takes part: every other part (variables, + /// command substitutions, quoted segments) is an opaque atom, so text an + /// expansion produces is never brace-expanded (`y='{a,b}'; echo $y` + /// prints `{a,b}`) and `{1..$n}` stays literal. Atoms are carried through + /// the string-based expander as private-use placeholder characters and + /// mapped back to the original parts afterwards. + /// + /// Backslashes stay escapes, as in the lexer's quoted-glob convention + /// (`a"{"b,c}{1,2}` arrives as `a\{b,c}{1,2}`). Fully quoted words never + /// brace-expand. + pub(super) fn brace_expand_word(&self, word: &Word) -> Option> { + if word.quoted && !word.has_unquoted_glob { + return None; + } + let quoted_at = |i: usize| word.part_quoted.get(i).copied().unwrap_or(word.quoted); + let has_brace = + word.parts.iter().enumerate().any( + |(i, p)| matches!(p, WordPart::Literal(t) if !quoted_at(i) && t.contains('{')), + ); + if !has_brace { + return None; + } + let mut text = String::new(); + for (i, part) in word.parts.iter().enumerate() { + match part { + WordPart::Literal(t) if !quoted_at(i) => { + for ch in t.chars() { + if is_brace_placeholder(ch) { + // Cannot encode this word; leave it unexpanded. + return None; + } + text.push(ch); + } + } + _ => text.push(placeholder_for(i)?), + } + } + let expanded = self.expand_braces(&text); + if expanded.len() == 1 && expanded[0] == text { + return None; + } + let mut words = Vec::with_capacity(expanded.len()); + for item in expanded { + // An unquoted brace item that expands to nothing is dropped + // (`echo {,a}` prints `a`). + if item.is_empty() { + continue; + } + let mut parts = Vec::new(); + let mut part_quoted = Vec::new(); + let mut lit = String::new(); + for ch in item.chars() { + if let Some(i) = placeholder_index(ch) { + if !lit.is_empty() { + parts.push(WordPart::Literal(std::mem::take(&mut lit))); + part_quoted.push(false); + } + parts.push(word.parts[i].clone()); + part_quoted.push(quoted_at(i)); + } else { + lit.push(ch); + } + } + if !lit.is_empty() { + parts.push(WordPart::Literal(lit)); + part_quoted.push(false); + } + words.push(Word { + parts, + quoted: word.quoted, + has_unquoted_glob: word.has_unquoted_glob, + part_quoted, + }); + } + Some(words) + } + /// Try to expand a range like 1..5, a..z, or 1..10..2 /// THREAT[TM-DOS-041]: Cap range size to prevent OOM from {1..999999999} pub(super) fn try_expand_range(&self, content: &str) -> Option> { @@ -180,16 +293,27 @@ impl Interpreter { -step_magnitude }; + // `{01..10}`: a leading zero on either end pads every item to the + // wider end's length (the `-` counts toward the width). + let zero_padded = |n: &str| { + let digits = n.strip_prefix('-').unwrap_or(n); + digits.len() > 1 && digits.starts_with('0') + }; + let width = if zero_padded(start) || zero_padded(end) { + start.len().max(end.len()) + } else { + 0 + }; let mut i = start_num as i128; let end = end_num as i128; if effective_step > 0 { while i <= end { - results.push(i.to_string()); + results.push(format!("{i:0width$}")); i += effective_step; } } else { while i >= end { - results.push(i.to_string()); + results.push(format!("{i:0width$}")); i += effective_step; } } @@ -248,9 +372,19 @@ impl Interpreter { let mut items = Vec::new(); let mut current = String::new(); let mut depth = 0; + let mut escaped = false; for ch in content.chars() { + if escaped { + escaped = false; + current.push(ch); + continue; + } match ch { + '\\' => { + escaped = true; + current.push(ch); + } '{' => { depth += 1; current.push(ch); @@ -273,3 +407,23 @@ impl Interpreter { items } } + +/// Opaque word part `i` is carried as `BASE + 1 + i`. +const BRACE_PLACEHOLDER_BASE: u32 = 0xF0000; +/// Words with more parts than this are not brace-expanded. +const BRACE_MAX_PARTS: u32 = 0xFFF0; + +fn is_brace_placeholder(ch: char) -> bool { + (BRACE_PLACEHOLDER_BASE..=BRACE_PLACEHOLDER_BASE + BRACE_MAX_PARTS).contains(&(ch as u32)) +} + +fn placeholder_for(i: usize) -> Option { + let i = u32::try_from(i).ok().filter(|i| *i < BRACE_MAX_PARTS)?; + char::from_u32(BRACE_PLACEHOLDER_BASE + 1 + i) +} + +fn placeholder_index(ch: char) -> Option { + let c = ch as u32; + (c > BRACE_PLACEHOLDER_BASE && c <= BRACE_PLACEHOLDER_BASE + BRACE_MAX_PARTS) + .then(|| (c - BRACE_PLACEHOLDER_BASE - 1) as usize) +} diff --git a/crates/bashkit/src/interpreter/expansion.rs b/crates/bashkit/src/interpreter/expansion.rs index 6eb8dc095..d9a629232 100644 --- a/crates/bashkit/src/interpreter/expansion.rs +++ b/crates/bashkit/src/interpreter/expansion.rs @@ -136,7 +136,7 @@ impl Interpreter { fn needs_glob_escape(ch: char) -> bool { matches!( ch, - '\\' | '*' | '?' | '[' | ']' | '{' | '}' | '@' | '!' | '+' | '(' | ')' | '|' + '\\' | '*' | '?' | '[' | ']' | '{' | '}' | ',' | '@' | '!' | '+' | '(' | ')' | '|' ) } diff --git a/crates/bashkit/src/interpreter/glob.rs b/crates/bashkit/src/interpreter/glob.rs index 5f42f5fb3..8254a9f4d 100644 --- a/crates/bashkit/src/interpreter/glob.rs +++ b/crates/bashkit/src/interpreter/glob.rs @@ -777,7 +777,19 @@ impl Interpreter { && let Some(&next) = chars.peek() && matches!( next, - '\\' | '*' | '?' | '[' | ']' | '{' | '}' | '@' | '!' | '+' | '(' | ')' | '|' + '\\' | '*' + | '?' + | '[' + | ']' + | '{' + | '}' + | ',' + | '@' + | '!' + | '+' + | '(' + | ')' + | '|' ) { result.push(next); diff --git a/crates/bashkit/src/interpreter/mod.rs b/crates/bashkit/src/interpreter/mod.rs index 279ee296a..e34920de7 100644 --- a/crates/bashkit/src/interpreter/mod.rs +++ b/crates/bashkit/src/interpreter/mod.rs @@ -3642,30 +3642,33 @@ impl Interpreter { // Get iteration values: expand fields, then apply brace/glob expansion let values: Vec = if let Some(words) = &for_cmd.words { let mut vals = Vec::new(); - for w in words { - let fields = self.expand_word_to_fields(w).await?; - - // Quoted words skip brace/glob expansion — unless the - // word has unquoted glob chars (e.g. `"$var"*.ext`) - if w.quoted && !w.has_unquoted_glob { - vals.extend(fields); - continue; - } + for w0 in words { + // Brace expansion runs first, on the unexpanded word. + let braced = self.brace_expand_word(w0); + for w in braced.as_deref().unwrap_or(std::slice::from_ref(w0)) { + let fields = self.expand_word_to_fields(w).await?; + + // Quoted words skip brace/glob expansion — unless the + // word has unquoted glob chars (e.g. `"$var"*.ext`) + if w.quoted && !w.has_unquoted_glob { + vals.extend(fields); + continue; + } - for expanded in fields { - let brace_expanded = self.expand_braces(&expanded); - for item in brace_expanded { - match self - .expand_glob_item(&item, w.quoted && w.has_unquoted_glob) - .await - { - Ok(items) => vals.extend(items), - Err(pat) => { - self.last_exit_code = 1; - return Ok(ExecResult::err( - format!("-bash: no match: {}\n", pat), - 1, - )); + for expanded in fields { + for item in [expanded] { + match self + .expand_glob_item(&item, w.quoted && w.has_unquoted_glob) + .await + { + Ok(items) => vals.extend(items), + Err(pat) => { + self.last_exit_code = 1; + return Ok(ExecResult::err( + format!("-bash: no match: {}\n", pat), + 1, + )); + } } } } @@ -3776,25 +3779,28 @@ impl Interpreter { // Expand word list let mut values = Vec::new(); - for w in &select_cmd.words { - let fields = self.expand_word_to_fields(w).await?; - if w.quoted && !w.has_unquoted_glob { - values.extend(fields); - } else { - for expanded in fields { - let brace_expanded = self.expand_braces(&expanded); - for item in brace_expanded { - match self - .expand_glob_item(&item, w.quoted && w.has_unquoted_glob) - .await - { - Ok(items) => values.extend(items), - Err(pat) => { - self.last_exit_code = 1; - return Ok(ExecResult::err( - format!("-bash: no match: {}\n", pat), - 1, - )); + for w0 in &select_cmd.words { + // Brace expansion runs first, on the unexpanded word. + let braced = self.brace_expand_word(w0); + for w in braced.as_deref().unwrap_or(std::slice::from_ref(w0)) { + let fields = self.expand_word_to_fields(w).await?; + if w.quoted && !w.has_unquoted_glob { + values.extend(fields); + } else { + for expanded in fields { + for item in [expanded] { + match self + .expand_glob_item(&item, w.quoted && w.has_unquoted_glob) + .await + { + Ok(items) => values.extend(items), + Err(pat) => { + self.last_exit_code = 1; + return Ok(ExecResult::err( + format!("-bash: no match: {}\n", pat), + 1, + )); + } } } } @@ -6899,34 +6905,37 @@ impl Interpreter { ) -> std::pin::Pin>> + Send + 'a>> { Box::pin(async move { let mut args: Vec = Vec::new(); - for word in &command.args { - // Use field expansion so "${arr[@]}" produces multiple args - let fields = self.expand_word_to_fields(word).await?; - - // Skip brace and glob expansion for quoted words — unless the - // word has unquoted glob chars (e.g. `"$var"*.ext`) in which case - // the quoted expansion suppresses IFS splitting but the unquoted - // portion must still undergo glob expansion. - if word.quoted && !word.has_unquoted_glob { - args.extend(fields); - continue; - } - - // For each field, apply brace and glob expansion - for expanded in fields { - // Step 1: Brace expansion (produces multiple strings) - let brace_expanded = self.expand_braces(&expanded); + for word0 in &command.args { + // Brace expansion runs first, on the unexpanded word. + let braced = self.brace_expand_word(word0); + for word in braced.as_deref().unwrap_or(std::slice::from_ref(word0)) { + // Use field expansion so "${arr[@]}" produces multiple args + let fields = self.expand_word_to_fields(word).await?; + + // Skip brace and glob expansion for quoted words — unless the + // word has unquoted glob chars (e.g. `"$var"*.ext`) in which case + // the quoted expansion suppresses IFS splitting but the unquoted + // portion must still undergo glob expansion. + if word.quoted && !word.has_unquoted_glob { + args.extend(fields); + continue; + } - // Step 2: For each brace-expanded item, do glob expansion - for item in brace_expanded { - match self - .expand_glob_item(&item, word.quoted && word.has_unquoted_glob) - .await - { - Ok(items) => args.extend(items), - Err(pat) => { - self.last_exit_code = 1; - return Ok(vec![format!("\x00ERR\x00-bash: no match: {}\n", pat)]); + // For each field, apply glob expansion + for expanded in fields { + for item in [expanded] { + match self + .expand_glob_item(&item, word.quoted && word.has_unquoted_glob) + .await + { + Ok(items) => args.extend(items), + Err(pat) => { + self.last_exit_code = 1; + return Ok(vec![format!( + "\x00ERR\x00-bash: no match: {}\n", + pat + )]); + } } } } diff --git a/crates/bashkit/src/parser/lexer.rs b/crates/bashkit/src/parser/lexer.rs index 696a19055..b225e9129 100644 --- a/crates/bashkit/src/parser/lexer.rs +++ b/crates/bashkit/src/parser/lexer.rs @@ -217,14 +217,14 @@ impl<'a> Lexer<'a> { // Look ahead to see if this is a brace expansion like {a,b,c} or {1..5} // vs a brace group like { cmd; } // Note: { must be followed by space/newline to be a brace group - if self.looks_like_brace_expansion() { - self.read_brace_expansion_word() - } else if self.is_brace_group_start() { + if self.is_brace_group_start() { self.advance(); Some(Token::LeftBrace) } else { - // {single} without comma/dot-dot is kept as literal word - self.read_brace_literal_word() + // Any other `{` starts an ordinary word: braces are plain + // word characters to the lexer, and brace expansion runs + // on the parsed word (quotes and `$` handled as usual). + self.read_word() } } '}' => { @@ -884,19 +884,6 @@ impl<'a> Lexer<'a> { has_unquoted_glob = true; word.push(ch); self.advance(); - let mut depth = 1; - while let Some(c) = self.peek_char() { - word.push(c); - self.advance(); - if c == '{' { - depth += 1; - } else if c == '}' { - depth -= 1; - if depth == 0 { - break; - } - } - } } else if ch == '`' { // Backtick command substitution: convert `cmd` to $(cmd) has_unquoted_expansion = true; @@ -1036,8 +1023,7 @@ impl<'a> Lexer<'a> { // An unmatched `}` inside a word is a literal. `}` is a // reserved word, not a metacharacter, so it only terminates a // word when it stands alone — bash prints `a}b`, not `a } b`. - // A balanced `{...}` never reaches here: the `'{'` arm above - // consumes it, so any `}` at this point has no opener. + // Brace expansion happens later, on the parsed word. word.push(ch); self.advance(); } else if self.is_word_char(ch) { @@ -1060,7 +1046,7 @@ impl<'a> Lexer<'a> { // classification: quoting it would suppress field splitting. let quoted_glob = quoted_ranges .iter() - .any(|&(start, end)| word[start..end].contains(['*', '?', '[', '{'])); + .any(|&(start, end)| word[start..end].contains(['*', '?', '[', '{', '}', ','])); if quoted_glob && has_unquoted_glob { return Some(Token::QuotedGlobWord( Self::escape_glob_metas_in_quoted_ranges(&word, "ed_ranges), @@ -1253,8 +1239,9 @@ impl<'a> Lexer<'a> { None => content.push('\\'), } } - Some(ch) if self.is_word_char(ch) => { - if matches!(ch, '*' | '?' | '[') { + Some(ch) if self.is_word_char(ch) || matches!(ch, '{' | '}') => { + // `{` marks possible brace expansion, like a glob char. + if matches!(ch, '*' | '?' | '[' | '{') { flags.has_unquoted_glob = true; } content.push(ch); @@ -1573,7 +1560,7 @@ impl<'a> Lexer<'a> { fn is_glob_escape_char(ch: char) -> bool { matches!( ch, - '\\' | '*' | '?' | '[' | ']' | '{' | '}' | '@' | '!' | '+' | '(' | ')' | '|' + '\\' | '*' | '?' | '[' | ']' | '{' | '}' | ',' | '@' | '!' | '+' | '(' | ')' | '|' ) } @@ -1700,7 +1687,7 @@ impl<'a> Lexer<'a> { // suppresses IFS splitting (the double-quoted segment) while still // performing glob expansion on the unquoted portion. if let Some(ch) = self.peek_char() - && (self.is_word_char(ch) || ch == '\'' || ch == '"' || ch == '$') + && (self.is_word_char(ch) || matches!(ch, '\'' | '"' | '$' | '{' | '}')) { let quoted_prefix_len = content.len(); let flags = self.read_continuation_into(&mut content); @@ -1822,6 +1809,7 @@ impl<'a> Lexer<'a> { | ']' | '{' | '}' + | ',' | '@' | '!' | '+' @@ -1980,54 +1968,6 @@ impl<'a> Lexer<'a> { Ok(()) } - /// Check if the content starting with { looks like a brace expansion - /// Brace expansion: {a,b,c} or {1..5} (contains , or ..) - /// Brace group: { cmd; } (contains spaces, semicolons, newlines) - /// THREAT[TM-DOS]: Caps lookahead to prevent O(n^2) scanning when input - /// contains many unmatched `{` characters (issue #997). - fn looks_like_brace_expansion(&self) -> bool { - const MAX_LOOKAHEAD: usize = 10_000; - - // Clone the iterator to peek ahead without consuming - let mut chars = self.chars.clone(); - - // Skip the opening { - if chars.next() != Some('{') { - return false; - } - - let mut depth = 1; - let mut has_comma = false; - let mut has_dot_dot = false; - let mut prev_char = None; - let mut scanned = 0usize; - - for ch in chars { - scanned += 1; - if scanned > MAX_LOOKAHEAD { - return false; - } - match ch { - '{' => depth += 1, - '}' => { - depth -= 1; - if depth == 0 { - // Found matching }, check if we have brace expansion markers - return has_comma || has_dot_dot; - } - } - ',' if depth == 1 => has_comma = true, - '.' if prev_char == Some('.') && depth == 1 => has_dot_dot = true, - // Brace groups have whitespace/newlines/semicolons at depth 1 - ' ' | '\t' | '\n' | ';' if depth == 1 => return false, - _ => {} - } - prev_char = Some(ch); - } - - false - } - /// Check if { is followed by whitespace (brace group start) fn is_brace_group_start(&self) -> bool { let mut chars = self.chars.clone(); @@ -2060,111 +2000,6 @@ impl<'a> Lexer<'a> { } } - /// Read a {literal} pattern without comma/dot-dot as a word - fn read_brace_literal_word(&mut self) -> Option { - let mut word = String::new(); - - // Read the opening { - if let Some('{') = self.peek_char() { - word.push('{'); - self.advance(); - } else { - return None; - } - - // Read until matching } - let mut depth = 1; - while let Some(ch) = self.peek_char() { - word.push(ch); - self.advance(); - match ch { - '{' => depth += 1, - '}' => { - depth -= 1; - if depth == 0 { - break; - } - } - _ => {} - } - } - - // Continue reading any suffix - while let Some(ch) = self.peek_char() { - if self.is_word_char(ch) { - word.push(ch); - self.advance(); - } else { - break; - } - } - - Some(Token::Word(word)) - } - - /// Read a brace expansion pattern as a word - fn read_brace_expansion_word(&mut self) -> Option { - let mut word = String::new(); - - // Read the opening { - if let Some('{') = self.peek_char() { - word.push('{'); - self.advance(); - } else { - return None; - } - - // Read until matching } - let mut depth = 1; - while let Some(ch) = self.peek_char() { - word.push(ch); - self.advance(); - match ch { - '{' => depth += 1, - '}' => { - depth -= 1; - if depth == 0 { - break; - } - } - _ => {} - } - } - - // Continue reading any suffix after the brace pattern - while let Some(ch) = self.peek_char() { - if self.is_word_char(ch) || ch == '{' { - if ch == '{' { - // Another brace pattern - include it - word.push(ch); - self.advance(); - let mut inner_depth = 1; - while let Some(c) = self.peek_char() { - word.push(c); - self.advance(); - match c { - '{' => inner_depth += 1, - '}' => { - inner_depth -= 1; - if inner_depth == 0 { - break; - } - } - _ => {} - } - } - } else { - word.push(ch); - self.advance(); - } - } else { - break; - } - } - - Some(Token::Word(word)) - } - /// Read a word starting with [ (glob bracket expression like [abc] or [a-z]) /// The opening [ has already been consumed fn read_bracket_word(&mut self) -> Option { @@ -2196,7 +2031,7 @@ impl<'a> Lexer<'a> { /// compound assignment like `([key]=val ...)`. Returns true when the /// first non-whitespace char after `(` is `[`. fn looks_like_assoc_assign(&self) -> bool { - // Cap the lookahead like looks_like_brace_expansion: an uncapped scan + // Cap the lookahead: an uncapped scan // over leading whitespace made `x=(` followed by megabytes of spaces // O(n) per call (TM-DOS-024). const MAX_LOOKAHEAD: usize = 10_000; diff --git a/crates/bashkit/src/parser/mod.rs b/crates/bashkit/src/parser/mod.rs index c7c339b64..f8c61a835 100644 --- a/crates/bashkit/src/parser/mod.rs +++ b/crates/bashkit/src/parser/mod.rs @@ -3188,10 +3188,15 @@ impl<'a> Parser<'a> { if let Some(literal_ch) = chars.next() { current.push(literal_ch); } - } else if ch == '\u{1e}' { - in_quoted_segment = true; - } else if ch == '\u{1f}' { - in_quoted_segment = false; + } else if ch == '\u{1e}' || ch == '\u{1f}' { + // A quote boundary ends the literal run, so `part_quoted` + // records which literal text was quoted (brace expansion + // only sees unquoted literals). + let quoted = ch == '\u{1e}'; + if quoted != in_quoted_segment && !current.is_empty() { + push_part!(WordPart::Literal(std::mem::take(&mut current))); + } + in_quoted_segment = quoted; } else if ch == '$' { // Flush current literal if !current.is_empty() { diff --git a/crates/bashkit/tests/spec_cases/bash/brace-expansion-order.test.sh b/crates/bashkit/tests/spec_cases/bash/brace-expansion-order.test.sh new file mode 100644 index 000000000..1b3f22ef3 --- /dev/null +++ b/crates/bashkit/tests/spec_cases/bash/brace-expansion-order.test.sh @@ -0,0 +1,68 @@ +### brace_not_applied_to_expansion_results +y='{a,b}' +echo $y +for i in $y; do echo "[$i]"; done +z='{1..3}'; echo $z x$z +### expect +{a,b} +[{a,b}] +{1..3} x{1..3} +### end + +### brace_range_with_variable_stays_literal +n=3 +echo {1..$n} +### expect +{1..3} +### end + +### brace_zero_padded_range +echo {01..10..3} +echo {08..11} +echo {-01..1} +echo {1..03} +### expect +01 04 07 10 +08 09 10 11 +-01 000 001 +01 02 03 +### end + +### brace_quoted_parts +echo "{a,b}"x{1,2} +echo '{a,b}'{c,d} +echo a"{"b,c} +echo {a,"b c"} +echo x{a,b}'y' +### expect +{a,b}x1 {a,b}x2 +{a,b}c {a,b}d +a{b,c} +a b c +xay xby +### end + +### brace_escaped_comma_and_brace +echo {a,\,} +echo a{b\,c,d} +echo \{a,b\} +### expect +a , +ab,c ad +{a,b} +### end + +### brace_invalid_group_then_valid +echo {x}{a,b} +echo {a{b,c}} +### expect +{x}a {x}b +{ab} {ac} +### end + +### brace_with_variable_item +v=V +echo {a,${v}}x pre{1,2}"$v" +### expect +ax Vx pre1V pre2V +### end diff --git a/crates/bashkit/tests/spec_cases/bash/brace-expansion.test.sh b/crates/bashkit/tests/spec_cases/bash/brace-expansion.test.sh index ae75cc64e..891a7ab9d 100644 --- a/crates/bashkit/tests/spec_cases/bash/brace-expansion.test.sh +++ b/crates/bashkit/tests/spec_cases/bash/brace-expansion.test.sh @@ -55,11 +55,10 @@ a1 a2 b1 b2 ### end ### brace_empty_item -### bash_diff: Bashkit preserves leading space from empty brace item, bash strips it -# Brace with empty item +# Brace with empty item: the empty word is dropped echo {,a,b} ### expect - a b +a b ### end ### brace_no_expand_single diff --git a/knowledge/foundations/parser.md b/knowledge/foundations/parser.md index 8c9927df2..9e30756a5 100644 --- a/knowledge/foundations/parser.md +++ b/knowledge/foundations/parser.md @@ -72,6 +72,17 @@ the closing side; both word readers (`read_word`, `read_word_starting_with`) keep a `}` that has no opener inside the word. `for`/`select` `in` lists apply the same distinction to `do`/`done`/`in` — see `for_in_reserved_word_tests`. +**Brace expansion.** Braces are ordinary word characters to the lexer (a +word may start with `{` unless it is the `{` reserved word). Expansion runs +first, on the parsed word (`brace_expand_word`): unquoted literal text is +expanded, every other part (variables, substitutions, quoted segments) is an +opaque atom carried through as a private-use placeholder. So text produced by +an expansion never brace-expands (`y='{a,b}'; echo $y` prints `{a,b}`) and +`{1..$n}` stays literal. Quoted or backslash-escaped `{`, `}` and `,` in a +`QuotedGlobWord` arrive backslash-escaped and stay literal. An invalid group +keeps its `{` and later groups still expand (`{x}{a,b}`). Gap: bash expands +raw text, so `$v{1,2}` reads `$v1`; we keep the `$v` part. + Treating `}` as a metacharacter is not just a cosmetic difference: it splits `v=a}b`, `for i in a}b`, and `case x}` at a point where the grammar expects a terminator, so they fail to parse rather than printing the wrong thing. diff --git a/knowledge/log.md b/knowledge/log.md index 196d2b0cc..8ba3e14a1 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -21,6 +21,7 @@ * **Fix**: `#` mid-word is literal (`echo a#b` printed `a`; it starts a comment only at word start), and the `[[ =~ ]]` operand is read from source as one regex word as bash does: `#`, `|` and spaces inside groups are pattern text (`^(#+)[[:space:]](.+)$` was a parse error), quoted text and `\c` match literally (`"a.b"` no longer matches `axb`), and a bare trailing `$` is an anchor. Found by gap telemetry: `expected 'done'` / `unexpected end of input in [[ ]]` were the top parse errors in replayed agent calls. * **Fix**: Option gaps from the replayed agent calls. `tar` takes GNU long options (`--strip-components`, `--exclude`, `--file`, `-C/--directory`, `--to-stdout`, ...), extracts named members, and rejects unknown long options with exit 2. `gzip`/`gunzip` take `-c`/`--stdout` and `-1..-9`, binary-safe through pipes. `grep -I`/`--binary-files=without-match` skip binary files, and `grep -r` names matches the way GNU does (`d/b`, `./b`, bare `b` with no operand, no prefix for a single file operand) instead of absolute VFS paths. `join` was rewritten to GNU semantics (`-o`/`auto`, `-e`, `-v`, `-a`, `-j`, `-i`, `--header`, cartesian products). `xargs` runs the command once on empty input, like GNU, and `-r`/`--no-run-if-empty` skips it. * **Feature**: `make` builtin, a GNU make 4.3 subset run as an execution-plan driver: assignments, conditionals, `define`, `include`, explicit/pattern/static/suffix rules, target-specific variables, special targets and the common functions incl. `$(shell)`/`$(wildcard)`. Recipes run as `sh -c` in the sandbox shell; output and make's messages stream in order (new `PlanStep::Capture`/`Emit`). 19 spec cases match real GNU make 4.3. Gaps L-MAKE-001..003 (no `$(eval)`/`$(file)`, no built-in or chained implicit rules or `vpath`, sequential `-j`, `SHELL` ignored, MAKELEVEL cap 4); caps TM-DOS-126. `make` left the "build tools are not available" hint list. +* **Fix**: Brace expansion runs before other expansions, as in bash. `y='{a,b}'; echo $y` prints `{a,b}` (it printed `a b`), `{1..$n}` stays literal, quoted and escaped braces/commas stay literal (`"{a,b}"x{1,2}`, `'{a,b}'{c,d}`, `{a,\,}`, `a{b\,c,d}`), `{a,"b c"}` keeps the quoted item as one word, `{x}{a,b}` and `{a{b,c}}` expand the valid group, `{,a}` drops the empty word, and `{01..10..3}` zero-pads. The lexer no longer has special readers for words starting with `{`. Found in the pseudo-linus oracle list (shell/brace-expansion, shell/expansion-order). * **Security**: A script that re-runs itself through `sh`/`bash` crashed the whole process with a stack overflow (TM-DOS-125); nothing bounded nested child shells. They now count against `max_function_depth` plus their own cap of 8, measured as what fits a 2 MiB debug-build stack (about 15 bare `sh` levels fit; a level costs several function calls). Found while bounding `make` recursion. * **Fix**: awk follows POSIX `opt_nls`: a newline after the `)` of an `if`/`for`/`while` header, after `else`/`do`, `&&`, `||` or `,` continues the statement (it became `;`, so `for (...)⏎ body` failed with `unexpected character: ;`, the most frequent awk error in gap telemetry, 22 replayed scripts). `do body; while (...)` accepts the `;`. A regex rule pattern can be an operand (`/=/ && !/^#/`, `/^$/ || /^;/`), and a bare `/re/` in an expression is `$0 ~ /re/` (it was the truthy pattern text, so `if (/^\[/)` matched every line). * **Feature**: Missing everyday commands: `arch`, `sum`, `shasum`, `egrep`, `fgrep`, `link`, `unlink`, `chgrp`, `nohup`, `nice`, `flock`, `getconf`, `tty`, `sync`, `hostid`, `groups`, `logname`, `users`, `who`, `uptime`, `free`. Output matches GNU coreutils/util-linux where the answer is not host identity (spec `system-commands.test.sh` checked against real bash); identity answers (`free`, `uptime`, `groups`, `hostid`) are virtual and agree with `nproc` and `/proc/meminfo`. `nohup`/`nice`/`flock` just run their command: there are no signals, priorities or competing lock holders inside one session. `/bin/bash` and `/bin/sh` now exist in the root filesystem, and `type`/`command -V` report interpreter-dispatched builtins (`builtin`, `command`, `let`, `typeset`, `exec`, `getopts`, `declare`) instead of "not found". diff --git a/knowledge/operations/limitations.md b/knowledge/operations/limitations.md index 33c0b5c89..2dee19bd1 100644 --- a/knowledge/operations/limitations.md +++ b/knowledge/operations/limitations.md @@ -108,6 +108,7 @@ Target: IEEE 1003.1-2024 Shell Command Language. | `time` | Reserved-word pipelines; `-p`, `--`, GNU `-f/-o/-a/-v`; elapsed time, exit status, and Bashkit counters | Host user/system CPU, RSS, and other process metrics are reported as `unavailable`, never fabricated | | `timeout` | Basic usage | `-k` kill timeout | | `bash`/`sh` | `-c`, `-n`, `-e`, `-x`, `-u`, `-f`, `-o option`, script files, stdin, `--version`, `--help` | Login shell | +| Brace expansion | Runs before other expansions on unquoted literal text; lists, nesting, ranges with step and zero padding, quoted/escaped braces and commas | Works on parsed words, not raw text: `$v{1,2}` expands `$v` then appends `1`/`2` (bash reads `$v1`, `$v2`); write `${v}` | ## Builtins