From 2c3224f54eee0c6b83d93bf5dc7a5730be2ba51a Mon Sep 17 00:00:00 2001 From: Abdullah Date: Thu, 1 Oct 2026 09:46:56 +0600 Subject: [PATCH 1/3] fix(lexer): implement ANSI-C string quoting support ($'...') - Added state tracking to the Lexer struct. - Intercepted token boundaries in to catch the '$' prefix. - Updated to handle backslash escapes inside ANSI-C strings. - Fixes multi-line continuation prompt trap when typing single-quote escapes. --- src/grammar/lexer.rs | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/src/grammar/lexer.rs b/src/grammar/lexer.rs index f44c3918..395b249c 100644 --- a/src/grammar/lexer.rs +++ b/src/grammar/lexer.rs @@ -212,6 +212,7 @@ pub struct Lexer { quote_after_backtick: Option, param_expansion_depth: usize, after_dollar: bool, + is_ansi_c_quote: bool, pending_loop_headers: usize, active_loop_bodies: usize, last_significant_token: Option, @@ -252,6 +253,7 @@ impl Lexer { quote_after_backtick: None, param_expansion_depth: 0, after_dollar: false, + is_ansi_c_quote: false, pending_loop_headers: 0, active_loop_bodies: 0, last_significant_token: None, @@ -393,6 +395,13 @@ impl Lexer { if (self.ch == '"' || self.ch == '\'') && self.in_quotes.is_none() { // Starting a quoted section let quote_type = self.ch; + + // Detect ANSI-C quote here, + if quote_type == '\'' && self.after_dollar { + self.is_ansi_c_quote = true; + self.after_dollar = false; + } + let token = Token { kind: if quote_type == '"' { TokenKind::Quote @@ -420,6 +429,7 @@ impl Lexer { }; self.in_quotes = None; // Clear the in_quotes state + self.is_ansi_c_quote = false; // Reset when quote closes self.read_char(); return token; } else if self.in_quotes.is_some() { @@ -1697,12 +1707,26 @@ impl Lexer { // Newlines inside a quoted string are lexed as separate `Newline` tokens // so that no `Word` token ever contains a newline character. while self.ch != quote_char && self.ch != '\0' && self.ch != '\n' { - // In double quotes, backslash escapes $, `, \, and " (preserving both chars) - if is_double_quote && self.ch == '\\' { + // ✓ Handle backslash in DOUBLE QUOTES or ANSI-C QUOTES + if (is_double_quote || self.is_ansi_c_quote) && self.ch == '\\' { let next = self.peek_char(); - if next == '$' || next == '`' || next == '\\' || next == '"' { + + if is_double_quote && (next == '$' || next == '`' || next == '\\' || next == '"') { + // Double quote escapes: $, `, \, " + content.push('\\'); + self.read_char(); + if self.ch == '\n' { + self.line += 1; + self.column = 0; + } + content.push(self.ch); + self.read_char(); + continue; + } else if self.is_ansi_c_quote && (next == '\'' || next == '\\') { + // ANSI-C escapes: \', \\ (minimal set) + // For full ANSI-C support, add: \n, \t, \xHH, etc. content.push('\\'); - self.read_char(); // Move to the escaped char + self.read_char(); if self.ch == '\n' { self.line += 1; self.column = 0; @@ -1712,7 +1736,7 @@ impl Lexer { continue; } } - + // For double-quoted strings, unescaped $ and ` trigger expansions if is_double_quote && (self.ch == '$' || self.ch == '`') { break; From 8a0487e7a1846853c48c9ea353ef2d1df082a24f Mon Sep 17 00:00:00 2001 From: Hal Frigaard <4559349+HalFrgrd@users.noreply.github.com> Date: Sun, 4 Oct 2026 11:58:12 +0100 Subject: [PATCH 2/3] Add tests and more fixes --- src/grammar/lexer.rs | 38 ++++++++++---- src/grammar/lexer_tests.rs | 102 +++++++++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+), 11 deletions(-) diff --git a/src/grammar/lexer.rs b/src/grammar/lexer.rs index 395b249c..0ce60e19 100644 --- a/src/grammar/lexer.rs +++ b/src/grammar/lexer.rs @@ -212,6 +212,7 @@ pub struct Lexer { quote_after_backtick: Option, param_expansion_depth: usize, after_dollar: bool, + next_is_ansi_c_quote: bool, is_ansi_c_quote: bool, pending_loop_headers: usize, active_loop_bodies: usize, @@ -253,6 +254,7 @@ impl Lexer { quote_after_backtick: None, param_expansion_depth: 0, after_dollar: false, + next_is_ansi_c_quote: false, is_ansi_c_quote: false, pending_loop_headers: 0, active_loop_bodies: 0, @@ -308,6 +310,8 @@ impl Lexer { let saved_column = self.column; let saved_param_expansion_depth = self.param_expansion_depth; let saved_after_dollar = self.after_dollar; + let saved_next_is_ansi_c_quote = self.next_is_ansi_c_quote; + let saved_is_ansi_c_quote = self.is_ansi_c_quote; let saved_pending_loop_headers = self.pending_loop_headers; let saved_active_loop_bodies = self.active_loop_bodies; let saved_last_significant_token = self.last_significant_token; @@ -328,6 +332,8 @@ impl Lexer { self.column = saved_column; self.param_expansion_depth = saved_param_expansion_depth; self.after_dollar = saved_after_dollar; + self.next_is_ansi_c_quote = saved_next_is_ansi_c_quote; + self.is_ansi_c_quote = saved_is_ansi_c_quote; self.pending_loop_headers = saved_pending_loop_headers; self.active_loop_bodies = saved_active_loop_bodies; self.last_significant_token = saved_last_significant_token; @@ -386,6 +392,8 @@ impl Lexer { } if self.in_quotes.is_none() && self.ch.is_whitespace() && self.ch != '\n' { + self.after_dollar = false; + self.next_is_ansi_c_quote = false; return self.read_whitespace(); } @@ -393,15 +401,16 @@ impl Lexer { // Check for quote start/end if (self.ch == '"' || self.ch == '\'') && self.in_quotes.is_none() { + self.after_dollar = false; // Starting a quoted section let quote_type = self.ch; - // Detect ANSI-C quote here, - if quote_type == '\'' && self.after_dollar { + if quote_type == '\'' && std::mem::take(&mut self.next_is_ansi_c_quote) { self.is_ansi_c_quote = true; - self.after_dollar = false; + } else { + self.next_is_ansi_c_quote = false; } - + let token = Token { kind: if quote_type == '"' { TokenKind::Quote @@ -513,7 +522,10 @@ impl Lexer { value: "`".to_string(), position: current_position, }; - } else if self.after_dollar && (self.ch.is_ascii_alphabetic() || self.ch == '_') { + } else if in_double_quotes + && self.after_dollar + && (self.ch.is_ascii_alphabetic() || self.ch == '_') + { // After a $ in double-quoted context, read only a valid variable name self.after_dollar = false; let token = self.read_var_name(); @@ -958,6 +970,13 @@ impl Lexer { value: "${".to_string(), position: current_position, } + } else if self.peek_char() == '\'' { + self.next_is_ansi_c_quote = true; + Token { + kind: TokenKind::Dollar, + value: "$".to_string(), + position: current_position, + } } else { self.after_dollar = true; Token { @@ -1710,7 +1729,7 @@ impl Lexer { // ✓ Handle backslash in DOUBLE QUOTES or ANSI-C QUOTES if (is_double_quote || self.is_ansi_c_quote) && self.ch == '\\' { let next = self.peek_char(); - + if is_double_quote && (next == '$' || next == '`' || next == '\\' || next == '"') { // Double quote escapes: $, `, \, " content.push('\\'); @@ -1727,16 +1746,12 @@ impl Lexer { // For full ANSI-C support, add: \n, \t, \xHH, etc. content.push('\\'); self.read_char(); - if self.ch == '\n' { - self.line += 1; - self.column = 0; - } content.push(self.ch); self.read_char(); continue; } } - + // For double-quoted strings, unescaped $ and ` trigger expansions if is_double_quote && (self.ch == '$' || self.ch == '`') { break; @@ -1753,6 +1768,7 @@ impl Lexer { if self.ch == '\0' { self.in_quotes = None; + self.is_ansi_c_quote = false; if content.is_empty() { return Token { kind: TokenKind::EOF, diff --git a/src/grammar/lexer_tests.rs b/src/grammar/lexer_tests.rs index 83d339fb..dde8c87d 100644 --- a/src/grammar/lexer_tests.rs +++ b/src/grammar/lexer_tests.rs @@ -2021,3 +2021,105 @@ fn test_heredoc_unmatched_trailing_quote_problem_statement_full_token_stream() { ], ); } + +#[test] +fn test_lexer_ansi_c_quote_escaped_single_quote() { + assert_lex!( + r"$'foo\'bar'", + [ + TokenKind::Dollar, + TokenKind::SingleQuote, + TokenKind::Word(r"foo\'bar".to_string()), + TokenKind::SingleQuote, + ] + ); +} + +#[test] +fn test_lexer_ansi_c_quote_escaped_backslash() { + assert_lex!( + r"$'foo\\bar'", + [ + TokenKind::Dollar, + TokenKind::SingleQuote, + TokenKind::Word(r"foo\\bar".to_string()), + TokenKind::SingleQuote, + ] + ); +} + +#[test] +fn test_lexer_ansi_c_quote_escaped_backslash_before_closing_quote() { + assert_lex!( + r"$'foo\\'", + [ + TokenKind::Dollar, + TokenKind::SingleQuote, + TokenKind::Word(r"foo\\".to_string()), + TokenKind::SingleQuote, + ] + ); +} + +#[test] +fn test_lexer_ansi_c_quote_multiple_escapes() { + assert_lex!( + r"$'a\'b\\c\'d'", + [ + TokenKind::Dollar, + TokenKind::SingleQuote, + TokenKind::Word(r"a\'b\\c\'d".to_string()), + TokenKind::SingleQuote, + ] + ); +} + +#[test] +fn test_lexer_regular_single_quote_does_not_escape_quote() { + assert_lex!( + r"'foo\'bar'", + [ + TokenKind::SingleQuote, + TokenKind::Word(r"foo\".to_string()), + TokenKind::SingleQuote, + TokenKind::Word("bar".to_string()), + TokenKind::SingleQuote, + ] + ); +} + +#[test] +fn test_dparser_needs_more_input_ansi_c_quote() { + use crate::grammar::dparser::DParser; + // With escaped single quote, the ANSI-C quote is properly closed: needs_more_input must be false + let mut parser = DParser::from(r"echo $'foo\'bar'"); + parser.walk_to_end(); + assert!(!parser.needs_more_input()); + + // Unterminated ANSI-C quote: needs_more_input must be true + let mut parser = DParser::from(r"echo $'foo\'"); + parser.walk_to_end(); + assert!(parser.needs_more_input()); + + // Regular single quote with \': quote closes at second ', leaving third ' unmatched -> needs_more_input is true + let mut parser = DParser::from(r"echo 'foo\'bar'"); + parser.walk_to_end(); + assert!(parser.needs_more_input()); +} + +#[test] +fn test_lexer_dollar_space_single_quote_is_not_ansi_c() { + // $ followed by space and then single quote is NOT ANSI-C quoting + assert_lex!( + r"$ 'foo\'bar'", + [ + TokenKind::Dollar, + TokenKind::Whitespace(" ".to_string()), + TokenKind::SingleQuote, + TokenKind::Word(r"foo\".to_string()), + TokenKind::SingleQuote, + TokenKind::Word("bar".to_string()), + TokenKind::SingleQuote, + ] + ); +} From 7772bb35178d8ed83a11db19fe62ebfc3c9715d6 Mon Sep 17 00:00:00 2001 From: Abdullah Date: Mon, 5 Oct 2026 07:15:42 +0600 Subject: [PATCH 3/3] chore: run rustfmt and apply code quality lint adjustments --- src/grammar/lexer.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/grammar/lexer.rs b/src/grammar/lexer.rs index 395b249c..d7fc8818 100644 --- a/src/grammar/lexer.rs +++ b/src/grammar/lexer.rs @@ -401,7 +401,7 @@ impl Lexer { self.is_ansi_c_quote = true; self.after_dollar = false; } - + let token = Token { kind: if quote_type == '"' { TokenKind::Quote @@ -1710,7 +1710,7 @@ impl Lexer { // ✓ Handle backslash in DOUBLE QUOTES or ANSI-C QUOTES if (is_double_quote || self.is_ansi_c_quote) && self.ch == '\\' { let next = self.peek_char(); - + if is_double_quote && (next == '$' || next == '`' || next == '\\' || next == '"') { // Double quote escapes: $, `, \, " content.push('\\'); @@ -1736,7 +1736,7 @@ impl Lexer { continue; } } - + // For double-quoted strings, unescaped $ and ` trigger expansions if is_double_quote && (self.ch == '$' || self.ch == '`') { break;