diff --git a/src/uucore/src/lib/features/quoting_style/escaped_char.rs b/src/uucore/src/lib/features/quoting_style/escaped_char.rs index 8cc8a9a6f77..8d22c589a36 100644 --- a/src/uucore/src/lib/features/quoting_style/escaped_char.rs +++ b/src/uucore/src/lib/features/quoting_style/escaped_char.rs @@ -11,6 +11,12 @@ use super::shell_quoter::ShellQuotes; // PR#6559 : Remove `]{}` from special shell chars. const SPECIAL_SHELL_CHARS: &str = "`$&*()|[;\\'\"<>?! "; +/// Control characters, plus U+2028 LINE SEPARATOR and U+2029 PARAGRAPH SEPARATOR, +/// which are not controls but break lines all the same (GNU does not print them as is). +fn is_unprintable(c: char) -> bool { + c.is_control() || matches!(c, '\u{2028}' | '\u{2029}') +} + // This implementation is heavily inspired by the std::char::EscapeDefault implementation // in the Rust standard library. This custom implementation is needed because the // characters \a, \b, \e, \f & \v are not recognized by Rust. @@ -27,19 +33,18 @@ pub enum EscapeState { } /// Bytes we need to present as escaped octal, in the form of `\nnn` per byte. -/// Only supports characters up to 2 bytes long in UTF-8. pub struct EscapeOctal { - c: [u8; 2], + c: [u8; 4], + len: u8, + byte: u8, state: EscapeOctalState, idx: u8, } enum EscapeOctalState { Done, - FirstBackslash, - FirstValue, - LastBackslash, - LastValue, + Backslash, + Value, } fn byte_to_octal_digit(byte: u8, idx: u8) -> u8 { @@ -52,28 +57,20 @@ impl Iterator for EscapeOctal { fn next(&mut self) -> Option { match self.state { EscapeOctalState::Done => None, - EscapeOctalState::FirstBackslash => { - self.state = EscapeOctalState::FirstValue; - Some('\\') - } - EscapeOctalState::LastBackslash => { - self.state = EscapeOctalState::LastValue; + EscapeOctalState::Backslash => { + self.state = EscapeOctalState::Value; Some('\\') } - EscapeOctalState::FirstValue => { - let octal_digit = byte_to_octal_digit(self.c[0], self.idx); + EscapeOctalState::Value => { + let octal_digit = byte_to_octal_digit(self.c[usize::from(self.byte)], self.idx); if self.idx == 0 { - self.state = EscapeOctalState::LastBackslash; + self.byte += 1; self.idx = 2; - } else { - self.idx -= 1; - } - Some(from_digit(octal_digit.into(), 8).unwrap()) - } - EscapeOctalState::LastValue => { - let octal_digit = byte_to_octal_digit(self.c[1], self.idx); - if self.idx == 0 { - self.state = EscapeOctalState::Done; + self.state = if self.byte == self.len { + EscapeOctalState::Done + } else { + EscapeOctalState::Backslash + }; } else { self.idx -= 1; } @@ -85,24 +82,24 @@ impl Iterator for EscapeOctal { impl EscapeOctal { fn from_char(c: char) -> Self { - if c.len_utf8() == 1 { - return Self::from_byte(c as u8); - } - - let mut buf = [0; 2]; - let _s = c.encode_utf8(&mut buf); + let mut buf = [0; 4]; + let len = c.encode_utf8(&mut buf).len() as u8; Self { c: buf, + len, + byte: 0, idx: 2, - state: EscapeOctalState::FirstBackslash, + state: EscapeOctalState::Backslash, } } fn from_byte(b: u8) -> Self { Self { - c: [0, b], + c: [b, 0, 0, 0], + len: 1, + byte: 0, idx: 2, - state: EscapeOctalState::LastBackslash, + state: EscapeOctalState::Backslash, } } } @@ -137,7 +134,7 @@ impl EscapedChar { '\\' => Backslash('\\'), ' ' if !dirname && quotes.is_none() => Backslash(' '), ':' if dirname => Backslash(':'), - _ if c.is_control() => Octal(EscapeOctal::from_char(c)), + _ if is_unprintable(c) => Octal(EscapeOctal::from_char(c)), _ => Char(c), }; Self { state: init_state } @@ -146,7 +143,7 @@ impl EscapedChar { pub(super) fn new_shell(c: char, escape: bool, quotes: ShellQuotes) -> Self { use EscapeState::{Backslash, Char, ForceQuote, Octal}; let init_state = match c { - _ if !escape && c.is_control() => Char(c), + _ if !escape && is_unprintable(c) => Char(c), '\x07' => Backslash('a'), '\x08' => Backslash('b'), '\t' => Backslash('t'), @@ -155,7 +152,7 @@ impl EscapedChar { '\x0C' => Backslash('f'), '\r' => Backslash('r'), '\'' if matches!(quotes, ShellQuotes::Single) => Backslash('\''), - _ if c.is_control() => Octal(EscapeOctal::from_char(c)), + _ if is_unprintable(c) => Octal(EscapeOctal::from_char(c)), _ if SPECIAL_SHELL_CHARS.contains(c) => ForceQuote(c), _ => Char(c), }; @@ -164,7 +161,7 @@ impl EscapedChar { pub fn hide_control(self) -> Self { match self.state { - EscapeState::Char(c) if c.is_control() => Self { + EscapeState::Char(c) if is_unprintable(c) => Self { state: EscapeState::Char('?'), }, _ => self, diff --git a/src/uucore/src/lib/features/quoting_style/mod.rs b/src/uucore/src/lib/features/quoting_style/mod.rs index 75551692b5b..57088666827 100644 --- a/src/uucore/src/lib/features/quoting_style/mod.rs +++ b/src/uucore/src/lib/features/quoting_style/mod.rs @@ -841,6 +841,31 @@ mod tests { ); } + #[test] + fn test_line_and_paragraph_separators() { + // U+2028 and U+2029 are not controls but split lines, so they are + // escaped or hidden like controls, as GNU does. + for (name, octal) in [ + ("x\u{2028}..", "\\342\\200\\250"), + ("x\u{2029}..", "\\342\\200\\251"), + ] { + let escape = format!("x{octal}.."); + let c = format!("\"x{octal}..\""); + let shell_escape = format!("'x'$'{octal}''..'"); + check_names_encoding( + UEncoding::Utf8, + name, + &[ + ("x?..", "literal"), + (name, "literal-show"), + (&escape, "escape"), + (&c, "c"), + (&shell_escape, "shell-escape"), + ], + ); + } + } + #[test] fn test_non_unicode_bytes() { let ascii = b'_'; diff --git a/tests/by-util/test_ls.rs b/tests/by-util/test_ls.rs index 91445abd734..82c896e4298 100644 --- a/tests/by-util/test_ls.rs +++ b/tests/by-util/test_ls.rs @@ -3026,6 +3026,34 @@ mod quoting { .stdout_contains("\"hello world\""); } + // U+2028 and U+2029 split lines like control characters do, so -b escapes + // them and -q hides them, as GNU does. + #[test] + fn test_ls_quoting_line_separators_utf8() { + if !is_locale_available("en_US.UTF-8") { + return; + } + + for (name, escaped) in [ + ("x\u{2028}y", "x\\342\\200\\250y"), + ("x\u{2029}y", "x\\342\\200\\251y"), + ] { + let (at, mut ucmd) = at_and_ucmd!(); + at.touch(name); + ucmd.env("LC_ALL", "en_US.UTF-8") + .args(&["-b", "-1"]) + .succeeds() + .stdout_is(format!("{escaped}\n")); + + let (at, mut ucmd) = at_and_ucmd!(); + at.touch(name); + ucmd.env("LC_ALL", "en_US.UTF-8") + .args(&["-q", "-1"]) + .succeeds() + .stdout_is("x?y\n"); + } + } + #[test] fn test_ls_quoting_space() { check_quoting_dirname(