expand: keep the column across read buffers and files

src/uu/expand/src/expand.rs

@@ -371,6 +371,7 @@ fn expand_buf(
     tabstops: &[usize],
     options: &Options,
     col: &mut usize,
+    init: &mut bool,
 ) -> std::io::Result<()> {
     use self::CharType::{Backspace, Other, Tab};
 
@@ -382,14 +383,22 @@ fn expand_buf(
         .any(|&b| b == b'\t' || b == b'\x08' || (!options.utf8 && b == b'\r'));
     if !needs_processing {
         output.write_all(buf)?;
-        if let Some(n) = buf.iter().rposition(|&b| b == b'\n') {
-            *col = buf.len() - n - 1;
-        }
+        // The last line may continue in the next buffer or file, so keep
+        // track of its column and whether it is still in leading blanks.
+        let line = match buf.iter().rposition(|&b| b == b'\n') {
+            Some(n) => {
+                *col = 0;
+                *init = true;
+                &buf[n + 1..]
+            }
+            None => buf,
+        };
+        *init = *init && line.iter().all(|&b| b == b' ');
+        *col += line_width(line, options.utf8);
         return Ok(());
     }
 
     let mut byte = 0;
-    let mut init = true;
 
     while byte < buf.len() {
         let (ctype, cwidth, nbytes) = classify_char(buf, byte, options.utf8);
@@ -402,7 +411,7 @@ fn expand_buf(
                 *col += nts;
 
                 // now dump out either spaces if we're expanding, or a literal tab if we're not
-                if init || !options.iflag {
+                if *init || !options.iflag {
                     write_spaces(output, nts)?;
                 } else {
                     output.write_all(&buf[byte..byte + nbytes])?;
@@ -414,7 +423,7 @@ fn expand_buf(
                 // if we're writing anything other than a space, then we're
                 // done with the line's leading spaces
                 if buf[byte] != b' ' {
-                    init = false;
+                    *init = false;
                 }
 
                 output.write_all(&buf[byte..byte + nbytes])?;
@@ -425,12 +434,12 @@ fn expand_buf(
                 // if we're writing anything other than a space, then we're
                 // done with the line's leading spaces
                 if buf[byte] != b' ' {
-                    init = false;
+                    *init = false;
                 }
 
                 if buf[byte] == b'\n' {
                     *col = 0;
-                    init = true;
+                    *init = true;
                 }
 
                 output.write_all(&buf[byte..byte + nbytes])?;
@@ -443,34 +452,60 @@ fn expand_buf(
     Ok(())
 }
 
+/// The number of columns that `line`, which has no tabs or backspaces, takes up.
+fn line_width(line: &[u8], utf8: bool) -> usize {
+    let mut width = 0;
+    let mut byte = 0;
+    while byte < line.len() {
+        let (_, cwidth, nbytes) = classify_char(line, byte, utf8);
+        width += cwidth;
+        byte += nbytes;
+    }
+    width
+}
+
 fn expand_file(
     file: &OsString,
     output: &mut BufWriter<std::io::Stdout>,
     options: &Options,
+    line_col: &mut usize,
+    line_init: &mut bool,
 ) -> UResult<()> {
     let mut input = open(file)?;
     let ts = options.tabstops.as_ref();
-    let mut col = 0;
-    loop {
+    // Work on local copies of the line state, so they can stay in registers
+    // while expanding, and store them back when done with this file.
+    let mut col = *line_col;
+    let mut init = *line_init;
+    let result: UResult<()> = loop {
         match input.fill_buf() {
-            Ok([]) => break,
+            Ok([]) => break Ok(()),
             Ok(buf) => {
-                expand_buf(buf, output, ts, options, &mut col)
-                    .map_err_context(|| translate!("expand-error-failed-to-write-output"))?;
+                if let Err(e) = expand_buf(buf, output, ts, options, &mut col, &mut init) {
+                    break Err(
+                        e.map_err_context(|| translate!("expand-error-failed-to-write-output"))
+                    );
+                }
                 let len = buf.len();
                 input.consume(len);
             }
-            Err(e) => return Err(e.map_err_context(|| file.maybe_quote().to_string())),
+            Err(e) => break Err(e.map_err_context(|| file.maybe_quote().to_string())),
         }
-    }
-    Ok(())
+    };
+    *line_col = col;
+    *line_init = init;
+    result
 }
 
 fn expand(options: &Options) -> UResult<()> {
     let mut output = BufWriter::new(stdout());
+    // Input files form a single stream: a line that does not end in a
+    // newline continues in the next file.
+    let mut col = 0;
+    let mut init = true;
 
     for file in &options.files {
-        if let Err(e) = expand_file(file, &mut output, options) {
+        if let Err(e) = expand_file(file, &mut output, options, &mut col, &mut init) {
             show!(e);
             set_exit_code(1);
         }

tests/by-util/test_expand.rs

@@ -7,6 +7,7 @@
 
 use rstest::rstest;
 use uucore::display::Quotable;
+use uutests::at_and_ucmd;
 use uutests::new_ucmd;
 
 #[test]
@@ -429,7 +430,6 @@ fn test_read_error() {
 #[cfg_attr(wasi_runner, ignore = "WASI: argv/filenames must be valid UTF-8")]
 fn test_expand_non_utf8_paths() {
     use std::os::unix::ffi::OsStringExt;
-    use uutests::at_and_ucmd;
 
     let (at, mut ucmd) = at_and_ucmd!();
 
@@ -468,3 +468,67 @@ fn test_buffered_reads_new_line_no_tabs_in_first_chunk() {
         .succeeds()
         .stdout_is_fixture("new_line_in_chunk_expected.txt");
 }
+
+#[test]
+fn test_line_continues_into_next_file() {
+    // The first file does not end in a newline, so the tab in the second
+    // file is at column 1 of the same line.
+    let (at, mut ucmd) = at_and_ucmd!();
+    at.write("f1", "a");
+    at.write("f2", "\tb\n");
+    ucmd.args(&["f1", "f2"]).succeeds().stdout_is("a       b\n");
+
+    // With -i, that tab follows a non-blank, so it is not converted.
+    let (at, mut ucmd) = at_and_ucmd!();
+    at.write("f1", "a");
+    at.write("f2", "\tb\n");
+    ucmd.args(&["-i", "f1", "f2"])
+        .succeeds()
+        .stdout_is("a\tb\n");
+}
+
+#[test]
+fn test_tab_after_chunk_without_newline() {
+    // The first 8 KiB read has no tab and no newline. The tab after it must
+    // be expanded from its real column (8193), not from the read boundary.
+    let line = "a".repeat(8190) + "bcd";
+    let (at, mut ucmd) = at_and_ucmd!();
+    at.write("in", &format!("{line}\tx\n"));
+    ucmd.args(&["-t3", "in"])
+        .succeeds()
+        .stdout_is(format!("{line}   x\n"));
+
+    // With -i, a tab starting the second read still follows non-blanks on
+    // the same line, so it is not converted.
+    let line = "a".repeat(8192);
+    let (at, mut ucmd) = at_and_ucmd!();
+    at.write("in", &format!("{line}\tx\n"));
+    ucmd.args(&["-i", "in"])
+        .succeeds()
+        .stdout_is(format!("{line}\tx\n"));
+}
+
+#[test]
+fn test_tab_after_chunk_ending_in_wide_chars() {
+    // The first 8 KiB read has no tab and ends with three U+4E2D (3 bytes,
+    // width 2 each). The tab after them is at column 6, not 9.
+    let first = "x".repeat(8192 - 1 - 9);
+    let (at, mut ucmd) = at_and_ucmd!();
+    at.write("in", &format!("{first}\n\u{4E2D}\u{4E2D}\u{4E2D}\t|\n"));
+    ucmd.arg("in")
+        .succeeds()
+        .stdout_is(format!("{first}\n\u{4E2D}\u{4E2D}\u{4E2D}  |\n"));
+}
+
+#[test]
+fn test_line_continues_past_unreadable_file() {
+    // A file that cannot be read does not end the line started before it.
+    let (at, mut ucmd) = at_and_ucmd!();
+    at.write("f1", "a");
+    at.mkdir("dir");
+    at.write("f2", "\tb\n");
+    ucmd.args(&["f1", "dir", "f2"])
+        .fails()
+        .stdout_is("a       b\n")
+        .stderr_contains("expand: dir: Is a directory");
+}