diff --git a/crates/bashkit/src/interpreter/coproc.rs b/crates/bashkit/src/interpreter/coproc.rs new file mode 100644 index 000000000..9cbb4b5b3 --- /dev/null +++ b/crates/bashkit/src/interpreter/coproc.rs @@ -0,0 +1,390 @@ +//! Coprocesses: `coproc [NAME] command`. +//! +//! Important decisions: +//! - A coproc is a background job (see `jobs.rs`) whose stdin and stdout +//! are two in-process [`pipe::Pipe`]s, the same bounded pipe concurrent +//! pipeline stages use. The body runs on a forked shell (subshell +//! isolation, nothing touches the host), polled together with the +//! foreground on the same task, so it makes progress whenever the shell +//! blocks: on a read of `${NAME[0]}`, on a full `${NAME[1]}`, in `wait`. +//! A request/response loop (`echo q >&${C[1]}; read r <&${C[0]}`) +//! therefore works line by line, and `tr`-style bodies that read to EOF +//! finish once the write end is closed. +//! - Descriptors: `${NAME[1]}` is an `exec_fd_table` entry +//! ([`FdTarget::Coproc`]), `${NAME[0]}` an [`InputFd::Pipe`] in +//! `coproc_buffers`. Both hold the pipe end behind an `Arc`, so dups +//! (`exec 5>&${C[1]}`) and subshell snapshots share it and the end closes +//! (EOF for the coproc, SIGPIPE for its writes) when the last copy goes. +//! The coproc's own fork never sees its write end, or another coproc's. +//! - fd numbers follow bash, which moves all four pipe ends to the highest +//! free fds below 64 (parent read, child write, child read, parent +//! write) and keeps the outer two: 63/60 for the first coproc, 62/58 +//! for a second one while the first is open. +//! - Cleanup is deterministic, not on SIGCHLD timing: once `wait` has +//! reaped the coproc's job, the next command boundary unsets NAME and +//! NAME_PID and closes its fds. Output it wrote before exiting stays +//! readable until then (bash may lose it if it reaps first). +//! - Jobs are scoped to one `exec()` call, so are coprocs: at the end of +//! `exec()` the shell's ends are closed (the coproc sees EOF) and NAME is +//! unset before the remaining jobs are awaited. +//! - Sequential mode (`concurrent_jobs(false)`) runs the body to completion +//! at `coproc` with stdin at end of input; its output stays readable. +//! - THREAT[TM-DOS-063]/[TM-DOS-122]: a coproc takes a background-job slot +//! and two persistent descriptors; pipes buffer at most +//! [`pipe::PIPE_CAPACITY`] plus one command's output before the writer +//! waits. + +use super::*; + +/// A readable descriptor (`exec 3), + /// Read end of a coproc's stdout pipe. + Pipe(Arc), +} + +impl InputFd { + pub(super) fn pipe(&self) -> Option<&Arc> { + match self { + Self::Pipe(end) => Some(&end.0), + Self::Lines(_) => None, + } + } +} + +/// Write end of a coproc's stdin pipe, as an fd-table target. +#[derive(Clone)] +pub(super) struct CoprocWriter(Arc); + +impl std::fmt::Debug for CoprocWriter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("CoprocWriter") + } +} + +impl CoprocWriter { + pub(super) fn pipe(&self) -> &Arc { + &self.0.0 + } + + /// Append `data`. A write after the coproc exited is dropped. + // WTF: bash gets SIGPIPE (the shell dies) writing to a coproc that + // exited but was not reaped yet; a sandboxed shell should not. + pub(super) fn write(&self, data: &[u8]) { + let _ = self.pipe().write(data); + } +} + +/// Wait while the coproc's stdin pipe holds a full buffer (it drains as +/// the coproc reads, or the coproc exits). +pub(super) async fn coproc_backpressure(w: &CoprocWriter) { + Box::pin(w.pipe().writable()).await; +} + +/// Read what a command with `demand` needs from a coproc's output pipe. +/// Line-wise demands leave the rest in the pipe. +pub(super) async fn read_coproc_input( + pipe: Arc, + demand: StdinDemand, +) -> crate::StreamData { + let mut buf = Vec::new(); + while !demand.satisfied(&buf) { + let chunk = if matches!(demand, StdinDemand::All | StdinDemand::Bytes(_)) { + pipe.read_some().await + } else { + pipe.read_line_chunk().await + }; + if chunk.is_empty() { + break; + } + buf.extend_from_slice(&chunk); + } + buf.into() +} + +/// The simple command a coproc body consists of (`cat`, `{ cat; }`). +fn lone_simple_command(body: &Command) -> Option<&SimpleCommand> { + match body { + Command::Simple(simple) => Some(simple), + Command::Compound(CompoundCommand::BraceGroup(cmds), redirects) if redirects.is_empty() => { + match cmds.as_slice() { + [Command::Simple(simple)] => Some(simple), + _ => None, + } + } + _ => None, + } +} + +/// A live coproc of this shell. +pub(super) struct CoprocEntry { + name: String, + /// Job-table key. + job: usize, + pid: u32, + read_fd: i32, + write_fd: i32, + /// The coproc's stdout pipe (read by `${NAME[0]}`). + output: Arc, + /// The coproc's stdin pipe (written by `${NAME[1]}`). + input: Arc, +} + +impl Interpreter { + /// `coproc [NAME] command`. Boxed: this is one arm of the compound + /// dispatch, so its state stays off that future. + pub(super) fn execute_coproc<'a>( + &'a mut self, + coproc: &'a CoprocCommand, + ) -> std::pin::Pin> + Send + 'a>> { + Box::pin(async move { + let mut stderr = String::new(); + if let Some(warning) = self.coproc_still_exists_warning() { + stderr.push_str(&warning); + } + let (read_fd, write_fd) = self.allocate_coproc_fds()?; + + let input = pipe::Pipe::new(); + let output = pipe::Pipe::new(); + let Some((mut child, slot)) = self.fork_for_coproc(&mut stderr) else { + self.last_exit_code = 1; + return Ok(ExecResult::err(stderr, 1)); + }; + child + .exec_fd_table + .retain(|_, t| !matches!(t, FdTarget::Coproc(_))); + if self.concurrent_jobs { + child.pipe_in = Some(Arc::clone(&input)); + child.pipe_out = Some(Arc::clone(&output)); + let sink = Arc::clone(&output); + child.output_callback = Some(Box::new(move |out, _err| { + sink.write(out.as_bytes()); + })); + } else { + // Nobody can write before the body has run: stdin is at EOF. + input.close_write(); + child.pipeline_stdin = Some(crate::StreamData::new()); + } + let body = coproc.body.clone(); + let streaming = self.concurrent_jobs; + let job_diag = self.diag_prefix(); + let read_end = pipe::ReadEnd(Arc::clone(&input)); + let write_end = pipe::WriteEnd(Arc::clone(&output)); + let mut fut: jobs::JobFuture = Box::pin(async move { + let _slot = slot; + let _read_end = read_end; + // A lone `cat`/`tr`/`grep` (`coproc cat`, `coproc { tr a-z + // A-Z; }`) streams like a pipeline stage: each line written + // in comes out before end of input. + if streaming + && let Some(simple) = lone_simple_command(&body) + && Self::streams_stdout(simple) + { + child.stream_stdout_command = Some(simple as *const SimpleCommand as usize); + } + let jobs = Arc::clone(&child.jobs); + let result = jobs::with_jobs(&jobs, child.execute_command(&body)).await; + jobs.finish_all().await; + let (out, err) = jobs.lock().take_finished_output(); + let mut r = match result { + Ok(r) => r, + Err(e) => return ExecResult::err(format!("{job_diag}{e}\n"), 1), + }; + r.stdout.append(&out); + r.stderr.append(&err); + // Output not yet sent at a command boundary goes now. + let sent = child.output_stream_stdout_bytes.min(r.stdout.len()); + write_end.0.write(&r.stdout.as_bytes()[sent..]); + r.stdout = crate::StreamData::new(); + if write_end.0.is_broken() { + r.exit_code = 141; + } else if let ControlFlow::Exit(code) | ControlFlow::Return(code) = r.control_flow { + r.exit_code = code; + } + r.control_flow = ControlFlow::None; + r + }); + + let text = format!("coproc {} {}", coproc.name, describe_command(&coproc.body)); + // Poll once (sequential mode: to completion). A body that never + // blocks finishes here; its stderr is this command's. + let finished = if self.concurrent_jobs { + match std::future::poll_fn(|cx| std::task::Poll::Ready(fut.as_mut().poll(cx))).await + { + std::task::Poll::Ready(r) => Some(r), + std::task::Poll::Pending => None, + } + } else { + Some(fut.as_mut().await) + }; + let (job, pid) = match finished { + Some(result) => { + stderr.push_str(&result.stderr.text_lossy()); + self.jobs.lock().spawn_finished(text, result) + } + None => self.jobs.lock().spawn_running(text, fut), + }; + + self.coproc_buffers.insert( + read_fd, + InputFd::Pipe(Arc::new(pipe::ReadEnd(Arc::clone(&output)))), + ); + self.exec_fd_table.insert( + write_fd, + FdTarget::Coproc(CoprocWriter(Arc::new(pipe::WriteEnd(Arc::clone(&input))))), + ); + let name = coproc.name.clone(); + let mut arr = HashMap::new(); + arr.insert(0, read_fd.to_string()); + arr.insert(1, write_fd.to_string()); + self.arrays_mut().insert(name.clone(), arr); + self.vars_mut() + .insert(format!("{name}_PID"), pid.to_string()); + self.last_bg_pid = Some(pid.to_string()); + self.coprocs.push(CoprocEntry { + name, + job, + pid, + read_fd, + write_fd, + output, + input, + }); + self.last_exit_code = 0; + Ok(ExecResult { + stderr: stderr.into(), + ..Default::default() + }) + }) + } + + /// bash keeps one coproc at a time and warns when a new one starts + /// while it runs (the new one still starts). + fn coproc_still_exists_warning(&self) -> Option { + let jobs = self.jobs.lock(); + let running: Vec = jobs.running_ids(); + let live = self + .coprocs + .iter() + .rev() + .find(|c| running.contains(&c.job))?; + Some(self.diag(format!( + "warning: execute_coproc: coproc [{}:{}] still exists\n", + live.pid, live.name + ))) + } + + /// The fd pair for a new coproc (see the module docs). + fn allocate_coproc_fds(&self) -> Result<(i32, i32)> { + let mut free = Vec::with_capacity(4); + let mut fd = redirection::COPROC_FIRST_FD; + while free.len() < 4 && fd >= 3 { + if !self.fd_is_open(fd) { + free.push(fd); + } + fd -= 1; + } + let max = self.limits.max_file_descriptors; + if free.len() < 4 || self.persistent_fd_count() + 2 > max { + return Err(crate::limits::LimitExceeded::MaxFileDescriptors(max).into()); + } + Ok((free[0], free[3])) + } + + /// Fork the shell for a coproc body and claim its job slot. On failure + /// the fork error is appended to `stderr`. + fn fork_for_coproc(&mut self, stderr: &mut String) -> Option<(Interpreter, jobs::JobSlot)> { + // THREAT[TM-DOS-122]: a coproc is a background job. + let slot = self.jobs.try_claim(self.limits.max_background_jobs); + let mut child = self.fork_for_job(); + let depth_ok = child.counters.push_subshell(&child.limits).is_ok(); + match slot.filter(|_| depth_ok) { + Some(slot) => Some((child, slot)), + None => { + stderr.push_str(&format!( + "{}: fork: retry: Resource temporarily unavailable (max {} background jobs, {} nested)\n", + self.diag_name(), + self.limits.max_background_jobs, + self.limits.max_subshell_depth + )); + None + } + } + } + + /// At a command boundary: forget coprocs whose job `wait` reaped, + /// unsetting NAME / NAME_PID and closing their fds like bash does. + #[inline(never)] + pub(super) fn reap_coprocs(&mut self) { + if self.coprocs.is_empty() { + return; + } + let ids = self.jobs.lock().ids(); + let (gone, live): (Vec<_>, Vec<_>) = std::mem::take(&mut self.coprocs) + .into_iter() + .partition(|c| !ids.contains(&c.job)); + self.coprocs = live; + for c in gone { + self.drop_coproc(c); + } + } + + /// End of `exec()`: close every coproc's fds (it sees EOF) and unset + /// its variables before the remaining jobs are awaited. + pub(super) fn close_coprocs(&mut self) { + for c in std::mem::take(&mut self.coprocs) { + self.drop_coproc(c); + } + } + + fn drop_coproc(&mut self, c: CoprocEntry) { + if self + .coproc_buffers + .get(&c.read_fd) + .and_then(InputFd::pipe) + .is_some_and(|p| Arc::ptr_eq(p, &c.output)) + { + self.coproc_buffers.remove(&c.read_fd); + } + if matches!(self.exec_fd_table.get(&c.write_fd), + Some(FdTarget::Coproc(w)) if Arc::ptr_eq(w.pipe(), &c.input)) + { + self.exec_fd_table.remove(&c.write_fd); + } + self.arrays_mut().remove(&c.name); + self.vars_mut().remove(&format!("{}_PID", c.name)); + } + + /// `read -u FD` on a readable fd: the line it reads, or the coproc pipe + /// to read it from. + pub(super) fn read_u_source(&mut self, args: &[String]) -> Option { + let mut iter = args.iter(); + let fd = loop { + let arg = iter.next()?; + if arg == "-u" { + break iter.next()?.parse::().ok()?; + } + if let Some(rest) = arg.strip_prefix("-u") + && let Ok(fd) = rest.parse::() + { + break fd; + } + }; + match self.coproc_buffers.get_mut(&fd)? { + InputFd::Lines(buf) => Some(ReadSource::Ready( + buf.pop().map(|l| format!("{l}\n")).unwrap_or_default(), + )), + InputFd::Pipe(end) => Some(ReadSource::Pipe(Arc::clone(&end.0))), + } + } +} + +/// Where `read -u FD` takes its input from. +pub(super) enum ReadSource { + /// Already available (empty at end of input). + Ready(String), + /// A coproc's output pipe. + Pipe(Arc), +} diff --git a/crates/bashkit/src/interpreter/jobs.rs b/crates/bashkit/src/interpreter/jobs.rs index 9c77f9190..eaab67451 100644 --- a/crates/bashkit/src/interpreter/jobs.rs +++ b/crates/bashkit/src/interpreter/jobs.rs @@ -169,6 +169,15 @@ impl JobTable { self.jobs.values().filter(|j| j.result.is_none()).count() } + /// Keys of the jobs still running. + pub fn running_ids(&self) -> Vec { + self.jobs + .iter() + .filter(|(_, j)| j.result.is_none()) + .map(|(id, _)| *id) + .collect() + } + /// Running jobs as `(number, key)`, oldest first. fn running_by_number(&self) -> Vec<(usize, usize)> { let mut v: Vec<(usize, usize)> = self diff --git a/crates/bashkit/src/interpreter/mod.rs b/crates/bashkit/src/interpreter/mod.rs index d12f0340b..95b357863 100644 --- a/crates/bashkit/src/interpreter/mod.rs +++ b/crates/bashkit/src/interpreter/mod.rs @@ -27,6 +27,7 @@ mod arithmetic; mod brace_expansion; +mod coproc; mod declare; mod expansion; mod glob; @@ -1328,7 +1329,11 @@ struct PipelineAcc { } /// Subshell state a pipeline stage rolls back. -type StageSnapshot = (SubshellSnapshot, Vec, HashMap>); +type StageSnapshot = ( + SubshellSnapshot, + Vec, + HashMap, +); /// State held across one pipeline stage (see `enter_pipeline_stage`). struct PipelineStageScope { @@ -1566,12 +1571,11 @@ pub struct Interpreter { /// Used to detect whether assignment value expansion ran a command substitution /// (for correct exit code: plain assignment → 0, assignment with subst → subst's exit code). subst_generation: u64, - /// Coprocess read buffers: maps virtual FD number to remaining lines. - /// When a coproc runs, its stdout is split into lines and stored here - /// so `read -u FD` or `read <&FD` can consume them one at a time. - coproc_buffers: HashMap>, - /// Next virtual FD to assign for coproc read ends (starts at 63, like bash). - coproc_next_fd: i32, + /// Readable descriptors (fd >= 3): remaining lines of `exec 3, + /// Live coprocs of this shell, for cleanup (`coproc.rs`). Not forked. + coprocs: Vec, /// Persistent fd output table set by `exec N>/path` redirections. /// Maps fd number to its output target. Used by `>&N` redirections; /// entries for fd 1 and 2 (`exec >log 2>&1`) route the shell's own @@ -2150,7 +2154,7 @@ impl Interpreter { history_loaded: false, subst_generation: 0, coproc_buffers: HashMap::new(), - coproc_next_fd: redirection::COPROC_FIRST_FD, + coprocs: Vec::new(), exec_fd_table: HashMap::new(), exec_input_fds: HashSet::new(), fd_redirect_scope: Vec::new(), @@ -2336,7 +2340,7 @@ impl Interpreter { history_loaded: true, subst_generation: self.subst_generation, coproc_buffers: HashMap::new(), - coproc_next_fd: self.coproc_next_fd, + coprocs: Vec::new(), exec_fd_table: self.exec_fd_table.clone(), exec_input_fds: self.exec_input_fds.clone(), fd_redirect_scope: self.fd_redirect_scope.clone(), @@ -3376,7 +3380,9 @@ impl Interpreter { jobs::with_jobs(&jobs, self.execute_script_body(script, true, true)).await; // Script boundary: background jobs are scoped to a single exec() // call. Like a pipe reader waiting for EOF, the call waits for - // every job and delivers output not yet reported. + // every job and delivers output not yet reported. Coprocs first + // lose the shell's ends, as when a script exits. + self.close_coprocs(); jobs.finish_all().await; let (out, err) = jobs.lock().take_finished_output(); if let Ok(r) = &mut result { @@ -3468,6 +3474,7 @@ impl Interpreter { result.stdout.append(&out); result.stderr.append(&err); } + self.reap_coprocs(); self.check_cancelled()?; if top_level { self.route_exec_output( @@ -5436,81 +5443,6 @@ impl Interpreter { Ok(()) } - /// Execute a coprocess command. - /// - /// Runs the command body synchronously (bashkit's deterministic model), - /// buffers its stdout for later reading via virtual FDs, sets the NAME - /// array with FD numbers, and stores a virtual PID in NAME_PID. - async fn execute_coproc(&mut self, coproc: &CoprocCommand) -> Result { - let name = &coproc.name; - - // Allocate virtual FD numbers (bash uses 63/60 by default) - let read_fd = self.coproc_next_fd; - let write_fd = self.coproc_next_fd - 1; - self.coproc_next_fd -= 2; // reserve pair for next coproc - - // Execute the command body while suppressing streaming callbacks. - // Coproc output must stay internal and be consumed only via read -u / <&FD. - let saved_callback = self.output_callback.take(); - let result = self.execute_command(&coproc.body).await; - if let Some(callback) = saved_callback { - self.output_callback = Some(callback); - } - let result = result?; - - // Buffer stdout lines for reading via the virtual read FD. - // Lines are stored in reverse order so pop() yields the first line. - let mut lines: Vec = result.stdout.lines().map(|l| l.to_string()).collect(); - lines.reverse(); - self.coproc_buffers.insert(read_fd, lines); - - // Set NAME array: NAME[0] = read FD, NAME[1] = write FD - let mut arr = HashMap::new(); - arr.insert(0, read_fd.to_string()); - arr.insert(1, write_fd.to_string()); - self.arrays_mut().insert(name.clone(), arr); - - // Set NAME_PID to a virtual PID (use job table counter) - let virtual_pid = self.jobs.lock().alloc_pid(); - self.vars_mut() - .insert(format!("{}_PID", name), virtual_pid.to_string()); - - // Also set $! (last background PID) - self.last_bg_pid = Some(virtual_pid.to_string()); - - // Coproc itself returns success with empty output (stdout was captured) - Ok(ExecResult::ok(String::new())) - } - - /// Check if `read -u FD` args reference a coproc FD and return next line if so. - fn try_coproc_read_stdin(&mut self, args: &[String]) -> Option { - let mut iter = args.iter(); - while let Some(arg) = iter.next() { - if arg == "-u" - && let Some(fd_str) = iter.next() - && let Ok(fd) = fd_str.parse::() - && let Some(buf) = self.coproc_buffers.get_mut(&fd) - { - return if let Some(line) = buf.pop() { - Some(format!("{}\n", line)) - } else { - Some(String::new()) // EOF - }; - } else if arg.starts_with("-u") - && arg.len() > 2 - && let Ok(fd) = arg[2..].parse::() - && let Some(buf) = self.coproc_buffers.get_mut(&fd) - { - return if let Some(line) = buf.pop() { - Some(format!("{}\n", line)) - } else { - Some(String::new()) // EOF - }; - } - } - None - } - /// Execute `bash` or `sh` command - interpret scripts using this interpreter. /// /// Supports: @@ -6376,6 +6308,8 @@ enum FdTarget { DevNull, /// Closed by `exec N>&-`: a write to it fails. Closed, + /// A coproc's stdin pipe (`${NAME[1]}`). + Coproc(coproc::CoprocWriter), } /// Route fd1/fd2/fd3+ content to their targets. Extracted from the async @@ -6415,6 +6349,7 @@ fn route_fd_table_content( } // A closed descriptor accepts nothing. FdTarget::DevNull | FdTarget::Closed => {} + FdTarget::Coproc(w) => w.write(data.as_bytes()), FdTarget::WriteFile(p, d) => { let entry = fw .entry(p.clone()) @@ -6524,6 +6459,7 @@ impl Interpreter { let emit_before = self.output_emit_count; self.sequence_accum = (stdout.len(), stderr.len()); let result = self.execute_command(command).await?; + self.reap_coprocs(); self.maybe_emit_output(&result.stdout, &result.stderr, emit_before); stdout.append(&result.stdout); stderr.append(&result.stderr); @@ -8138,7 +8074,7 @@ impl Interpreter { // Handle input redirections first let stdin = match self - .process_input_redirections(stdin, &command.redirects) + .process_input_redirections(stdin, &command.redirects, stdin_demand(name, &args)) .await { Ok(s) => s, @@ -8148,11 +8084,17 @@ impl Interpreter { Err(e) => return Err(e), }; - // For `read -u FD`, check if FD is a coproc read FD and inject data as stdin - let stdin = if name == "read" && stdin.is_none() { - self.try_coproc_read_stdin(&args).map(Into::into).or(stdin) - } else { - stdin + // `read -u FD` on a readable fd (`exec 3 Some(line.into()), + Some(coproc::ReadSource::Pipe(pipe)) => { + Some(Box::pin(coproc::read_coproc_input(pipe, stdin_demand(name, &args))).await) + } + None => stdin, }; if stdin.is_none() && self.pipe_in.is_some() { @@ -8305,6 +8247,10 @@ impl Interpreter { FdTarget::WriteFile(path, _) | FdTarget::AppendFile(path, _) => { self.fs.append_file(&path, data.as_bytes()).await?; } + FdTarget::Coproc(ref w) => { + w.write(data.as_bytes()); + coproc::coproc_backpressure(w).await; + } } } if write_failed { @@ -8362,6 +8308,14 @@ impl Interpreter { } } + /// Persistent descriptors open in this shell (TM-DOS-063). + fn persistent_fd_count(&self) -> usize { + let mut open_fds: HashSet = self.exec_fd_table.keys().copied().collect(); + open_fds.extend(self.coproc_buffers.keys().copied()); + open_fds.extend(self.exec_input_fds.iter().copied()); + open_fds.len() + } + fn ensure_persistent_fd_capacity(&self, fd: i32) -> Result<()> { if fd < 0 { return Err(crate::error::Error::Execution(format!( @@ -8378,11 +8332,7 @@ impl Interpreter { return Ok(()); } - let mut open_fds: HashSet = self.exec_fd_table.keys().copied().collect(); - open_fds.extend(self.coproc_buffers.keys().copied()); - open_fds.extend(self.exec_input_fds.iter().copied()); - - if open_fds.len() >= self.limits.max_file_descriptors { + if self.persistent_fd_count() >= self.limits.max_file_descriptors { return Err(crate::limits::LimitExceeded::MaxFileDescriptors( self.limits.max_file_descriptors, ) @@ -11267,7 +11217,10 @@ impl Interpreter { } else { redirects }; - match self.process_input_redirections(None, redirects).await { + match self + .process_input_redirections(None, redirects, StdinDemand::Nothing) + .await + { Ok(_) => {} Err(crate::error::Error::CommandFailure(msg)) => { self.last_exit_code = 1; @@ -11345,8 +11298,14 @@ impl Interpreter { if let Some(line) = end_line { self.current_line = line; } - // Process input redirections before executing compound - let stdin = match self.process_input_redirections(None, redirects).await { + // Process input redirections before executing compound. + // WTF: a coproc fd as a compound's stdin (`while read l; do + // ...; done <&${C[0]}`) is read to end of input up front, not + // line by line as the loop runs. + let stdin = match self + .process_input_redirections(None, redirects, StdinDemand::All) + .await + { Ok(s) => s, Err(crate::error::Error::CommandFailure(msg)) => { return Ok(ExecResult::err(msg, 1)); @@ -11449,7 +11408,8 @@ impl Interpreter { let read = if self.disabled_redirect_error(redirects).is_some() { Err(crate::error::Error::CommandFailure(String::new())) } else { - self.process_input_redirections(None, redirects).await + self.process_input_redirections(None, redirects, StdinDemand::All) + .await }; match read { Ok(content) => { diff --git a/crates/bashkit/src/interpreter/redirection.rs b/crates/bashkit/src/interpreter/redirection.rs index 465398e9b..1ec24e522 100644 --- a/crates/bashkit/src/interpreter/redirection.rs +++ b/crates/bashkit/src/interpreter/redirection.rs @@ -41,11 +41,13 @@ impl Interpreter { self.diag(format!("{path}: {reason}\n")) } - /// Process input redirections (< file, <<< string) + /// Process input redirections (< file, <<< string). `demand` is how + /// much the command reads, for stdin taken from a coproc's pipe. pub(super) async fn process_input_redirections( &mut self, existing_stdin: Option, redirects: &[Redirect], + demand: StdinDemand, ) -> Result> { let mut stdin = existing_stdin; // Readable fds this command's own redirects open (`4< { let target = self.expand_word(&redirect.target).await?; + if target.is_empty() && high_fd.is_none() { + // `<&${C[0]}` after the coproc was reaped: an error, + // not a read of the shell's stdin. + // WTF: quoted (`<&"${C[0]}"`) bash says `Bad file + // descriptor` and names the word with its quotes; + // the AST does not keep that quoting here. + return Err(crate::error::Error::CommandFailure(self.redirect_error( + &redirect_target_label(&redirect.target), + "ambiguous redirect", + ))); + } let Ok(src) = target.parse::() else { continue; }; @@ -160,13 +173,22 @@ impl Interpreter { if high_fd.is_some() { continue; } - // <&FD - if FD is a coproc read FD, consume next line - if let Some(buf) = self.coproc_buffers.get_mut(&src) { - if let Some(line) = buf.pop() { - stdin = Some(format!("{}\n", line).into()); - } else { - stdin = Some(crate::StreamData::new()); // EOF + // <&FD on a readable fd: its next line, or what the + // command reads of a coproc's output. + match self.coproc_buffers.get_mut(&src) { + Some(coproc::InputFd::Lines(buf)) => { + stdin = Some( + buf.pop() + .map(|line| format!("{line}\n")) + .unwrap_or_default() + .into(), + ); + } + Some(coproc::InputFd::Pipe(end)) => { + let pipe = Arc::clone(&end.0); + stdin = Some(Box::pin(coproc::read_coproc_input(pipe, demand)).await); } + None => {} } } _ => { @@ -502,6 +524,10 @@ impl Interpreter { FdTarget::WriteFile(path, _) | FdTarget::AppendFile(path, _) => { self.fs.append_file(path, data.as_bytes()).await?; } + FdTarget::Coproc(w) => { + w.write(data.as_bytes()); + coproc::coproc_backpressure(w).await; + } } } else { match (src_fd, target_fd) { @@ -711,6 +737,15 @@ impl Interpreter { &self.pending_fd_targets, &self.pending_fd_output, ); + // A coproc fed past its pipe's capacity drains first. + for target in [&fd1, &fd2] + .into_iter() + .chain(self.pending_fd_targets.iter().map(|(_, t)| t)) + { + if let FdTarget::Coproc(w) = target { + coproc::coproc_backpressure(w).await; + } + } self.clear_pending_fd_redirect_state(); // Write files @@ -794,8 +829,8 @@ pub(super) fn has_high_fd_file_redirect(redirects: &[Redirect]) -> bool { // Important decisions: // - The shell's descriptors are virtual, per interpreter (tenant isolation): // output fds live in `exec_fd_table`, readable fds in `coproc_buffers` -// (remaining lines), and dups of stdin in `exec_input_fds`. No host fd is -// ever touched. +// (remaining lines, or a coproc's output pipe: `coproc.rs`), and dups of +// stdin in `exec_input_fds`. No host fd is ever touched. // - `{var}>file` allocates the lowest free fd >= 10 (bash's // `fcntl(F_DUPFD, 10)`) and stores it in `var`. bash never undoes a // `{var}` redirection, so on any command (not just `exec`) the fd stays @@ -885,8 +920,6 @@ impl Interpreter { || self.coproc_buffers.contains_key(&fd) || self.exec_input_fds.contains(&fd) || self.fd_redirect_scope.contains(&fd) - // coproc pairs, read and write ends - || (fd > self.coproc_next_fd && fd <= COPROC_FIRST_FD) } fn close_fd(&mut self, fd: i32) { @@ -1022,7 +1055,8 @@ impl Interpreter { Some(n) => { self.ensure_persistent_fd_capacity(n)?; self.close_fd(n); - self.coproc_buffers.insert(n, Vec::new()); + self.coproc_buffers + .insert(n, coproc::InputFd::Lines(Vec::new())); if read_write { self.exec_fd_table.insert(n, FdTarget::DevNull); } @@ -1051,7 +1085,8 @@ impl Interpreter { Some(n) => { self.ensure_persistent_fd_capacity(n)?; self.close_fd(n); - self.coproc_buffers.insert(n, reversed_lines(&text)); + self.coproc_buffers + .insert(n, coproc::InputFd::Lines(reversed_lines(&text))); if read_write { self.exec_fd_table .insert(n, FdTarget::AppendFile(path, target_path)); @@ -1070,7 +1105,8 @@ impl Interpreter { Some(n) => { self.ensure_persistent_fd_capacity(n)?; self.close_fd(n); - self.coproc_buffers.insert(n, reversed_lines(&content)); + self.coproc_buffers + .insert(n, coproc::InputFd::Lines(reversed_lines(&content))); } None => self.pipeline_stdin = Some(content.into()), } @@ -1104,9 +1140,17 @@ impl Interpreter { } match n { 0 => { - // `exec <&N`: stdin reads what is left of fd N. - if let Some(buf) = self.coproc_buffers.get(&m) { - self.pipeline_stdin = Some(buffer_text(buf)); + // `exec <&N`: stdin reads what is left of fd N + // (a coproc's output: as commands read it). + match self.coproc_buffers.get(&m) { + Some(coproc::InputFd::Lines(buf)) => { + self.pipeline_stdin = Some(buffer_text(buf)); + } + Some(coproc::InputFd::Pipe(end)) => { + self.pipeline_stdin = None; + self.pipe_in = Some(Arc::clone(&end.0)); + } + None => {} } } 1 | 2 => { diff --git a/crates/bashkit/src/parser/ast.rs b/crates/bashkit/src/parser/ast.rs index f5b6d55ce..96eb77fe0 100644 --- a/crates/bashkit/src/parser/ast.rs +++ b/crates/bashkit/src/parser/ast.rs @@ -127,8 +127,8 @@ pub enum CompoundCommand { /// Coprocess command - runs a command with bidirectional communication. /// -/// In bashkit's sandboxed model, the coprocess runs synchronously and its -/// stdout is buffered for later reading via the NAME array FDs. +/// The body runs as a background job on a forked shell, its stdin and +/// stdout connected to in-process pipes (`interpreter/coproc.rs`). /// `NAME[0]` = virtual read FD, `NAME[1]` = virtual write FD, `NAME_PID` = virtual PID. #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub struct CoprocCommand { diff --git a/crates/bashkit/tests/integration/coproc_tests.rs b/crates/bashkit/tests/integration/coproc_tests.rs index d4e697587..864e1a130 100644 --- a/crates/bashkit/tests/integration/coproc_tests.rs +++ b/crates/bashkit/tests/integration/coproc_tests.rs @@ -24,7 +24,7 @@ echo "pid=$COPROC_PID" .unwrap(); assert_eq!(result.exit_code, 0); assert!(result.stdout.contains("read_fd=63")); - assert!(result.stdout.contains("write_fd=62")); + assert!(result.stdout.contains("write_fd=60")); assert!(result.stdout.contains("pid=")); // PID should be a number > 0 let pid_line = result @@ -231,3 +231,146 @@ echo "visible:$line" "coproc body output must not be visible to streaming consumers", ); } + +async fn run(script: &str) -> bashkit::ExecResult { + Bash::new().exec(script).await.unwrap() +} + +/// Writes to `${NAME[1]}` feed the coproc's stdin; closing the write end +/// delivers EOF (bash-oracle `coproc-basic`). +#[tokio::test] +async fn coproc_stdin_fed_by_write_fd() { + let result = run(r#" +coproc UPPER { tr a-z A-Z; } +echo "texto" >&"${UPPER[1]}" +exec {UPPER[1]}>&- +read -r line <&"${UPPER[0]}" +echo "$line" +wait +echo "after wait: [${UPPER[@]}] [${UPPER_PID-unset}] ${#UPPER[@]}" +"#) + .await; + assert_eq!(result.stdout, "TEXTO\nafter wait: [] [unset] 0\n"); + assert_eq!(result.stderr, ""); +} + +/// Request/response: write a line, read a line, repeatedly, without +/// deadlock. Default name COPROC, fds 63/60 like bash. +#[tokio::test] +async fn coproc_request_response_loop() { + let result = run(r#" +coproc { while read l; do echo "got $l"; done; } +echo "${COPROC[@]}" +[ "$!" = "$COPROC_PID" ] && echo bang-ok +for i in 1 2 3; do echo "req $i" >&"${COPROC[1]}"; read -r r <&"${COPROC[0]}"; echo "$r"; done +echo x >&${COPROC[1]}; read -u ${COPROC[0]} r; echo "u: $r" +exec {COPROC[1]}>&- +read -r r <&"${COPROC[0]}"; echo "eof rc=$? [$r]" +wait $COPROC_PID; echo "wait rc=$?" +echo "[${COPROC[@]}] [${COPROC_PID-unset}]" +"#) + .await; + assert_eq!( + result.stdout, + "63 60\nbang-ok\ngot req 1\ngot req 2\ngot req 3\nu: got x\neof rc=1 []\nwait rc=0\n[] [unset]\n" + ); + assert_eq!(result.stderr, ""); +} + +/// `wait $NAME_PID` reports the coproc's status, then NAME and NAME_PID +/// are unset and the fd pair is free again. +#[tokio::test] +async fn coproc_exit_status_and_cleanup() { + let result = run(r#" +coproc C { exit 3; } +wait $C_PID; echo "C rc=$?" +echo "[${C[@]}] [${C_PID-unset}]" +coproc D { read x; exit 4; } +echo "[${D[@]}]" +echo hi >&${D[1]} +wait $D_PID; echo "D rc=$?" +echo "[${D[@]}] [${D_PID-unset}]" +"#) + .await; + assert_eq!( + result.stdout, + "C rc=3\n[] [unset]\n[63 60]\nD rc=4\n[] [unset]\n" + ); +} + +/// A second coproc while one still runs: bash warns and starts it anyway. +#[tokio::test] +async fn coproc_still_exists_warning() { + let result = run(r#" +coproc A { cat; } +coproc B { cat; } +echo "rc=$? ${A[@]} ${B[@]}" +exec {A[1]}>&- {B[1]}>&- +wait +"#) + .await; + assert_eq!(result.stdout, "rc=0 63 60 62 58\n"); + assert!( + result.stderr.contains("warning: execute_coproc: coproc [") + && result.stderr.contains(":A] still exists"), + "stderr: {}", + result.stderr + ); +} + +/// After the coproc is reaped its descriptors are closed. +#[tokio::test] +async fn coproc_fds_closed_after_reap() { + let result = run(r#" +coproc cat +echo one >&${COPROC[1]} +read -r -u ${COPROC[0]} x; echo "x=$x" +fd=${COPROC[1]} +exec {COPROC[1]}>&- +wait +echo hi >&$fd; echo "write rc=$?" +"#) + .await; + assert_eq!(result.stdout, "x=one\nwrite rc=1\n"); + assert!( + result.stderr.contains("Bad file descriptor"), + "stderr: {}", + result.stderr + ); +} + +/// The end of `exec()` closes the shell's coproc ends, so a coproc waiting +/// for input ends instead of hanging the call. +#[tokio::test] +async fn coproc_unclosed_ends_with_exec() { + let mut bash = Bash::new(); + let result = bash.exec("coproc cat\necho done").await.unwrap(); + assert_eq!(result.stdout, "done\n"); + let result = bash.exec("echo \"[${COPROC[@]}]\"").await.unwrap(); + assert_eq!(result.stdout, "[]\n"); +} + +/// Sequential mode (no concurrent jobs) runs the body up front with its +/// stdin at end of input. +#[tokio::test] +async fn coproc_sequential_mode_runs_body_eagerly() { + let mut bash = Bash::builder().concurrent_jobs(false).build(); + let result = bash + .exec("coproc { echo a; cat; echo b; }\nread -r x <&${COPROC[0]}; read -r y <&${COPROC[0]}; echo \"$x$y\"") + .await + .unwrap(); + assert_eq!(result.stdout, "ab\n"); +} + +/// A coproc holds a job slot: the background-job cap bounds coprocs too. +#[tokio::test] +async fn coproc_counts_against_job_limit() { + let mut bash = Bash::builder() + .limits(bashkit::ExecutionLimits::new().max_background_jobs(1)) + .build(); + let result = bash + .exec("coproc A { cat; }\ncoproc B { cat; } 2>/dev/null\necho \"rc=$? [${B[@]}]\"\nexec {A[1]}>&-") + .await + .unwrap(); + assert_eq!(result.stdout, "rc=1 []\n"); +} diff --git a/knowledge/foundations/parallel-execution.md b/knowledge/foundations/parallel-execution.md index 30f9e0c29..cf2243a01 100644 --- a/knowledge/foundations/parallel-execution.md +++ b/knowledge/foundations/parallel-execution.md @@ -46,6 +46,37 @@ Implemented becomes 128+signal. - Limit: `max_background_jobs` (TM-DOS-122). +## Coprocesses + +`coproc [NAME] cmd` is a background job whose stdin and stdout are two +bounded in-process pipes (`interpreter/coproc.rs`, the same `pipe::Pipe` as +pipeline stages). + +- `${NAME[1]}` is an `exec_fd_table` entry (`FdTarget::Coproc`), `${NAME[0]}` + a pipe read end in the readable-fd table. Both are `Arc`-shared ends, so + dups and subshell snapshots share them; the last copy closing is EOF for + the coproc (or SIGPIPE for its writes). The coproc's own fork drops every + coproc write end it would inherit (bash leaks other coprocs' fds there). +- The body is polled with the foreground, so it runs whenever the shell + blocks: reading `${NAME[0]}` (a `read` pulls one line, other commands read + to end of input), writing past the pipe's 4 KiB (the writer waits), `wait`. + Request/response loops work line by line. A lone `cat`/`tr`/`grep` body + streams like a pipeline stage; other bodies see each command's output at + its end. +- fds: bash's numbering, 63/60 for the first coproc, 62/58 for a second + while the first is open. A second coproc while one runs prints bash's + `warning: execute_coproc: coproc [PID:NAME] still exists` and starts. +- Cleanup is deterministic: after `wait` reaps the job, the next command + boundary unsets NAME/NAME_PID and closes the fds. bash does this on + SIGCHLD, so whether a finished coproc's output can still be read there + depends on timing; here it can, until `wait`. +- End of `exec()`: the shell's ends close and NAME is unset before + `finish_all`, so `coproc cat` without a close does not hang the call. +- `concurrent_jobs(false)`: the body runs to completion at `coproc` with + stdin at end of input. +- Limits: a job slot (TM-DOS-122), two persistent fds (TM-DOS-063). Gaps: + L-PROC-005. + ## Pipelines - Every stage of a multi-command pipeline runs in a subshell: variables, diff --git a/knowledge/log.md b/knowledge/log.md index 592515b0d..36e1ebbab 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -2,6 +2,7 @@ ## 2026-10-08 +* **Fix**: coprocesses run concurrently. `coproc NAME cmd` is a background job wired to the shell through two bounded in-process pipes: writes to `${NAME[1]}` feed its stdin, `exec {NAME[1]}>&-` delivers EOF, `read <&${NAME[0]}` / `read -u` take its output line by line, so `coproc UPPER { tr a-z A-Z; }` and request/response loops (`echo q >&${C[1]}; read r <&${C[0]}`) work; the body used to run once at `coproc` with empty input. fds follow bash (63/60, then 62/58), `$NAME_PID`/`$!`/`wait $NAME_PID` give the job, a second coproc while one runs prints bash's `still exists` warning, NAME and NAME_PID are unset and the fds closed once `wait` reaps it, and `exec()` closes the shell's ends before waiting for jobs. `<&$empty` is now an error instead of reading the shell's stdin. See foundations/parallel-execution "Coprocesses", L-PROC-005. bash-oracle match 102 to 103, strict 100 to 101. * **Fix**: `$SHLVL` follows bash's fork suppression. bash execs some simple commands in place: the last one of a `bash -c` string or `$( )` body (also the right side of a trailing `&&`/`||`/`;`; no redirections, no EXIT/ERR or signal trap set), the last one of a `( )`/`<( )` body (a sole one even with redirections), and any simple command run with `&`; a shell started there lowers `$SHLVL` first and so keeps the level. `exec bash` lowers it too, except directly inside `( )`; a substitution inside a pipeline stage does not. `bash -c 'bash -c "echo \$SHLVL"'` printed 3 in a script started at 1, now 2. A `bash`/`sh` child no longer inherits trap handlers (an ignored signal stays ignored), so `bash -c 'trap "echo bye" EXIT; bash -c true'` prints `bye` once. L-PROC-003 narrowed. bash-oracle match 101 to 102, strict 99 to 100. ## 2026-10-07 diff --git a/knowledge/operations/limitations.md b/knowledge/operations/limitations.md index 1790a7217..ed7ee538f 100644 --- a/knowledge/operations/limitations.md +++ b/knowledge/operations/limitations.md @@ -45,6 +45,7 @@ execution model. Evidence is a threat-model ID, a test, or `stance` | L-PROC-001 | `exec` does not replace the process; `exec cmd` runs cmd then stops execution. fd redirects work; `exec >log 2>&1` routes the shell's output per top-level command, so output written earlier in the same top-level command (not streamed yet) also lands in the log, and output under the redirect reaches a streaming caller only when that command ends. `exec N>&-` closes fd 1 or 2 for the rest of the script: later writes to it are dropped, the command reports `write error: Bad file descriptor` naming itself, and its status is 1. An `exec` redirect inside a subshell applies to the subshell's own output, routed at the subshell boundary; output the subshell wrote before the `exec` still reaches the caller. Inside a `bash -c` child it still does not take effect. `exec &>file` and `exec >&word` are ignored | True process replace would break sandbox containment; there is no real fd 1 to swap, the shell's output is a value returned to the caller | TM-ESC-005, `exec-redirect-shell-output.test.sh` | | L-PROC-002 | Background jobs run concurrently (`jobs`, `wait -n`, `kill`, `ps`, `pgrep`), but cannot be stopped or resumed: `kill -STOP/-CONT` are ignored, there is no `suspend`/Ctrl-Z, `bg` is a no-op and `fg` just waits; jobs end when `exec()` returns | No terminal process groups in a virtual shell; a job must not outlive the call that owns it (TM-DOS-122) | `l_proc_002_no_job_control` | | L-PROC-003 | No process spawning; external commands run as builtins. bash's fork suppression is modeled only for `$SHLVL` (see `NoforkScope` in the interpreter): a `bash`/`sh` child bash would exec in place keeps the level, but another command run there still sees the unlowered value (bash's `bash -c 'printenv SHLVL'` prints one less than the child's level), nor do `env`/`timeout`/`xargs` starting a shell, and a `SHLVL=n` prefix on the in-place command counts | Core sandbox model: no fork/exec escape surface | `l_proc_003_no_process_spawning` | +| L-PROC-005 | Coprocesses run concurrently with bash's fd numbers and cleanup, with these gaps: a coproc fd as a compound command's stdin (`while read l; do ...; done <&${C[0]}`) is read to end of input up front; `read -n`/`-N` on a coproc fd take a whole line (the rest of it is lost); writing to a coproc that exited is silently dropped (bash takes SIGPIPE); a finished coproc stays (NAME set, output readable) until `wait` reaps it, where bash reaps on SIGCHLD; coprocs end with the `exec()` call; an empty `<&"${C[0]}"` reports `ambiguous redirect` without the quotes where bash says `Bad file descriptor` | Deterministic, sandboxed job model (no SIGCHLD, no process outliving the call) | `coproc_tests.rs` | | L-PROC-004 | `bash`/`sh` child shells nest at most 8 deep (and count against `max_function_depth`); deeper nesting fails `maximum nesting depth exceeded` | Child shells run in-process on one stack; unbounded nesting crashed the process (TM-DOS-125) | `l_proc_004_child_shell_depth` | | L-PIPE-001 | Pipeline stages stream only from the first stage that runs shell code (loop, group, function, `eval`). Leading single-builtin stages run to completion first and hand over their whole output, so they never get SIGPIPE (`sort big \| head -1` gives `PIPESTATUS` `0 0`, bash `141 0`); `yes`, `seq`, plain `cat`, `grep` and `tr` are the exception and stream. Downstream of a streaming stage, a single builtin reads all its input before it starts, except `head`, `read`, plain `cat`, `grep` (not `-c/-l/-L/-q`, context, `-b`) and `tr` (not `-s`): `while :; do echo; done \| sed p \| head -1` runs until a limit. Pipes hold 4 KiB, not 64 KiB, so a finite loop writing more than that into an early-exiting reader exits 141 where bash would exit 0. Pipelines nested more than 4 subshells deep run their stages in sequence | Builtins return their output as one value, not a stream; streaming them needs a reader/writer `Context`. The smaller pipe bounds how far a producer runs ahead (each byte costs budgeted commands); the nesting cap bounds native stack (TM-DOS-124) | `l_pipe_001_stages_run_sequentially` | | L-SUDO-001 | `sudo` runs its command as the one sandbox user: `-u`, `-g`, `-i` change nothing, `whoami` still prints `sandbox`, and there is never a password prompt. `busybox APPLET` runs bashkit's builtin of that name, not BusyBox's own applet (flags and messages are bashkit's) | The sandbox has no privilege boundary to cross; the VFS has no permission enforcement (L-FS-002) | `sudo-busybox.test.sh` | diff --git a/knowledge/security/threat-model.md b/knowledge/security/threat-model.md index 5caf38fc6..ae13d6503 100644 --- a/knowledge/security/threat-model.md +++ b/knowledge/security/threat-model.md @@ -277,7 +277,7 @@ runaway scripts without permanently breaking the session. | TM-DOS-060 | Sparse array huge-index allocation | `arr[999999999]=x` could allocate ~1B empty slots if arrays are Vec-backed; negative indices could cause OOB | HashMap-based arrays; `max_array_entries` caps total entries | **MITIGATED** | | TM-DOS-061 | Snapshot function restore bypasses parser/function limits | Crafted snapshot restores functions exceeding the tenant's parser depth or function memory budget | Restored function source re-parsed with current `ExecutionLimits`; function memory budget re-applied before insertion | **MITIGATED** | | TM-DOS-062 | jq file binding amplification | Repeated `--rawfile` / `--slurpfile` bindings to one max-sized VFS file multiply retained jq globals and `$ARGS.named` values without consuming more VFS quota | `MAX_FILE_VAR_REQUESTS` caps binding count; `MAX_FILE_VAR_BYTES` counts cumulative file bytes per binding before retaining globals | **MITIGATED** | -| TM-DOS-063 | Persistent fd exhaustion | `exec N>/tmp/f` across many `N` values, or `cmd {v}>/tmp/f` in a loop (bash keeps `{var}` fds open), grows `exec_fd_table`/coproc fd buffers for the session | `ExecutionLimits::max_file_descriptors` (default 1024) caps persistent custom descriptors; reused fds and standard 0/1/2 don't count | **MITIGATED** | +| TM-DOS-063 | Persistent fd exhaustion | `exec N>/tmp/f` across many `N` values, or `cmd {v}>/tmp/f` in a loop (bash keeps `{var}` fds open), grows `exec_fd_table`/coproc fd buffers for the session | `ExecutionLimits::max_file_descriptors` (default 1024) caps persistent custom descriptors; reused fds and standard 0/1/2 don't count. A `coproc` needs two free slots and also takes a background-job slot (TM-DOS-122); its pipes hold at most 4 KiB plus one command's output before the writer waits | **MITIGATED** | | TM-DOS-109 | Silent truncation at builtin caps | A builtin that stops at a resource cap but exits 0 with no diagnostic (awk loops/recursion/getline caps, `seq`, `yes`, `numfmt`, `history`), or an awk `sprintf` expression that reports a format cap yet returns an empty value, gives scripts and agents incomplete output as success (#2446, #2456, like sed #1005) | Reaching a builtin cap is never silent: `builtins::limits::cap_exceeded` keeps the partial output, writes `: limit () exceeded; output truncated` and exits 1; awk caps and formatting errors are gawk-style fatal errors (`awk: fatal: ...`, exit 2, END skipped). Regression tests: `builtin_cap_security_tests` | **MITIGATED** | | TM-DOS-110 | In-builtin memory growth aborts the host | A value that keeps growing inside awk (`s = s s`, `gsub` with a long replacement, `sprintf("%s%s%s", s, s, s)`, `$1000000000 = 1`, filling arrays, deep recursion shadowing large values) or jq (`until(false; . + .)`, `"x" * 1e12`, `[range(1e12)]`, a non-emitting `until(false; .)` growing evaluator state) allocates until the allocator fails and the embedding process aborts with SIGABRT (#2444) | awk: every string is checked against `AWK_MAX_STRING_BYTES` (16 MiB) *before* it is allocated (concat, `SUBSEP` keys, `sub`/`gsub`/`gensub` as the result grows, `sprintf`/`printf`, `print` joins, rebuilt `$0`); `$N` assignment is capped at `AWK_MAX_FIELD_INDEX`; variables, array elements and values parked in call frames are accounted and capped at `ExecutionLimits::max_live_intermediate_bytes` (default 32 MB); each is a fatal awk error (exit 2) and the script carries on. jq: jaq-json is vendored with a live-memory meter charging every string and array/object body to the same limit, checked before growth (`jq: error: value size limit (N bytes) exceeded`, exit 5), sticky across `try`; value operations poll the execution deadline so non-emitting loops stop at the timeout; output conversion is capped by `max_stdout_bytes`; `--stream` events are produced incrementally with shared ancestor keys, metered cursor/path/event bodies and pre-allocation checks, including `--slurp`, and every traversal step checks shared work/cancellation/deadline limits. Regression tests: stream memory/work coverage in `execution_budget_tests` and `memory_growth_security_tests`. Jq evaluator contexts are now counted (64 live-context ceiling) and every context clone polls the deadline, so pure tail recursion times out and non-tail recursion fails before host stack exhaustion (#2465). The unwind-based abort requires `panic = "unwind"`; panic-abort embedders need process isolation for untrusted jq | **MITIGATED** | | TM-DOS-114 | Arrays bypass the retained-memory byte budget | `max_array_entries` counted entries but never their keys or values, so `x` doubled to 4 MiB and parked in ten array slots retained ~40 MB past the 10 MB variable-byte budget and exited 0 (#2462) | Array keys and values charge `max_total_variable_bytes` alongside scalars (`MemoryBudget::array_bytes`; indexed keys are machine integers already covered by the entry count, associative keys are attacker-controlled so they count). Element writes, `+=` appends, bulk `a=(...)` replacement, and `declare -A` compound writes are checked before insertion; bytes are released on element unset, whole-array unset, replacement, and local scope pop. An over-budget write records the first memory-limit error and fails execution (TM-DOS-111 mechanism) rather than dropping the value; reaching `max_array_entries` keeps its established silent-skip behaviour (TM-DOS-060). Regression tests: `array_budget_security_tests` | **MITIGATED** | diff --git a/scripts/bash-oracle/floor.txt b/scripts/bash-oracle/floor.txt index 2102eb532..021b3f448 100644 --- a/scripts/bash-oracle/floor.txt +++ b/scripts/bash-oracle/floor.txt @@ -1 +1 @@ -102 100 +103 101