From 7f1eb18247dac15e74ca2be47dbda0331fdd7380 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Wed, 7 Oct 2026 19:45:17 +0000 Subject: [PATCH] fix(jq): follow jq's input stream, per-file names and per-input errors -n leaves the stream to input/inputs, -s on empty input gives [], files are read one by one (input_filename, jq's line counting), missing files are skipped with exit 2, runtime errors are reported with their location and processing continues, input fails with 'No more inputs', -e uses the last output, values before a syntax error are processed. Claude-Session: https://claude.ai/code/session_01VscpEzxtqpCKx5eRMpywNj --- crates/bashkit/docs/jq.md | 19 +- crates/bashkit/src/builtins/jq/convert.rs | 35 +-- crates/bashkit/src/builtins/jq/errors.rs | 27 +- crates/bashkit/src/builtins/jq/input.rs | 109 ++++++++ crates/bashkit/src/builtins/jq/mod.rs | 307 +++++++++++++--------- crates/bashkit/src/builtins/jq/tests.rs | 175 +++++++++++- knowledge/log.md | 1 + 7 files changed, 514 insertions(+), 159 deletions(-) diff --git a/crates/bashkit/docs/jq.md b/crates/bashkit/docs/jq.md index 154fb9cf4..116fc778b 100644 --- a/crates/bashkit/docs/jq.md +++ b/crates/bashkit/docs/jq.md @@ -55,15 +55,22 @@ shim adds or overrides: | `leaf_paths` | Defined as `paths(scalars)` since jaq's stdlib lacks it. | | `match(re; flags)` / `match(re)` | Overridden to add `"name": null` to unnamed captures, matching jq output. | | `scan(re; flags)` / `scan(re)` | Overridden so `scan` defaults to global ("g") matching, matching jq. | -| `input_filename` | Stub returning `null` (#1486). Bashkit reads inputs as a single concatenated stream via shell redirection, so per-input filenames are not tracked. | -| `input_line_number` | Stub returning `0` (#1486). Per-line input tracking is not implemented. | -| `input` / `inputs` | Real jaq implementations, pull from the shared input iterator. | +| `input_filename` | Name of the file the current value came from (`null` for stdin). | +| `input_line_number` | Lines read so far, as jq counts them (jq reads a line at a time). | +| `input` / `inputs` | Pull from the same stream as the main loop; with `-n` the whole stream is theirs. `input` fails with `No more inputs` at the end. | | Most other 1.7/1.8 stdlib filters | Forwarded from `jaq-std` (`getpath`, `paths`, `to_entries`, `group_by`, `ltrimstr`/`rtrimstr`, `splits`, `test`, `now`, `debug`, `limit`, etc.). | ## Errors -Filter parse failures and runtime errors return exit code `3` and `5` -respectively, matching jq. Long error operands are summarised so failures +Filter compile failures exit `3`. A runtime error is reported as +`jq: error (at FILE:LINE): ...` (``, or `` under `-n`) and jq +moves on to the next input; the exit status follows the last input (`5` if +it failed). `error("msg")` prints `msg` unquoted, other values get jq's +`(not a string)` marker. Values before a JSON syntax error are processed, +then the error exits `5`. A file that cannot be opened prints +`Could not open file F: reason`, is skipped, and makes the exit status `2`. +With `-e` the last output decides: `1` for `null`/`false`, `4` when nothing +was output. Long error operands are summarised so failures do not blow up an LLM context window, see [#1485](https://github.com/everruns/bashkit/issues/1485). @@ -86,8 +93,6 @@ need a larger `max_live_intermediate_bytes`. Bashkit's jq is intentionally minimal in places where the host model differs from upstream jq: -- File inputs are concatenated into a single stream by the shell, so - per-file metadata (`input_filename`, `input_line_number`) is stubbed. - Exotic numeric formatting modes (`@base32`, `@base64d`, etc.) follow whatever `jaq-json` ships. diff --git a/crates/bashkit/src/builtins/jq/convert.rs b/crates/bashkit/src/builtins/jq/convert.rs index d91752441..fb79523d3 100644 --- a/crates/bashkit/src/builtins/jq/convert.rs +++ b/crates/bashkit/src/builtins/jq/convert.rs @@ -287,36 +287,39 @@ fn format_f64_canonical(f: f64) -> String { /// Parse multiple JSON values from a stream (handles NDJSON, multi-line, /// concatenated). Each value is depth-checked and keeps its key order. pub(super) fn parse_json_stream(input: &str) -> std::result::Result, String> { - use serde::de::DeserializeSeed; - - let trimmed = input.trim(); - if trimmed.is_empty() { - return Ok(Vec::new()); + match parse_json_stream_partial(input) { + (vals, None) => Ok(vals), + (_, Some(e)) => Err(e), } +} + +/// Parse a whitespace-separated JSON stream, returning the values before +/// the first syntax error together with that error. +pub(super) fn parse_json_stream_partial(input: &str) -> (Vec, Option) { + use serde::de::DeserializeSeed; - let too_deep = std::cell::Cell::new(None); - let mut de = serde_json::Deserializer::from_str(trimmed); let mut vals = Vec::new(); - loop { - // Skip inter-value whitespace; stop at end of input. - if de.end().is_ok() { - break; - } + // Split into values first so a literal glued to junk (`1\u{1}2`) is one + // invalid token, as in jq, instead of a valid `1` followed by an error. + for (start, end) in super::input::value_spans(input) { + let too_deep = std::cell::Cell::new(None); + let mut de = serde_json::Deserializer::from_str(&input[start..end]); let seed = JqSeed { depth: 0, max: MAX_JQ_JSON_DEPTH, too_deep: &too_deep, }; - match seed.deserialize(&mut de) { + match seed.deserialize(&mut de).and_then(|v| de.end().map(|()| v)) { Ok(v) => vals.push(v), Err(e) => { - return Err(too_deep + let msg = too_deep .take() - .unwrap_or_else(|| format!("jq: invalid JSON: {e}"))); + .unwrap_or_else(|| format!("jq: invalid JSON: {e}")); + return (vals, Some(msg)); } } } - Ok(vals) + (vals, None) } #[cfg(test)] diff --git a/crates/bashkit/src/builtins/jq/errors.rs b/crates/bashkit/src/builtins/jq/errors.rs index f41e7aa6a..99ac06297 100644 --- a/crates/bashkit/src/builtins/jq/errors.rs +++ b/crates/bashkit/src/builtins/jq/errors.rs @@ -79,16 +79,33 @@ pub(super) fn format_load_errors

(errs: jaq_core::load::Errors<&str, P>) -> St format!("jq: error: {}\n", truncate_text(&body, MAX_JQ_DIAG_CHARS)) } -/// Format jaq runtime errors without dumping full input values into stderr. +/// Format a jaq runtime error the way jq reports an error while filtering +/// one input: `jq: error (at :): ...`. `error("msg")` prints the +/// message as is, `error({...})` adds jq's `(not a string)` marker. +/// /// stderr is agent-facing API: value-bearing diagnostics summarize operand /// types and keep generic fallbacks bounded. -pub(super) fn format_runtime_error(error: &jaq_core::Error) -> String { +pub(super) fn format_runtime_error_at(error: &jaq_core::Error, location: &str) -> String { // THREAT[TM-DOS-110]: an error quoting a huge or deeply shared value // would render it in full; stop rendering past what the message keeps. let message = render_capped(error, 64 * 1024); - let body = humanize(&message) - .unwrap_or_else(|| capitalize_first(&truncate_text(&message, MAX_JQ_RUNTIME_ERROR_CHARS))); - format!("jq: error: {body}\n") + let thrown = super::convert::val_to_jq_capped(&error.clone().into_val(), 64 * 1024); + match thrown { + Some(super::convert::JqJson::String(s)) if message.starts_with('"') => format!( + "jq: error (at {location}): {}\n", + truncate_text(&s, MAX_JQ_RUNTIME_ERROR_CHARS) + ), + Some(super::convert::JqJson::String(_)) | None => { + let body = humanize(&message).unwrap_or_else(|| { + capitalize_first(&truncate_text(&message, MAX_JQ_RUNTIME_ERROR_CHARS)) + }); + format!("jq: error (at {location}): {body}\n") + } + Some(_) => format!( + "jq: error (at {location}) (not a string): {}\n", + truncate_text(&message, MAX_JQ_RUNTIME_ERROR_CHARS) + ), + } } /// `Display` into a string, stopping after `max` bytes. diff --git a/crates/bashkit/src/builtins/jq/input.rs b/crates/bashkit/src/builtins/jq/input.rs index 7eec55570..9f67a5db4 100644 --- a/crates/bashkit/src/builtins/jq/input.rs +++ b/crates/bashkit/src/builtins/jq/input.rs @@ -108,6 +108,93 @@ pub(super) fn normalize<'a>( }) } +/// Byte spans of the top-level values in a JSON text stream, split the way +/// jq tokenizes: strings, bracketed values, and literal tokens that run to +/// the next whitespace or structural character. The parser validates each +/// span; this only finds boundaries. +pub(super) fn value_spans(text: &str) -> Vec<(usize, usize)> { + let b = text.as_bytes(); + let mut out = Vec::new(); + let mut i = 0usize; + loop { + while i < b.len() && b[i].is_ascii_whitespace() { + i += 1; + } + if i >= b.len() { + break; + } + let start = i; + let scalar = !matches!(b[i], b'[' | b'{' | b'"'); + let (mut depth, mut in_str, mut esc) = (0usize, false, false); + while i < b.len() { + let c = b[i]; + i += 1; + if in_str { + if esc { + esc = false; + } else if c == b'\\' { + esc = true; + } else if c == b'"' { + in_str = false; + if depth == 0 { + break; + } + } + continue; + } + if scalar { + if c.is_ascii_whitespace() + || matches!(c, b'[' | b'{' | b'"' | b']' | b'}' | b',' | b':') + { + i -= 1; + if i == start { + // A stray structural character: its own (invalid) span. + i += 1; + } + break; + } + continue; + } + match c { + b'"' => in_str = true, + b'[' | b'{' => depth += 1, + b']' | b'}' => { + depth = depth.saturating_sub(1); + if depth == 0 { + break; + } + } + _ => {} + } + } + out.push((start, i)); + } + out +} + +/// Line number jq reports for each top-level value of `text`: jq reads +/// input a line at a time, so a value counts every newline up to the end +/// of the line it ends on. +pub(super) fn value_lines(text: &str) -> Vec { + let b = text.as_bytes(); + let (mut counted, mut lines) = (0usize, 0usize); + value_spans(text) + .into_iter() + .map(|(_, end)| { + let last = end.saturating_sub(1); + let eol = b[last..] + .iter() + .position(|&c| c == b'\n') + .map_or(b.len(), |p| last + p + 1); + if eol > counted { + lines += b[counted..eol].iter().filter(|&&c| c == b'\n').count(); + counted = eol; + } + lines + }) + .collect() +} + #[cfg(test)] mod tests { use std::sync::Arc; @@ -173,4 +260,26 @@ mod tests { )) )); } + + #[test] + fn value_spans_split_like_jq() { + let text = "1 \"a b\" [1, [2]] {\"k\": \"}\"}2[3]"; + let parts: Vec<&str> = super::value_spans(text) + .into_iter() + .map(|(a, b)| &text[a..b]) + .collect(); + assert_eq!( + parts, + ["1", "\"a b\"", "[1, [2]]", "{\"k\": \"}\"}", "2", "[3]"] + ); + let glued = "1\u{1}2"; + assert_eq!(super::value_spans(glued), [(0, glued.len())]); + } + + #[test] + fn value_lines_count_lines_read() { + assert_eq!(super::value_lines("1\n2\n3"), [1, 2, 2]); + assert_eq!(super::value_lines("[\n1\n]\n2"), [3, 3]); + assert_eq!(super::value_lines("1 2"), [0, 0]); + } } diff --git a/crates/bashkit/src/builtins/jq/mod.rs b/crates/bashkit/src/builtins/jq/mod.rs index 1895b807a..2e8caab50 100644 --- a/crates/bashkit/src/builtins/jq/mod.rs +++ b/crates/bashkit/src/builtins/jq/mod.rs @@ -65,7 +65,7 @@ use compat::{ build_compat_prefix, }; use convert::{JqJson, jq_to_val, parse_json_stream, val_to_jq_capped}; -use errors::{format_compile_errors, format_load_errors, format_runtime_error}; +use errors::{format_compile_errors, format_load_errors, format_runtime_error_at}; use format::{Indent, render, sort_keys as sort_jq_keys}; /// Custom DataT that holds both the LUT and a shared input iterator. @@ -140,7 +140,7 @@ pub struct Jq; struct FilterInput { value: Val, filename: Val, - lineno: Val, + lineno: usize, } #[async_trait] @@ -200,32 +200,32 @@ async fn run_jq(ctx: Context<'_>, parsed: JqArgs<'_>) -> Result { // Build $ARGS object. positional: [...], named: {name: val, ...}. let args_obj = build_args_obj(&parsed.positional_args, &all_named_args); - // Read input (stdin or files). - let file_content: String; - let input = if !parsed.file_args.is_empty() { - let mut combined = String::new(); + // Read input sources: stdin, or each FILE in order. A file that cannot + // be opened is reported like jq (`Could not open file`), skipped, and + // turns the final exit status into 2. + let mut stderr_out = String::new(); + let mut input_failed = false; + let mut sources: Vec<(Option<&str>, String)> = Vec::new(); + if parsed.file_args.is_empty() { + sources.push((None, ctx.stdin.map(|s| s.to_string()).unwrap_or_default())); + } else { for file_arg in &parsed.file_args { let path = resolve_path(ctx.cwd, file_arg); - let text = match read_text_file(&*ctx.fs, &path, "jq").await { - Ok(t) => t, - Err(e) => return Ok(e), - }; - ctx.consume_budget_input(text.len())?; - if !combined.is_empty() && !combined.ends_with('\n') { - combined.push('\n'); + match ctx.fs.read_file(&path).await { + Ok(bytes) => { + let text = crate::StreamData::from(bytes).to_string(); + ctx.consume_budget_input(text.len())?; + sources.push((Some(*file_arg), text)); + } + Err(e) => { + input_failed = true; + stderr_out.push_str(&format!( + "jq: error: Could not open file {file_arg}: {}\n", + crate::error::io_error_reason(&e) + )); + } } - combined.push_str(&text); } - file_content = combined; - file_content.as_str() - } else { - ctx.stdin.map(|stdin| &**stdin).unwrap_or("") - }; - - // Empty stdin without -n yields empty output (matches real jq for files - // and stdin alike), but -Rs explicitly produces "" — keep that path. - if input.trim().is_empty() && !parsed.null_input && !(parsed.raw_input && parsed.slurp) { - return Ok(ExecResult::ok(String::new())); } // Build shell env object for the custom `env` filter / $ENV. @@ -293,9 +293,24 @@ async fn run_jq(ctx: Context<'_>, parsed: JqArgs<'_>) -> Result { var_names.push(ARGS_VAR_NAME); type D = InputData; + // jq's `input` fails with "No more inputs" when the stream is + // exhausted; jaq-std's yields nothing. + let input_run: jaq_core::RunPtr = |cv| { + let next = cv.0.data().inputs().next(); + Box::new(std::iter::once(match next { + Some(r) => r.map_err(|e| jaq_core::Exn::from(jaq_core::Error::str(e))), + None => Err(jaq_core::Exn::from(jaq_core::Error::str("No more inputs"))), + })) + }; let input_funs: Vec> = jaq_std::input::funs::() .into_vec() .into_iter() + .filter(|(name, _, _)| *name != "input") + .chain(std::iter::once(( + "input", + jaq_core::native::v(0), + input_run, + ))) .map(|(name, arity, run)| (name, arity, jaq_core::Native::::new(run))) .collect(); // Replace jaq-std's `regex`-crate-backed natives (matches/split_matches/ @@ -350,56 +365,66 @@ async fn run_jq(ctx: Context<'_>, parsed: JqArgs<'_>) -> Result { let args_val = jq_to_val(&args_obj); let pre_var_vals: Vec = all_var_bindings.iter().map(|(_, v)| jq_to_val(v)).collect(); - // Build inputs to process. - let inputs_to_process: Vec = if parsed.null_input { - vec![FilterInput { - value: Val::Null, - filename: Val::Null, - lineno: Val::from(0isize), - }] - } else if parsed.raw_input && parsed.slurp { - vec![FilterInput { - value: Val::from(input.to_string()), - filename: stdin_filename(&parsed), - lineno: Val::from(0isize), - }] + // Build the input stream. Each value keeps its file name and the line + // number jq reports for it in errors. + let mut parse_error: Option = None; + let mut items: Vec = Vec::new(); + let name_val = |name: &Option<&str>| match name { + Some(n) => Val::from((*n).to_string()), + None => Val::Null, + }; + if parsed.raw_input && parsed.slurp { + let text: String = sources.iter().map(|(_, t)| t.as_str()).collect(); + items.push(FilterInput { + value: Val::from(text), + filename: sources + .last() + .map(|(n, _)| name_val(n)) + .unwrap_or(Val::Null), + lineno: 0, + }); } else if parsed.raw_input { - let fname = stdin_filename(&parsed); - input - .lines() - .enumerate() - .map(|(i, line)| FilterInput { - value: Val::from(line.to_string()), - filename: fname.clone(), - lineno: Val::from(isize::try_from(i + 1).unwrap_or(isize::MAX)), - }) - .collect() - } else if parsed.slurp { - match parse_jq_json_stream(&ctx, input)? { - Ok(vals) => { - let arr: JqJson = JqJson::Array(vals); - vec![FilterInput { - value: jq_to_val(&arr), - filename: stdin_filename(&parsed), - lineno: Val::from(0isize), - }] + for (name, text) in &sources { + for (i, line) in text.lines().enumerate() { + items.push(FilterInput { + value: Val::from(line.to_string()), + filename: name_val(name), + lineno: i + 1, + }); } - Err(e) => return Ok(ExecResult::err(format!("{e}\n"), 5)), } } else { - match parse_jq_json_stream(&ctx, input)? { - Ok(jq_vals) => jq_vals - .iter() - .enumerate() - .map(|(i, v)| FilterInput { - value: jq_to_val(v), - filename: stdin_filename(&parsed), - lineno: Val::from(isize::try_from(i + 1).unwrap_or(isize::MAX)), - }) - .collect(), - Err(e) => return Ok(ExecResult::err(format!("{e}\n"), 5)), + let mut slurped: Vec = Vec::new(); + for (name, text) in &sources { + let (vals, err) = parse_jq_json_stream_partial(&ctx, text)?; + if parsed.slurp { + slurped.extend(vals); + } else { + let lines = input::value_lines(text); + for (i, v) in vals.iter().enumerate() { + items.push(FilterInput { + value: jq_to_val(v), + filename: name_val(name), + lineno: lines.get(i).copied().unwrap_or(0), + }); + } + } + if let Some(e) = err { + parse_error = Some(e); + break; + } } - }; + if parsed.slurp && parse_error.is_none() { + items.push(FilterInput { + value: jq_to_val(&JqJson::Array(slurped)), + filename: sources + .last() + .map(|(n, _)| name_val(n)) + .unwrap_or(Val::Null), + lineno: 0, + }); + } + } let indent = if parsed.compact_output { Indent::Compact @@ -408,8 +433,6 @@ async fn run_jq(ctx: Context<'_>, parsed: JqArgs<'_>) -> Result { }; let mut output = String::new(); - let mut has_output = false; - let mut all_null_or_false = true; // THREAT[TM-DOS-093]: jaq evaluation is a synchronous iterator that the // async execution timeout cannot preempt. Unbounded generators @@ -457,41 +480,63 @@ async fn run_jq(ctx: Context<'_>, parsed: JqArgs<'_>) -> Result { )) }; - // Drive the outer loop from the shared input iterator so `input`/`inputs` - // inside the filter consume from the same source (matching real jq: - // `[inputs]` on a 3-value stream returns the *remaining* values, not all). - // Metadata (filename, lineno) is tracked via a parallel index. - let metadata: Vec<(Val, Val)> = inputs_to_process - .iter() - .map(|fi| (fi.filename.clone(), fi.lineno.clone())) - .collect(); - let value_iter: Box>> = Box::new( - inputs_to_process - .into_iter() - .map(|fi| Ok::(fi.value)), - ); + // `input`/`inputs` inside the filter consume from the same stream as + // the outer loop (`[inputs]` on a 3-value stream returns the remaining + // values). With `-n` the filter runs once on null and the whole stream + // is left to `input`/`inputs`. Pulling a value records its file and + // line for `input_filename` and error locations. + let current: std::rc::Rc> = + std::rc::Rc::new(std::cell::RefCell::new((Val::Null, 0))); + let tracker = current.clone(); + let value_iter: Box>> = + Box::new(items.into_iter().map(move |fi| { + *tracker.borrow_mut() = (fi.filename, fi.lineno); + Ok::(fi.value) + })); let shared_inputs = RcIter::new(value_iter); + let null_input = parsed.null_input; + let location = |current: &(Val, usize)| -> String { + if null_input { + return "".to_string(); + } + match ¤t.0 { + Val::Null => format!(":{}", current.1), + name => format!("{}:{}", name.to_string().trim_matches('"'), current.1), + } + }; + + // Exit status follows the last input processed, like jq: 5 after a + // runtime error, else 0; with `-e`, 1 when the last output was null or + // false and 4 when nothing was ever output. + let mut status: Option = None; // THREAT[TM-DOS-110]: the meter aborts a run by unwinding where jaq // gives it no error channel (see jaq_json::meter::Abort). let run_filter = std::panic::AssertUnwindSafe(|| -> Result> { - for (outer_idx, jaq_input) in (&shared_inputs).enumerate() { - let jaq_input: Val = match jaq_input { - Ok(v) => v, - Err(e) => { - return Ok(Some(ExecResult::err(format!("jq: input error: {e}\n"), 5))); + let mut first = true; + loop { + let jaq_input: Val = if null_input { + if !first { + break; + } + Val::Null + } else { + match (&shared_inputs).next() { + Some(Ok(v)) => v, + Some(Err(e)) => { + return Ok(Some(ExecResult::err(format!("jq: input error: {e}\n"), 5))); + } + None => break, } }; - let (filename_val, lineno_val) = metadata - .get(outer_idx) - .cloned() - .unwrap_or((Val::Null, Val::from(0isize))); + first = false; + let (filename_val, lineno) = current.borrow().clone(); let mut var_vals: Vec = pre_var_vals.clone(); var_vals.push(env_val.clone()); // $__bashkit_env__ var_vals.push(env_val.clone()); // $ENV var_vals.push(filename_val); // $__bashkit_filename__ - var_vals.push(lineno_val); // $__bashkit_lineno__ + var_vals.push(Val::from(isize::try_from(lineno).unwrap_or(isize::MAX))); // $__bashkit_lineno__ var_vals.push(args_val.clone()); // $ARGS let data = InputDataRef { @@ -501,6 +546,7 @@ async fn run_jq(ctx: Context<'_>, parsed: JqArgs<'_>) -> Result { }; let cv_ctx = Ctx::>::new(data, Vars::new(var_vals)); + let mut input_status: Option = if parsed.exit_status { None } else { Some(0) }; for result in filter.id.run((cv_ctx, jaq_input)) { ctx.consume_budget_work(1)?; if meter.tripped() { @@ -509,7 +555,6 @@ async fn run_jq(ctx: Context<'_>, parsed: JqArgs<'_>) -> Result { } match jaq_core::unwrap_valr(result) { Ok(val) => { - has_output = true; let Some(mut jq) = val_to_jq_capped(&val, max_output_bytes) else { return Ok(Some(ExecResult::err( format!("jq: output limit exceeded ({max_output_bytes} bytes)\n"), @@ -519,8 +564,8 @@ async fn run_jq(ctx: Context<'_>, parsed: JqArgs<'_>) -> Result { if parsed.sort_keys { jq = sort_jq_keys(jq); } - if !(jq.is_null() || jq.is_false()) { - all_null_or_false = false; + if parsed.exit_status { + input_status = Some(i32::from(jq.is_null() || jq.is_false())); } let effective_raw = parsed.raw_output || parsed.join_output; @@ -560,10 +605,17 @@ async fn run_jq(ctx: Context<'_>, parsed: JqArgs<'_>) -> Result { } } Err(e) => { - return Ok(Some(ExecResult::err(format_runtime_error(&e), 5))); + // jq reports the error and moves on to the next input. + let loc = location(¤t.borrow()); + stderr_out.push_str(&format_runtime_error_at(&e, &loc)); + input_status = Some(5); + break; } } } + if input_status.is_some() { + status = input_status; + } } Ok(None) }); @@ -588,20 +640,43 @@ async fn run_jq(ctx: Context<'_>, parsed: JqArgs<'_>) -> Result { }, } - // Real jq exit codes for -e: - // - 4 if there was no output at all - // - 1 if all outputs were null or false - // - 0 otherwise - if parsed.exit_status { - if !has_output { - return Ok(ExecResult::with_code(output, 4)); - } - if all_null_or_false { - return Ok(ExecResult::with_code(output, 1)); - } + let mut code = status.unwrap_or(if parsed.exit_status { 4 } else { 0 }); + if let Some(e) = parse_error { + stderr_out.push_str(&e); + stderr_out.push('\n'); + code = 5; + } + if input_failed { + code = 2; } + Ok(ExecResult { + stdout: output.into(), + stderr: stderr_out.into(), + exit_code: code, + ..Default::default() + }) +} - Ok(ExecResult::ok(output)) +/// Like [`parse_jq_json_stream`], but keeps the values before a syntax +/// error: jq processes them, then reports the error. +fn parse_jq_json_stream_partial( + ctx: &Context<'_>, + input: &str, +) -> Result<(Vec, Option)> { + let execution_budget = ctx + .execution_budget() + .map(|budget| { + budget + .try_with(Clone::clone) + .map_err(|_| crate::Error::Cancelled) + }) + .transpose()?; + let normalized = match input::normalize(execution_budget.as_ref(), input) { + Ok(normalized) => normalized, + Err(input::NormalizeError::InvalidJson(message)) => return Ok((Vec::new(), Some(message))), + Err(input::NormalizeError::Resource(error)) => return Err(error), + }; + Ok(convert::parse_json_stream_partial(normalized.as_str())) } fn parse_jq_json_stream( @@ -658,15 +733,3 @@ fn build_args_obj(positional: &[JqJson], named: &[(String, JqJson)]) -> JqJson { ("named".to_string(), JqJson::Object(named_map)), ]) } - -/// `--slurpfile` / `--rawfile` reuse this to derive the filename Val. -fn stdin_filename(parsed: &JqArgs<'_>) -> Val { - // Real jq reports the per-file path while reading FILE..., else null. - // We thread a single filename per call when files are passed; if no - // files, return null. (Per-file granularity within one call would - // need re-architecting input dispatch.) - match parsed.file_args.first() { - Some(p) => Val::from((*p).to_string()), - None => Val::Null, - } -} diff --git a/crates/bashkit/src/builtins/jq/tests.rs b/crates/bashkit/src/builtins/jq/tests.rs index f228f89b8..5eba77f14 100644 --- a/crates/bashkit/src/builtins/jq/tests.rs +++ b/crates/bashkit/src/builtins/jq/tests.rs @@ -338,11 +338,15 @@ async fn exit_status_truthy_returns_0() { } #[tokio::test] -async fn exit_status_mixed_truthy_wins() { - // If any output is non-null/non-false, exit is 0 (unless no output). +async fn exit_status_last_value_wins() { + // jq -e: the last output decides (here false -> 1). let result = run_jq_result_with_args(&["-e", ".[]"], "[null, 1, false]") .await .unwrap(); + assert_eq!(result.exit_code, 1); + let result = run_jq_result_with_args(&["-e", ".[]"], "[null, false, 1]") + .await + .unwrap(); assert_eq!(result.exit_code, 0); } @@ -651,10 +655,14 @@ async fn input_filename_returns_path_when_file_given() { } #[tokio::test] -async fn input_line_number_increments_per_input() { - // For NDJSON stdin, each value advances the line counter by 1. +async fn input_line_number_counts_lines_read() { + // jq reads a line at a time: the last value has no newline after it, + // so it still reports line 2 (matches jq 1.7). let result = run_jq("input_line_number", "1\n2\n3").await.unwrap(); let lines: Vec<&str> = result.trim().split('\n').collect(); + assert_eq!(lines, vec!["1", "2", "2"]); + let result = run_jq("input_line_number", "1\n2\n\n3").await.unwrap(); + let lines: Vec<&str> = result.trim().split('\n').collect(); assert_eq!(lines, vec!["1", "2", "3"]); } @@ -721,8 +729,11 @@ async fn missing_file_errors() { let result = run_jq_with_files(&[".", "/missing.json"], &[]) .await .unwrap(); - assert_eq!(result.exit_code, 1); - assert!(result.stderr.contains("jq: /missing.json:")); + assert_eq!(result.exit_code, 2); + assert_eq!( + result.stderr, + "jq: error: Could not open file /missing.json: No such file or directory\n" + ); } // ========================================================================= @@ -915,7 +926,7 @@ async fn regex_invalid_pattern_yields_short_error() { .await .unwrap(); assert_ne!(result.exit_code, 0); - assert!(result.stderr.starts_with("jq: error: ")); + assert!(result.stderr.starts_with("jq: error (at :0): ")); assert!(result.stderr.len() < 300); // Must not leak fancy-regex internal Debug shapes assert!(!result.stderr.contains("ParseError")); @@ -1000,7 +1011,7 @@ async fn runtime_error_summarizes_index_operands() { assert_eq!(result.exit_code, 5); assert_eq!( result.stderr, - "jq: error: Cannot index array with string \"product_name\"\n" + "jq: error (at :0): Cannot index array with string \"product_name\"\n" ); } @@ -1010,7 +1021,7 @@ async fn runtime_error_iterate_over_null() { assert_eq!(result.exit_code, 5); assert_eq!( result.stderr, - "jq: error: Cannot iterate over null (null)\n" + "jq: error (at :0): Cannot iterate over null (null)\n" ); } @@ -1762,3 +1773,149 @@ async fn filter_from_file_treats_positionals_as_inputs() { assert_eq!(result.exit_code, 2); assert!(result.stderr.contains("Could not open /missing.jq")); } + +// ========================================================================= +// Input stream: -n with input/inputs, per-file names, errors per input +// (behavior checked against jq 1.7) +// ========================================================================= + +#[tokio::test] +async fn null_input_leaves_stream_to_input_and_inputs() { + assert_eq!( + run_jq_with_args(&["-n", "input"], "41 42").await.unwrap(), + "41\n" + ); + assert_eq!( + run_jq_with_args(&["-nc", "[inputs]"], "1 \"a\" [2]") + .await + .unwrap(), + "[1,\"a\",[2]]\n" + ); + assert_eq!( + run_jq_with_args(&["-nc", "[limit(2; inputs)], input"], "1 2 3 4") + .await + .unwrap(), + "[1,2]\n3\n" + ); + assert_eq!( + run_jq_with_args(&["-n", "reduce inputs as $x (0; . + $x)"], "1 2 39") + .await + .unwrap(), + "42\n" + ); +} + +#[tokio::test] +async fn slurp_of_empty_input_is_empty_array() { + assert_eq!(run_jq_with_args(&["-s", "."], "").await.unwrap(), "[]\n"); + assert_eq!(run_jq_with_args(&["."], "").await.unwrap(), ""); +} + +#[tokio::test] +async fn input_fails_when_stream_is_exhausted() { + let result = run_jq_result_with_args(&["-c", "[., input]"], "1 2 3") + .await + .unwrap(); + assert_eq!(result.stdout, "[1,2]\n"); + assert_eq!(result.stderr, "jq: error (at :0): No more inputs\n"); + assert_eq!(result.exit_code, 5); +} + +#[tokio::test] +async fn input_filename_follows_each_file() { + let result = run_jq_with_files( + &["-c", "[., input_filename]", "/a.json", "/b.json"], + &[("/a.json", "{\"f\":\"a\"}"), ("/b.json", "1 2")], + ) + .await + .unwrap(); + assert_eq!( + result.stdout, + "[{\"f\":\"a\"},\"/a.json\"]\n[1,\"/b.json\"]\n[2,\"/b.json\"]\n" + ); +} + +#[tokio::test] +async fn missing_file_among_others_is_skipped_with_exit_2() { + let result = run_jq_with_files( + &["-c", ".", "/a.json", "/nope.json"], + &[("/a.json", "{\"ok\":true}")], + ) + .await + .unwrap(); + assert_eq!(result.stdout, "{\"ok\":true}\n"); + assert_eq!( + result.stderr, + "jq: error: Could not open file /nope.json: No such file or directory\n" + ); + assert_eq!(result.exit_code, 2); +} + +#[tokio::test] +async fn runtime_error_moves_on_to_next_input() { + let result = run_jq_result_with_args( + &["-c", ".a | ascii_downcase"], + "{\"a\":\"X\"} 1 {\"a\":\"Y\"}", + ) + .await + .unwrap(); + assert_eq!(result.stdout, "\"x\"\n\"y\"\n"); + assert_eq!( + result.stderr, + "jq: error (at :0): Cannot index number with string \"a\"\n" + ); + // The last input succeeded. + assert_eq!(result.exit_code, 0); +} + +#[tokio::test] +async fn runtime_error_location_names_file_and_line() { + let result = run_jq_with_files( + &[".[] | .n | ascii_downcase", "/d.json"], + &[("/d.json", "[\n {\"n\": \"A\"},\n {\"n\": 5}\n]\n")], + ) + .await + .unwrap(); + assert_eq!(result.stdout, "\"a\"\n"); + assert!( + result.stderr.starts_with("jq: error (at /d.json:4): "), + "stderr={}", + result.stderr + ); + assert_eq!(result.exit_code, 5); +} + +#[tokio::test] +async fn error_with_string_prints_message_unquoted() { + let result = run_jq_result_with_args(&["-n", "error(\"boom\")"], "") + .await + .unwrap(); + assert_eq!(result.stderr, "jq: error (at ): boom\n"); + let result = run_jq_result_with_args(&["-n", "error({a:1})"], "") + .await + .unwrap(); + assert_eq!( + result.stderr, + "jq: error (at ) (not a string): {\"a\":1}\n" + ); +} + +#[tokio::test] +async fn values_before_a_parse_error_are_processed() { + let result = run_jq_result_with_args(&["-c", "."], "{\"a\":1}\n{a:2}") + .await + .unwrap(); + assert_eq!(result.stdout, "{\"a\":1}\n"); + assert!( + result.stderr.contains("invalid JSON"), + "stderr={}", + result.stderr + ); + assert_eq!(result.exit_code, 5); + // A literal glued to junk is one bad token: nothing is emitted. + let result = run_jq_result_with_args(&["-c", "."], "1\u{0001}2") + .await + .unwrap(); + assert_eq!(result.stdout, ""); + assert_eq!(result.exit_code, 5); +} diff --git a/knowledge/log.md b/knowledge/log.md index 6b08534d9..b5176bf8a 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -2,6 +2,7 @@ ## 2026-10-07 +* **Fix**: jq input stream follows jq 1.7: `-n` leaves the stream to `input`/`inputs`, `-s` on empty input gives `[]`, each FILE is read separately (`input_filename`, `input_line_number` as jq counts lines), a missing file is reported and skipped (exit 2), a runtime error prints `jq: error (at FILE:LINE): ...` and moves on to the next input, `input` fails with `No more inputs`, `-e` uses the last output, values before a JSON syntax error are processed. Debian-oracle jq score 67.5% to 74.2%. * **Fix**: awk `for (k in a)` without `PROCINFO["sorted_in"]` now visits keys in gawk's order (integer arrays ascending with other keys first; string keys in gawk's hash-bucket order), so unsorted loops print like Debian awk. All gawk cases in the Debian-oracle awk set now pass (L-AWK-001 narrowed to post-`delete` order). * **Feature**: awk rewritten to target gawk 5.2 (Debian's `awk`). New lexer, parser, async evaluator, ERE translation with POSIX leftmost-longest matching, gawk number formatting and `printf`, gawk's `rand()` sequence, `substr`/`getline`/`typeof` edge cases, `-f`/`-e`/`--` options and gawk's error messages. `system()`, `print | cmd` and `cmd | getline` now run through the sandbox shell (execution-plan driver). Debian-oracle score for awk went from 51% to 94% of 260 cases; the remaining gawk gap is unsorted `for (k in a)` order (L-AWK-001). All resource caps kept; new TM-DOS-128 for awk commands. See foundations/builtins "awk". * **Decision**: CPython's workload priority is agents running `python3` for scripted logic (processing a file, transforming data). Low-use stdlib modules are no longer shipped: `unittest`, `doctest`, profilers, `compileall`/`zipapp`, `dbm`/`shelve`, `plistlib`, `wave`, `netrc`, `cmd`, `tty`/`pty`, and `bz2`/`lzma`/`compression.zstd` (no C codec in the guest anyway). Stdlib zip 3.58 to 3.16 MB. L-CPY-009 updated. See [CPython WebAssembly Runtime](runtimes/cpython-wasm.md#workload-priority).