From 05d0000393a222e7644437e405eb9133e6558ed1 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Wed, 7 Oct 2026 20:11:54 +0000 Subject: [PATCH 1/2] fix(jq): assign through null and grow arrays like jq Claude-Session: https://claude.ai/code/session_01VscpEzxtqpCKx5eRMpywNj --- .../bashkit/src/builtins/jq/jaq_json/jqmsg.rs | 5 + .../bashkit/src/builtins/jq/jaq_json/mod.rs | 249 ++++++++++++------ crates/bashkit/src/builtins/jq/tests.rs | 37 +++ knowledge/log.md | 1 + knowledge/runtimes/jaq-json-vendor.md | 5 + 5 files changed, 210 insertions(+), 87 deletions(-) diff --git a/crates/bashkit/src/builtins/jq/jaq_json/jqmsg.rs b/crates/bashkit/src/builtins/jq/jaq_json/jqmsg.rs index 23cbeab65..247f18475 100644 --- a/crates/bashkit/src/builtins/jq/jaq_json/jqmsg.rs +++ b/crates/bashkit/src/builtins/jq/jaq_json/jqmsg.rs @@ -69,6 +69,11 @@ pub(crate) fn index(l: Val, r: Val) -> Error { Error::str(std::format!("Cannot index {} with {right}", type_name(&l))) } +/// `.[]` over a value that is not an array or object. +pub(crate) fn iter(v: Val) -> Error { + Error::str(std::format!("Cannot iterate over {}", described(&v))) +} + pub(crate) fn is_zero(v: &Val) -> bool { matches!(v, Val::Num(n) if n.as_f64() == 0.0) } diff --git a/crates/bashkit/src/builtins/jq/jaq_json/mod.rs b/crates/bashkit/src/builtins/jq/jaq_json/mod.rs index 58d951a05..f1330ef91 100644 --- a/crates/bashkit/src/builtins/jq/jaq_json/mod.rs +++ b/crates/bashkit/src/builtins/jq/jaq_json/mod.rs @@ -157,7 +157,7 @@ impl jaq_core::ValT for Val { match self { Self::Arr(a) => Box::new(rc_unwrap_or_clone(a).into_iter().enumerate().map(arr_idx)), Self::Obj(o) => Box::new(rc_unwrap_or_clone(o).into_iter().map(Ok)), - _ => box_once(Err(Error::typ(self, Type::Iter.as_str()))), + _ => box_once(Err(jqmsg::iter(self))), // BASHKIT PATCH } } @@ -166,7 +166,7 @@ impl jaq_core::ValT for Val { match self { Self::Arr(a) => Box::new(rc_unwrap_or_clone(a).into_iter().map(Ok)), Self::Obj(o) => Box::new(rc_unwrap_or_clone(o).into_iter().map(|(_k, v)| Ok(v))), - _ => box_once(Err(Error::typ(self, Type::Iter.as_str()))), + _ => box_once(Err(jqmsg::iter(self))), // BASHKIT PATCH } } @@ -207,110 +207,47 @@ impl jaq_core::ValT for Val { let iter = iter.filter_map(|(k, v)| f(v).next().map(|v| Ok((k, v?)))); Ok(Self::obj(iter.collect::>>()?)) } - v => opt.fail(v, |v| Exn::from(Error::typ(v, Type::Iter.as_str()))), + v => opt.fail(v, |v| Exn::from(jqmsg::iter(v))), // BASHKIT PATCH } } fn map_index<'a, I: Iterator>>( - mut self, + self, index: &Self, opt: path::Opt, f: impl Fn(Self) -> I, ) -> ValX<'a> { - meter::tick(); // BASHKIT PATCH (TM-DOS-110) - if let (Val::BStr(_) | Val::TStr(_) | Val::Arr(_), Val::Obj(o)) = (&self, index) { - let range = o.get(&Val::utf8_str("start"))..o.get(&Val::utf8_str("end")); - return self.map_range(range, opt, f); + // BASHKIT PATCH: like jq, updating through null creates the + // container the index implies (`null | .a = 1`, `.[2] = 1`), and + // stays null when the update yields nothing (`.a |= empty`). + let empty = match (&self, index) { + (Val::Null, Val::TStr(_) | Val::BStr(_)) => Val::obj(Map::default()), + (Val::Null, Val::Num(_) | Val::Obj(_)) => core::iter::empty::().collect(), + _ => return self.map_index_inner(index, opt, f), }; - match self { - Val::Obj(ref mut o) => { - use indexmap::map::Entry::{Occupied, Vacant}; - match Rc::make_mut(o).entry(index.clone()) { - Occupied(mut e) => { - let v = core::mem::take(e.get_mut()); - match f(v).next().transpose()? { - Some(y) => e.insert(y), - // this runs in constant time, at the price of - // changing the order of the elements - None => e.swap_remove(), - }; - } - Vacant(e) => { - if let Some(y) = f(Val::Null).next().transpose()? { - e.insert(y); - } - } - } - // BASHKIT PATCH: charge in-place growth (TM-DOS-110). - Rc::make_mut(o).resync(); - meter::check(0)?; - Ok(self) - } - Val::Arr(ref mut a) => { - let oob = || Error::str(format_args!("index {index} out of bounds")); - let abs_or = |i| abs_index(i, a.len()).ok_or_else(oob); - let i = match index.as_pos_usize().and_then(abs_or) { - Ok(i) => i, - Err(e) => return opt.fail(self, |_| Exn::from(e)), - }; - - let a = Rc::make_mut(a); - let x = core::mem::take(&mut a[i]); - if let Some(y) = f(x).next().transpose()? { - a[i] = y; - } else { - a.remove(i); - } - a.resync(); // BASHKIT PATCH (TM-DOS-110) - Ok(self) - } - _ => opt.fail(self, |v| Exn::from(Error::typ(v, Type::Iter.as_str()))), - } + let hit = core::cell::Cell::new(false); + let y = empty.map_index_inner(index, opt, |v| f(v).inspect(|_| hit.set(true)))?; + Ok(if hit.get() { y } else { Val::Null }) } + fn map_range<'a, I: Iterator>>( - mut self, + self, range: val::Range<&Self>, opt: path::Opt, f: impl Fn(Self) -> I, ) -> ValX<'a> { - let fs = |b: Bytes, range, skip_take: SkipTakeFn, from: ValBytesFn, into: BytesValFn| { - let (skip, take) = match Self::range_int(range) { - Ok(range) => skip_take(range, &b), - Err(e) => return opt.fail(into(b), |_| Exn::from(e)), - }; - let str = into(b.slice(skip..skip + take)); - let y = f(str).map(|y| from(y?).map_err(Exn::from)).next(); - let y = y.transpose()?.unwrap_or_default(); - let mut b = BytesMut::from(b); - // BASHKIT PATCH: size-check and meter the result (TM-DOS-110). - meter::check(b.len() - take + y.len())?; - bytes_splice(&mut b, skip, take, &y); - Ok(into(b.freeze())) - }; - let stb = skip_take_bytes; - let stc = skip_take_chars; - match self { - Val::Arr(ref mut a) => { - let (skip, take) = match Self::range_int(range) { - Ok(range) => skip_take(range, a.len()), - Err(e) => return opt.fail(self, |_| Exn::from(e)), - }; - let arr = a.iter().skip(skip).take(take).cloned().collect(); - let y = f(arr).map(|y| y?.into_arr().map_err(Exn::from)).next(); - let y = y.transpose()?.unwrap_or_default(); - meter::check((a.len() - take + y.len()) * core::mem::size_of::())?; // BASHKIT PATCH - let a = Rc::make_mut(a); - a.splice(skip..skip + take, (**y).clone()); - a.resync(); // BASHKIT PATCH (TM-DOS-110) - Ok(self) - } - Val::BStr(b) => fs(b.into_bytes(), range, stb, Val::into_byte_str, Val::byte_str), - Val::TStr(b) => fs(b.into_bytes(), range, stc, Val::into_utf8_str, Val::utf8_str), - _ => opt.fail(self, |v| Exn::from(Error::typ(v, Type::Arr.as_str()))), + // BASHKIT PATCH: a slice update through null works on `[]`, like jq. + if !matches!(self, Val::Null) { + return self.map_range_inner(range, opt, f); } + let empty: Val = core::iter::empty::().collect(); + let hit = core::cell::Cell::new(false); + let y = empty.map_range_inner(range, opt, |v| f(v).inspect(|_| hit.set(true)))?; + Ok(if hit.get() { y } else { Val::Null }) } + /// True if the value is neither null nor false. fn as_bool(&self) -> bool { meter::tick(); // BASHKIT PATCH (TM-DOS-110) @@ -433,6 +370,144 @@ fn abs_index(i: num::PosUsize, len: usize) -> Option { } impl Val { + /// `map_index` for a value that is not null. + fn map_index_inner<'a, I: Iterator>>( + mut self, + index: &Self, + opt: path::Opt, + f: impl Fn(Self) -> I, + ) -> ValX<'a> { + meter::tick(); // BASHKIT PATCH (TM-DOS-110) + if let (Val::BStr(_) | Val::TStr(_) | Val::Arr(_), Val::Obj(o)) = (&self, index) { + let range = o.get(&Val::utf8_str("start"))..o.get(&Val::utf8_str("end")); + return self.map_range_inner(range, opt, f); + }; + match self { + // BASHKIT PATCH: jq objects have string keys only. + Val::Obj(_) if !matches!(index, Val::TStr(_) | Val::BStr(_)) => { + let e = jqmsg::index(self.clone(), index.clone()); + opt.fail(self, |_| Exn::from(e)) + } + Val::Obj(ref mut o) => { + use indexmap::map::Entry::{Occupied, Vacant}; + match Rc::make_mut(o).entry(index.clone()) { + Occupied(mut e) => { + let v = core::mem::take(e.get_mut()); + match f(v).next().transpose()? { + Some(y) => e.insert(y), + // this runs in constant time, at the price of + // changing the order of the elements + None => e.swap_remove(), + }; + } + Vacant(e) => { + if let Some(y) = f(Val::Null).next().transpose()? { + e.insert(y); + } + } + } + // BASHKIT PATCH: charge in-place growth (TM-DOS-110). + Rc::make_mut(o).resync(); + meter::check(0)?; + Ok(self) + } + Val::Arr(ref mut a) => { + // BASHKIT PATCH: jq truncates a fractional index, rejects a + // negative one before the start, and pads with null to grow + // the array for an index past the end. + let pos = match index { + Val::Num(n) if n.as_f64().is_finite() => jqmsg::trunc_int(n).as_pos_usize(), + _ => None, + }; + let len = a.len(); + let i = match pos { + Some(p) => p.wrap(len), + None => { + let e = jqmsg::index(Val::Arr(a.clone()), index.clone()); + return opt.fail(self, |_| Exn::from(e)); + } + }; + let Some(i) = i else { + let e = Error::str("Out of bounds negative array index"); + return opt.fail(self, |_| Exn::from(e)); + }; + if i >= len { + // jq's own cap on growing an array by index. + if i > (i32::MAX >> 2) as usize { + let e = Error::str("Array index too large"); + return opt.fail(self, |_| Exn::from(e)); + } + let Some(y) = f(Val::Null).next().transpose()? else { + return Ok(self); + }; + let grow = (i + 1 - len).saturating_mul(core::mem::size_of::()); + meter::check(grow)?; + let a = Rc::make_mut(a); + a.resize(i, Val::Null); + a.push(y); + a.resync(); // BASHKIT PATCH (TM-DOS-110) + meter::check(0)?; + return Ok(self); + } + + let a = Rc::make_mut(a); + let x = core::mem::take(&mut a[i]); + if let Some(y) = f(x).next().transpose()? { + a[i] = y; + } else { + a.remove(i); + } + a.resync(); // BASHKIT PATCH (TM-DOS-110) + Ok(self) + } + _ => opt.fail(self, |v| Exn::from(jqmsg::iter(v))), // BASHKIT PATCH + } + } + + /// `map_range` for a value that is not null. + fn map_range_inner<'a, I: Iterator>>( + mut self, + range: val::Range<&Self>, + opt: path::Opt, + f: impl Fn(Self) -> I, + ) -> ValX<'a> { + let fs = |b: Bytes, range, skip_take: SkipTakeFn, from: ValBytesFn, into: BytesValFn| { + let (skip, take) = match Self::range_int(range) { + Ok(range) => skip_take(range, &b), + Err(e) => return opt.fail(into(b), |_| Exn::from(e)), + }; + let str = into(b.slice(skip..skip + take)); + let y = f(str).map(|y| from(y?).map_err(Exn::from)).next(); + let y = y.transpose()?.unwrap_or_default(); + let mut b = BytesMut::from(b); + // BASHKIT PATCH: size-check and meter the result (TM-DOS-110). + meter::check(b.len() - take + y.len())?; + bytes_splice(&mut b, skip, take, &y); + Ok(into(b.freeze())) + }; + let stb = skip_take_bytes; + let stc = skip_take_chars; + match self { + Val::Arr(ref mut a) => { + let (skip, take) = match Self::range_int(range) { + Ok(range) => skip_take(range, a.len()), + Err(e) => return opt.fail(self, |_| Exn::from(e)), + }; + let arr = a.iter().skip(skip).take(take).cloned().collect(); + let y = f(arr).map(|y| y?.into_arr().map_err(Exn::from)).next(); + let y = y.transpose()?.unwrap_or_default(); + meter::check((a.len() - take + y.len()) * core::mem::size_of::())?; // BASHKIT PATCH + let a = Rc::make_mut(a); + a.splice(skip..skip + take, (**y).clone()); + a.resync(); // BASHKIT PATCH (TM-DOS-110) + Ok(self) + } + Val::BStr(b) => fs(b.into_bytes(), range, stb, Val::into_byte_str, Val::byte_str), + Val::TStr(b) => fs(b.into_bytes(), range, stc, Val::into_utf8_str, Val::utf8_str), + _ => opt.fail(self, |v| Exn::from(Error::typ(v, Type::Arr.as_str()))), + } + } + /// Construct an object value. pub fn obj(m: Map) -> Self { Self::Obj(metered(m)) diff --git a/crates/bashkit/src/builtins/jq/tests.rs b/crates/bashkit/src/builtins/jq/tests.rs index c437acd29..02fc0dba8 100644 --- a/crates/bashkit/src/builtins/jq/tests.rs +++ b/crates/bashkit/src/builtins/jq/tests.rs @@ -2035,3 +2035,40 @@ async fn gamma_is_log_gamma() { .unwrap(); assert_eq!(out, "[8.525161361065413,1,3]\n"); } + +#[tokio::test] +async fn update_through_null_creates_containers() { + let out = run_jq_with_args( + &[ + "-nc", + "(.a.b.c = 1), (.[2] = 1), (.[1:3] = [\"x\"]), (.a |= empty), \ + ([1] | .[3] = 9), ([1] | .[1.7] = 2), ([1] | .[5] |= empty)", + ], + "", + ) + .await + .unwrap(); + assert_eq!( + out, + "{\"a\":{\"b\":{\"c\":1}}}\n[null,null,1]\n[\"x\"]\nnull\n[1,null,null,9]\n[1,2]\n[1]\n" + ); +} + +#[tokio::test] +async fn update_index_errors_follow_jq() { + let out = run_jq_with_args( + &[ + "-nc", + "(try (.[-1] = 1) catch .), (try ({} | .[1] = 2) catch .), \ + (try (.[1e9] = 1) catch .), (try (.[] = 1) catch .)", + ], + "", + ) + .await + .unwrap(); + assert_eq!( + out, + "\"Out of bounds negative array index\"\n\"Cannot index object with number\"\n\ + \"Array index too large\"\n\"Cannot iterate over null (null)\"\n" + ); +} diff --git a/knowledge/log.md b/knowledge/log.md index 42a148c15..39c3c11b2 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -2,6 +2,7 @@ ## 2026-10-07 +* **Fix**: jq assignments through null work like jq: `null | .a.b.c = 1`, `.[2] = 1`, `.a += 1` and slice updates create the containers (they failed with `Cannot iterate over null`); `|= empty` on a missing path keeps null; an index past the end pads the array with null; fractional update indexes truncate; jq's error texts for negative and too-large indexes and for iterating a scalar. Debian-oracle jq score 75.8% to 79.9%. * **Fix**: sed follows GNU on character escapes, `-z`, unreadable inputs and long loops. `\xHH`, `\dNNN`, `\oNNN`, `\cX` work in regexes, replacements and `y` and always produce a literal character (`s/z/\x26/` inserts `&`); `\t`/`\n` work inside bracket expressions (`s/^[ \t]*//`). Under `-z`, `N`/`G`/`H` join with NUL, `D`/`P` split on it, and `M` anchors match around it. An unreadable input file is reported and skipped (exit 2 at the end). The loop budget now counts per input line read, so `$!N;P;D` no longer stopped after 1000 lines. Debian-oracle sed score 94.2% to 99.4%. * **Fix**: jq numbers and value errors follow jq 1.7: computed floats print with jq's formatting (`1024`, `1e+17`, `1e-05`, NaN as null, infinities as the largest double), input literals keep their token until computed, division by zero and `% 0` are errors, `%` truncates operands, fractional array indexes truncate, indexing an object with a non-string is an error, `gamma` is log-gamma, and math/index errors use jq's wording (also inside `catch`). Patched in the vendored jaq-json (`jqmsg.rs`, `jq_format_f64`, `defs.jq`); L-JQ-004 records the literal-preservation gap. * **Fix**: jq input stream follows jq 1.7: `-n` leaves the stream to `input`/`inputs`, `-s` on empty input gives `[]`, each FILE is read separately (`input_filename`, `input_line_number` as jq counts lines), a missing file is reported and skipped (exit 2), a runtime error prints `jq: error (at FILE:LINE): ...` and moves on to the next input, `input` fails with `No more inputs`, `-e` uses the last output, values before a JSON syntax error are processed. Debian-oracle jq score 67.5% to 74.2%. diff --git a/knowledge/runtimes/jaq-json-vendor.md b/knowledge/runtimes/jaq-json-vendor.md index f8282b72e..5ed55f986 100644 --- a/knowledge/runtimes/jaq-json-vendor.md +++ b/knowledge/runtimes/jaq-json-vendor.md @@ -109,6 +109,11 @@ Besides the meter, a few `BASHKIT PATCH` hunks make values behave like jq (`string ("a") and number (1) cannot be added`, operands cut at 11 bytes). - `mod.rs`: division and `%` by zero fail, `%` truncates operands, a fractional array index truncates, an object indexed by a non-string fails. +- `mod.rs` path updates: assigning through null creates the container + (`null | .a.b = 1`, `.[2] = 1`, slices), stays null when the update yields + nothing, an index past the end pads with null (capped like jq: `Array + index too large`), a negative index before the start fails, and `.[]` on a + scalar says `Cannot iterate over number (1)`. - `defs.jq`: `nan`/`infinite` without dividing by zero (and the jaq-std filters built on them), `gamma` as log-gamma. From f8b7f423d19dcfcc30f153031d6ecce49b69902f Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Wed, 7 Oct 2026 20:20:54 +0000 Subject: [PATCH 2/2] fix(jq): join, trims, scan groups, capture nulls, from_entries and tostream like jq Claude-Session: https://claude.ai/code/session_01VscpEzxtqpCKx5eRMpywNj --- crates/bashkit/src/builtins/jq/compat.rs | 53 ++++++++++++-- crates/bashkit/src/builtins/jq/format.rs | 3 +- .../bashkit/src/builtins/jq/jaq_json/jqmsg.rs | 5 ++ .../bashkit/src/builtins/jq/jaq_json/mod.rs | 10 ++- .../bashkit/src/builtins/jq/regex_compat.rs | 13 +++- crates/bashkit/src/builtins/jq/tests.rs | 69 +++++++++++++++++++ knowledge/log.md | 1 + knowledge/runtimes/jaq-json-vendor.md | 2 + 8 files changed, 148 insertions(+), 8 deletions(-) diff --git a/crates/bashkit/src/builtins/jq/compat.rs b/crates/bashkit/src/builtins/jq/compat.rs index 69e87404a..6afc33c4e 100644 --- a/crates/bashkit/src/builtins/jq/compat.rs +++ b/crates/bashkit/src/builtins/jq/compat.rs @@ -10,6 +10,11 @@ //! For `scan`, real jq's `scan(re; flags)` with empty flags is global by //! default — we explicitly add `g` if not already present in `flags` to //! avoid a double-`g` regex compile error from `"g" + "g"`. +//! - `scan` / `capture` / `join` / `ltrimstr` / `rtrimstr` / +//! `from_entries` / `IN` / `tostream` / `fromstream` / +//! `truncate_stream`: jq 1.7's definitions (capture groups in `scan`, +//! null for an unmatched group, null joins as "", non-strings pass +//! through the trim filters). //! - `@tsv` / `@csv`: jaq-std doesn't define them. Strict variants reject //! non-scalars with a runtime error matching real jq's wording. //! - `input_filename` / `input_line_number`: bashkit threads these as @@ -59,13 +64,53 @@ def match(re; flags): matches(re; flags)[] | .[0] as $m | { offset: $m.offset, length: $m.length, string: $m.string, - captures: [.[1:][] | { offset: .offset, length: .length, string: .string, - name: (if has("name") then .name else null end) }] }; + captures: [.[1:][] | + (if has("name") then .name else null end) as $name | + if .offset < 0 then { offset: .offset, string: null, length: .length, name: $name } + else { offset: .offset, length: .length, string: .string, name: $name } end] }; def match(re): match(re; ""); def scan(re; flags): - matches(re; if (flags | test("g")) then flags else "g" + flags end)[] - | .[0].string; + match(re; if (flags | test("g")) then flags else "g" + flags end) + | if (.captures | length) > 0 then [.captures[].string] else .string end; def scan(re): scan(re; ""); +def capture(re; flags): + match(re; flags) + | reduce (.captures[] | select(.name != null)) as $c ({}; . + {($c.name): $c.string}); +def capture(re): capture(re; ""); +def join($x): + reduce .[] as $i (null; + (if . == null then "" else . + $x end) + + ($i | if . == null then "" + elif type == "boolean" or type == "number" then tojson + else . end)) // ""; +def _bk_ltrimstr($x): ltrimstr($x); +def _bk_rtrimstr($x): rtrimstr($x); +def ltrimstr($x): + if type == "string" and ($x | type) == "string" then _bk_ltrimstr($x) else . end; +def rtrimstr($x): + if type == "string" and ($x | type) == "string" then _bk_rtrimstr($x) else . end; +def from_entries: + reduce .[] as $x ({}; + . + { ($x | .key // .Key // .name // .Name): + ($x | if has("value") then .value else .Value end) }); +def IN(s): any(s == .; .); +def IN(src; s): any(src == s; .); +def _bk_postorder: (.[]? | _bk_postorder), .; +def tostream: + path(_bk_postorder) as $p + | getpath($p) + | reduce path(.[]?) as $q ([$p, .]; [$p + $q]); +def fromstream(f): + { x: null, e: false } as $init + | foreach f as $i ($init; + if .e then $init else . end + | if $i | length == 2 + then setpath(["e"]; $i[0] | length == 0) | setpath(["x"] + $i[0]; $i[1]) + else setpath(["e"]; $i[0] | length == 1) end; + if .e then .x else empty end); +def truncate_stream(stream): + . as $n | null | stream + | if (.[0] | length) > $n then setpath([0]; .[0][$n:]) else empty end; def @tsv: [.[] | if type == "string" then diff --git a/crates/bashkit/src/builtins/jq/format.rs b/crates/bashkit/src/builtins/jq/format.rs index 4e0761146..7b9e2c7b6 100644 --- a/crates/bashkit/src/builtins/jq/format.rs +++ b/crates/bashkit/src/builtins/jq/format.rs @@ -151,7 +151,8 @@ fn write_json_string(s: &str, out: &mut String) { '\t' => out.push_str("\\t"), '\x08' => out.push_str("\\b"), '\x0c' => out.push_str("\\f"), - c if (c as u32) < 0x20 => { + // jq also escapes DEL. + c if (c as u32) < 0x20 || c == '\x7f' => { out.push_str(&format!("\\u{:04x}", c as u32)); } c => out.push(c), diff --git a/crates/bashkit/src/builtins/jq/jaq_json/jqmsg.rs b/crates/bashkit/src/builtins/jq/jaq_json/jqmsg.rs index 247f18475..eefac0ec3 100644 --- a/crates/bashkit/src/builtins/jq/jaq_json/jqmsg.rs +++ b/crates/bashkit/src/builtins/jq/jaq_json/jqmsg.rs @@ -74,6 +74,11 @@ pub(crate) fn iter(v: Val) -> Error { Error::str(std::format!("Cannot iterate over {}", described(&v))) } +/// `{(k): v}` with a key that is not a string. +pub(crate) fn object_key(k: &Val) -> Error { + Error::str(std::format!("Cannot use {} as object key", described(k))) +} + pub(crate) fn is_zero(v: &Val) -> bool { matches!(v, Val::Num(n) if n.as_f64() == 0.0) } diff --git a/crates/bashkit/src/builtins/jq/jaq_json/mod.rs b/crates/bashkit/src/builtins/jq/jaq_json/mod.rs index f1330ef91..ef6b8ee92 100644 --- a/crates/bashkit/src/builtins/jq/jaq_json/mod.rs +++ b/crates/bashkit/src/builtins/jq/jaq_json/mod.rs @@ -148,7 +148,15 @@ impl jaq_core::ValT for Val { } fn from_map>(iter: I) -> ValR { - Ok(Self::obj(iter.into_iter().collect())) + // BASHKIT PATCH: jq object keys are strings only. + let mut m = Map::default(); + for (k, v) in iter { + if !matches!(k, Val::TStr(_) | Val::BStr(_)) { + return Err(jqmsg::object_key(&k)); + } + m.insert(k, v); + } + Ok(Self::obj(m)) } fn key_values(self) -> Box>> { diff --git a/crates/bashkit/src/builtins/jq/regex_compat.rs b/crates/bashkit/src/builtins/jq/regex_compat.rs index 088972397..65c30de97 100644 --- a/crates/bashkit/src/builtins/jq/regex_compat.rs +++ b/crates/bashkit/src/builtins/jq/regex_compat.rs @@ -167,8 +167,17 @@ fn run_regex( let mut match_objs: Vec = Vec::with_capacity(names.len()); for (idx, name) in names.into_iter().enumerate() { let Some(m) = caps.get(idx) else { - // Optional group that didn't match — skip, matching jaq's - // filter_map behavior. + // Optional group that didn't match: jq reports it with + // offset -1 and a null string (`capture` gives null). + // `string` is left out here; `.string` reads it as null. + let mut fields: Vec<(V, V)> = vec![ + (V::from(String::from("offset")), V::from(-1isize)), + (V::from(String::from("length")), V::from(0isize)), + ]; + if let Some(n) = name { + fields.push((V::from(String::from("name")), V::from(n.to_string()))); + } + match_objs.push(V::from_map(fields)?); continue; }; let offset = bc.byte_to_char(m.start()); diff --git a/crates/bashkit/src/builtins/jq/tests.rs b/crates/bashkit/src/builtins/jq/tests.rs index 02fc0dba8..7c5d78ddf 100644 --- a/crates/bashkit/src/builtins/jq/tests.rs +++ b/crates/bashkit/src/builtins/jq/tests.rs @@ -2072,3 +2072,72 @@ async fn update_index_errors_follow_jq() { \"Array index too large\"\n\"Cannot iterate over null (null)\"\n" ); } + +#[tokio::test] +async fn string_builtins_follow_jq() { + let out = run_jq_with_args( + &[ + "-nc", + "([1, null, \"a\", true] | join(\"-\")), ([null] | join(\",\")), \ + (try ([[1]] | join(\",\")) catch .), (1 | ltrimstr(\"a\")), \ + (\"ab\" | rtrimstr(1)), (\"a1b2\" | [scan(\"([a-z])([0-9])\")]), \ + (\"ab\" | capture(\"(?a)(?z)?\")), \"\\u007f\"", + ], + "", + ) + .await + .unwrap(); + assert_eq!( + out, + "\"1--a-true\"\n\"\"\n\"string (\\\"\\\") and array ([1]) cannot be added\"\n1\n\"ab\"\n\ + [[\"a\",\"1\"],[\"b\",\"2\"]]\n{\"x\":\"a\",\"y\":null}\n\"\\u007f\"\n" + ); +} + +#[tokio::test] +async fn unmatched_group_is_reported_like_jq() { + let out = run_jq_with_args(&["-c", "[match(\"(a)(z)?\").captures[]]"], "\"ab\"") + .await + .unwrap(); + assert_eq!( + out, + "[{\"offset\":0,\"length\":1,\"string\":\"a\",\"name\":null},\ + {\"offset\":-1,\"string\":null,\"length\":0,\"name\":null}]\n" + ); +} + +#[tokio::test] +async fn from_entries_and_object_keys_follow_jq() { + let out = run_jq_with_args( + &["-c", "from_entries"], + r#"[{"key":"a","value":1},{"name":"c","value":3},{"Name":"N","value":4},{"key":"d"},{"name":"f","Value":6}]"#, + ) + .await + .unwrap(); + assert_eq!(out, "{\"a\":1,\"c\":3,\"N\":4,\"d\":null,\"f\":6}\n"); + let result = run_jq_result_with_args(&["-c", "from_entries"], r#"[{"key":null,"value":4}]"#) + .await + .unwrap(); + assert_eq!( + result.stderr, + "jq: error (at :0): Cannot use null (null) as object key\n" + ); + assert_eq!(result.exit_code, 5); +} + +#[tokio::test] +async fn tostream_round_trips() { + let out = run_jq_with_args( + &[ + "-c", + "[tostream], fromstream(tostream), [1 | truncate_stream([[0], 1], [[1, 0], 2], [[1, 0]], [[1]])], IN(.[]; 1)", + ], + r#"[1,{"a":2},[]]"#, + ) + .await + .unwrap(); + assert_eq!( + out, + "[[[0],1],[[1,\"a\"],2],[[1,\"a\"]],[[2],[]],[[2]]]\n[1,{\"a\":2},[]]\n[[[0],2],[[0]]]\ntrue\n" + ); +} diff --git a/knowledge/log.md b/knowledge/log.md index 39c3c11b2..a491ffb8f 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -2,6 +2,7 @@ ## 2026-10-07 +* **Fix**: jq builtins follow jq 1.7: `join` prints null as "" and fails on arrays/objects, `ltrimstr`/`rtrimstr` pass non-strings through, `scan` with capture groups yields the groups, `match` reports an unmatched group (offset -1, string null) so `capture` gives null, `from_entries` takes `key`/`Key`/`name`/`Name` and `value`/`Value`, object keys must be strings, `tostream`/`fromstream`/`truncate_stream`/`IN` are defined, DEL prints as `\u007f`. Debian-oracle jq score 79.9% to 84.0%. * **Fix**: jq assignments through null work like jq: `null | .a.b.c = 1`, `.[2] = 1`, `.a += 1` and slice updates create the containers (they failed with `Cannot iterate over null`); `|= empty` on a missing path keeps null; an index past the end pads the array with null; fractional update indexes truncate; jq's error texts for negative and too-large indexes and for iterating a scalar. Debian-oracle jq score 75.8% to 79.9%. * **Fix**: sed follows GNU on character escapes, `-z`, unreadable inputs and long loops. `\xHH`, `\dNNN`, `\oNNN`, `\cX` work in regexes, replacements and `y` and always produce a literal character (`s/z/\x26/` inserts `&`); `\t`/`\n` work inside bracket expressions (`s/^[ \t]*//`). Under `-z`, `N`/`G`/`H` join with NUL, `D`/`P` split on it, and `M` anchors match around it. An unreadable input file is reported and skipped (exit 2 at the end). The loop budget now counts per input line read, so `$!N;P;D` no longer stopped after 1000 lines. Debian-oracle sed score 94.2% to 99.4%. * **Fix**: jq numbers and value errors follow jq 1.7: computed floats print with jq's formatting (`1024`, `1e+17`, `1e-05`, NaN as null, infinities as the largest double), input literals keep their token until computed, division by zero and `% 0` are errors, `%` truncates operands, fractional array indexes truncate, indexing an object with a non-string is an error, `gamma` is log-gamma, and math/index errors use jq's wording (also inside `catch`). Patched in the vendored jaq-json (`jqmsg.rs`, `jq_format_f64`, `defs.jq`); L-JQ-004 records the literal-preservation gap. diff --git a/knowledge/runtimes/jaq-json-vendor.md b/knowledge/runtimes/jaq-json-vendor.md index 5ed55f986..e83f522f5 100644 --- a/knowledge/runtimes/jaq-json-vendor.md +++ b/knowledge/runtimes/jaq-json-vendor.md @@ -114,6 +114,8 @@ Besides the meter, a few `BASHKIT PATCH` hunks make values behave like jq nothing, an index past the end pads with null (capped like jq: `Array index too large`), a negative index before the start fails, and `.[]` on a scalar says `Cannot iterate over number (1)`. +- `mod.rs` `from_map`: object keys must be strings + (`Cannot use null (null) as object key`). - `defs.jq`: `nan`/`infinite` without dividing by zero (and the jaq-std filters built on them), `gamma` as log-gamma.