diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 97e3b3edd..742f01b87 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,6 +16,7 @@ env: CARGO_TERM_COLOR: always RUST_BACKTRACE: 1 RG_VERSION: 15.1.0 + JQ_VERSION: 1.8.1 jobs: lint: @@ -257,7 +258,14 @@ jobs: test: name: Test - runs-on: ubuntu-latest + # Pinned, not `ubuntu-latest`: this job's `*_match_gnu` differential tests + # need real GNU coreutils as the oracle, and 24.04 is the last image that + # ships it as the default. Ubuntu 26.04 installs uutils as `coreutils` + # instead, so `dd`/`join`/`od`/`pr` stopped reporting "GNU coreutils" and + # those tests fail closed by design rather than skipping. Unpin once the + # GNU oracle is installed explicitly, the way `rg` and `jq` already are; + # see "Pinned External Oracles" in knowledge/operations/testing.md. + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -302,6 +310,12 @@ jobs: - name: Install pinned ripgrep for differential tests run: scripts/install-ripgrep-ci.sh "$RG_VERSION" + # Same reason for `jq`, and the runner image already proved it: an + # unpinned oracle made the jq parity suite a function of which + # `ubuntu-latest` image the job landed on. + - name: Install pinned jq for differential tests + run: scripts/install-jq-ci.sh "$JQ_VERSION" + # Examples have a dedicated job below; skipping them here avoids # duplicated example links that can exhaust runner disk on main. - name: Run tests diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index bc34ca831..c70828d04 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -20,11 +20,14 @@ env: CARGO_TERM_COLOR: always LLVM_COV_IGNORE_FILENAME_REGEX: "(tests/|test_cases/|/tests\\.rs$)" RG_VERSION: 15.1.0 + JQ_VERSION: 1.8.1 jobs: coverage: name: Generate Rust Coverage - runs-on: ubuntu-latest + # Pinned for the GNU coreutils oracle, like ci.yml's `test` job; see the + # note there. + runs-on: ubuntu-24.04 timeout-minutes: 60 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -48,6 +51,9 @@ jobs: - name: Install pinned ripgrep for differential tests run: scripts/install-ripgrep-ci.sh "$RG_VERSION" + - name: Install pinned jq for differential tests + run: scripts/install-jq-ci.sh "$JQ_VERSION" + # bashkit-eval is excluded: it enables bashkit's cpython and sqlite # features, and under --workspace feature unification that would pull # the slow CPython/sqlite tests into the instrumented bashkit binaries diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 045f3a294..7772d04a5 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -14,6 +14,7 @@ env: CARGO_TERM_COLOR: always RUST_BACKTRACE: 1 RG_VERSION: 15.1.0 + JQ_VERSION: 1.8.1 jobs: miri: @@ -124,7 +125,9 @@ jobs: asan: name: AddressSanitizer - runs-on: ubuntu-latest + # Pinned for the GNU coreutils oracle, like ci.yml's `test` job; see the + # note there. + runs-on: ubuntu-24.04 # ASAN with --test-threads=1 plus the expanded rg/coreutils # differential corpora needs well over the previous 30-minute cap; # the last two nightly runs cancelled mid-suite at exactly 30 min. @@ -149,10 +152,14 @@ jobs: with: cache-on-failure: true - # ASAN runs the Rust test suite, including `rg` differential tests. + # ASAN runs the Rust test suite, including the `rg` and `jq` + # differential tests. - name: Install pinned ripgrep for differential tests run: scripts/install-ripgrep-ci.sh "$RG_VERSION" + - name: Install pinned jq for differential tests + run: scripts/install-jq-ci.sh "$JQ_VERSION" + - name: Run tests with AddressSanitizer env: RUSTFLAGS: "-Z sanitizer=address" diff --git a/.github/workflows/publish-python.yml b/.github/workflows/publish-python.yml index 2dbe6a5ce..b57e60e57 100644 --- a/.github/workflows/publish-python.yml +++ b/.github/workflows/publish-python.yml @@ -240,8 +240,11 @@ jobs: os: [linux, macos, windows] python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] include: + # linux is pinned rather than `ubuntu-latest` because Ubuntu 26.04 + # ships no hosted 3.9 build; see the note on `python.yml`'s `test` + # job, which is pinned for the same reason. - os: linux - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 - os: macos runs-on: macos-latest - os: windows diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index d55d01e7d..ef8acbf9a 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -69,7 +69,16 @@ jobs: test: name: Test (Python ${{ matrix.python-version }}) needs: [build-wheel] - runs-on: ubuntu-latest + # Pinned, not `ubuntu-latest`: Ubuntu 26.04 ships no hosted 3.9 build, and + # `setup-python` has no fallback, so the 3.9 leg died the moment + # `ubuntu-latest` began rolling onto it ("The version '3.9' with + # architecture 'x64' was not found for Ubuntu 26.04"). The wheel is + # cp39-abi3 and the package claims >=3.9, so the fix is an image that can + # host the whole declared range, not a smaller matrix. Unpin when 3.9 + # support is dropped; `publish-python.yml`'s `test-builds` is pinned for + # the same reason and the two matrices are kept in step by + # `just check-workflow-parity`. + runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: diff --git a/Cargo.lock b/Cargo.lock index c55678c50..42454453c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4936,9 +4936,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.15" +version = "0.103.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" dependencies = [ "aws-lc-rs", "ring", @@ -5667,7 +5667,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix", "windows-sys 0.61.2", @@ -6740,9 +6740,9 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +checksum = "58ec80e73fd8baf57bf0d9615554c75348bda990d6d5bbeab964dd068319fecb" dependencies = [ "rustls-pki-types", ] diff --git a/crates/bashkit/docs/jq.md b/crates/bashkit/docs/jq.md index f4bafdf96..37fd893aa 100644 --- a/crates/bashkit/docs/jq.md +++ b/crates/bashkit/docs/jq.md @@ -27,6 +27,7 @@ Implemented: | `-e`, `--exit-status` | Set exit code based on the output | | `-j` | Like `-r` but suppresses trailing newlines | | `--tab` | Use tabs for indentation | +| `--indent N` | Indent with N spaces, 0..7 (default 2). `0` still pretty-prints — newlines, no leading spaces — so `-c` is the only way to get compact output | | `-a`, `--ascii-output` | Escape non-ASCII characters as `\uXXXX` (strings stay quoted even with `-r`) | | `--raw-output0` | Like `-r` with a NUL after each output; a string containing NUL is an error | | `--seq` | Write RS (0x1e) before each output; RS separates input values | @@ -38,6 +39,10 @@ Implemented: | `-h`, `--help` | Print help | | Combined flags like `-snr` | Treated as the union of the individual flags | +`-c`, `--tab` and `--indent N` all choose the same thing — the output shape — +so the last one on the command line wins. `jq --indent 4 -c` is compact and +`jq -c --indent 4` is indented four spaces. + ## Variables | Variable | Behaviour | diff --git a/crates/bashkit/fuzz/Cargo.lock b/crates/bashkit/fuzz/Cargo.lock index 6f64bcf4e..0bf111d1b 100644 --- a/crates/bashkit/fuzz/Cargo.lock +++ b/crates/bashkit/fuzz/Cargo.lock @@ -718,7 +718,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1564,7 +1564,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1593,9 +1593,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.15" +version = "0.103.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" dependencies = [ "ring", "rustls-pki-types", @@ -2271,7 +2271,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/crates/bashkit/src/builtins/jq/args.rs b/crates/bashkit/src/builtins/jq/args.rs index 5b0d7b93e..d59e39a51 100644 --- a/crates/bashkit/src/builtins/jq/args.rs +++ b/crates/bashkit/src/builtins/jq/args.rs @@ -41,7 +41,6 @@ pub(super) struct JqArgs<'a> { pub raw_input: bool, pub raw_output: bool, pub join_output: bool, - pub compact_output: bool, pub null_input: bool, pub slurp: bool, pub sort_keys: bool, @@ -55,6 +54,10 @@ pub(super) struct JqArgs<'a> { pub stream: bool, /// `-e` flag — set exit status from output. pub exit_status: bool, + /// Output shape, set by `-c`/`--compact-output`, `--tab` and + /// `--indent N`. One field rather than a compact flag beside an indent + /// width, because real jq lets the last of those three on the command + /// line win: `jq --indent 4 -c` is compact and `jq -c --indent 4` is not. pub indent: Indent, pub file_args: Vec<&'a str>, /// Named string/JSON variables: `(name_with_$, value)`. @@ -147,7 +150,6 @@ pub(super) fn parse<'a>(args: &'a [String]) -> ParseOutcome<'a> { raw_input: false, raw_output: false, join_output: false, - compact_output: false, null_input: false, slurp: false, ascii_output: false, @@ -212,7 +214,7 @@ pub(super) fn parse<'a>(args: &'a [String]) -> ParseOutcome<'a> { match arg.as_str() { "--raw-output" => out.raw_output = true, "--raw-input" => out.raw_input = true, - "--compact-output" => out.compact_output = true, + "--compact-output" => out.indent = Indent::Compact, "--null-input" => out.null_input = true, "--sort-keys" => out.sort_keys = true, "--slurp" => out.slurp = true, @@ -230,9 +232,7 @@ pub(super) fn parse<'a>(args: &'a [String]) -> ParseOutcome<'a> { "--indent" => match args.get(i + 1) { Some(n) => match n.parse::() { Ok(n) if n <= MAX_JQ_INDENT => { - if !matches!(out.indent, Indent::Tab) { - out.indent = Indent::Spaces(n); - } + out.indent = Indent::Spaces(n); i += 2; continue; } @@ -338,7 +338,7 @@ pub(super) fn parse<'a>(args: &'a [String]) -> ParseOutcome<'a> { 'f' => takes_file = true, 'r' => out.raw_output = true, 'R' => out.raw_input = true, - 'c' => out.compact_output = true, + 'c' => out.indent = Indent::Compact, 'n' => out.null_input = true, 'S' => out.sort_keys = true, 's' => out.slurp = true, diff --git a/crates/bashkit/src/builtins/jq/format.rs b/crates/bashkit/src/builtins/jq/format.rs index 7b9e2c7b6..a817402bc 100644 --- a/crates/bashkit/src/builtins/jq/format.rs +++ b/crates/bashkit/src/builtins/jq/format.rs @@ -7,8 +7,8 @@ use super::convert::JqJson; -/// Output indentation choice. `Spaces(0)` matches `-c` semantics in tools -/// that bypass the compact path. +/// Output indentation choice. `-c` is the only way to get compact output; +/// `Spaces(0)` is still pretty-printed, just with zero-width indentation. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(super) enum Indent { /// Compact output, no whitespace between tokens (matches `-c`). @@ -36,13 +36,14 @@ pub(super) fn sort_keys(v: JqJson) -> JqJson { /// Render a `JqJson` value using the chosen indent style. /// -/// Note: real jq treats `--indent 0` as equivalent to `-c` (compact, no -/// newlines), not "newlines without indentation". We match that. +/// Note: `--indent 0` is pretty output with zero-width indentation — newlines +/// between tokens, no leading spaces — not compact output. jq 1.7 and earlier +/// let `--indent 0` imply `-c`; jq 1.8 removed that implication +/// (jqlang/jq#3232) and we match current jq. `-c` is the only compact switch. pub(super) fn render(v: &JqJson, indent: Indent) -> String { let mut out = String::new(); match indent { Indent::Compact => render_compact(v, &mut out), - Indent::Spaces(0) => render_compact(v, &mut out), Indent::Spaces(n) => render_pretty(v, &mut out, &" ".repeat(n as usize), 0), Indent::Tab => render_pretty(v, &mut out, "\t", 0), } @@ -194,11 +195,25 @@ mod tests { } #[test] - fn indent_zero_renders_compact() { - // Real jq: --indent 0 is equivalent to -c (no newlines at all). + fn indent_zero_renders_newlines_without_indent() { + // jq >= 1.8: --indent 0 keeps the newlines and drops the spaces. let v = JqJson::Object(vec![("a".into(), n("1"))]); let out = render(&v, Indent::Spaces(0)); - assert_eq!(out, "{\"a\":1}"); + assert_eq!(out, "{\n\"a\": 1\n}"); + } + + #[test] + fn indent_zero_nests_without_indent() { + let v = JqJson::Object(vec![("a".into(), JqJson::Array(vec![n("1"), n("2")]))]); + let out = render(&v, Indent::Spaces(0)); + assert_eq!(out, "{\n\"a\": [\n1,\n2\n]\n}"); + } + + #[test] + fn compact_is_the_only_compact_switch() { + // -c stays byte-for-byte compact even though --indent 0 no longer is. + let v = JqJson::Object(vec![("a".into(), JqJson::Array(vec![n("1"), n("2")]))]); + assert_eq!(render(&v, Indent::Compact), "{\"a\":[1,2]}"); } #[test] diff --git a/crates/bashkit/src/builtins/jq/mod.rs b/crates/bashkit/src/builtins/jq/mod.rs index 501cabf10..517feef81 100644 --- a/crates/bashkit/src/builtins/jq/mod.rs +++ b/crates/bashkit/src/builtins/jq/mod.rs @@ -71,7 +71,7 @@ use compat::{ }; use convert::{JqJson, jq_to_val, parse_json_stream, val_to_jq_capped}; use errors::{format_compile_errors, format_load_errors, format_runtime_error_at}; -use format::{Indent, render, sort_keys as sort_jq_keys}; +use format::{render, sort_keys as sort_jq_keys}; /// Custom DataT that holds both the LUT and a shared input iterator. /// Required by jaq 3.0 for `input`/`inputs` filter support. @@ -455,11 +455,7 @@ async fn run_jq(ctx: Context<'_>, parsed: JqArgs<'_>) -> Result { } } - let indent = if parsed.compact_output { - Indent::Compact - } else { - parsed.indent - }; + let indent = parsed.indent; let mut output = String::new(); diff --git a/crates/bashkit/src/builtins/jq/tests.rs b/crates/bashkit/src/builtins/jq/tests.rs index 1330f93f6..cef57a531 100644 --- a/crates/bashkit/src/builtins/jq/tests.rs +++ b/crates/bashkit/src/builtins/jq/tests.rs @@ -270,12 +270,45 @@ async fn indent_4_uses_4_spaces() { } #[tokio::test] -async fn indent_0_renders_compact() { - // Real jq: --indent 0 is equivalent to -c. +async fn indent_0_renders_newlines_without_indent() { + // jq >= 1.8: --indent 0 keeps the newlines and drops the spaces. Only -c + // is compact. let result = run_jq_with_args(&["--indent", "0", "."], r#"{"a":1}"#) .await .unwrap(); - assert_eq!(result.trim(), r#"{"a":1}"#); + assert_eq!(result, "{\n\"a\": 1\n}\n"); +} + +#[tokio::test] +async fn output_shape_flags_are_last_wins() { + // -c, --tab and --indent N set the same thing, so the last one wins. + let cases: [(&[&str], &str); 6] = [ + (&["-c", "--indent", "0", "."], "{\n\"a\": [\n1,\n2\n]\n}\n"), + (&["--indent", "0", "-c", "."], "{\"a\":[1,2]}\n"), + ( + &["-c", "--indent", "4", "."], + "{\n \"a\": [\n 1,\n 2\n ]\n}\n", + ), + (&["--indent", "4", "-c", "."], "{\"a\":[1,2]}\n"), + (&["--tab", "-c", "."], "{\"a\":[1,2]}\n"), + ( + &["-c", "--tab", "."], + "{\n\t\"a\": [\n\t\t1,\n\t\t2\n\t]\n}\n", + ), + ]; + for (args, expected) in cases { + let result = run_jq_with_args(args, r#"{"a":[1,2]}"#).await.unwrap(); + assert_eq!(result, expected, "args: jq {}", args.join(" ")); + } +} + +#[tokio::test] +async fn later_indent_overrides_tab() { + // --tab used to stick regardless of a later --indent N. + let result = run_jq_with_args(&["--tab", "--indent", "4", "."], r#"{"a":1}"#) + .await + .unwrap(); + assert_eq!(result, "{\n \"a\": 1\n}\n"); } #[tokio::test] @@ -1512,11 +1545,48 @@ mod differential { use super::*; fn real_jq_available() -> bool { - std::process::Command::new("jq") + real_jq_version().is_some() + } + + /// `(major, minor)` of the host `jq`, or `None` when there is no usable + /// one. `jq --version` prints `jq-1.8.1`, older builds `jq-1.7`. + fn real_jq_version() -> Option<(u32, u32)> { + let out = std::process::Command::new("jq") .arg("--version") .output() - .map(|o| o.status.success()) - .unwrap_or(false) + .ok()?; + if !out.status.success() { + return None; + } + let text = String::from_utf8_lossy(&out.stdout); + let version = text.trim().strip_prefix("jq-")?; + let mut parts = version.split('.'); + let major = parts.next()?.parse().ok()?; + // `jq-1.7` has no minor component; treat the missing one as 0. + let minor = parts.next().unwrap_or("0").parse().unwrap_or(0); + Some((major, minor)) + } + + /// Like [`assert_matches`], for a case whose expected output only holds + /// from a given jq upwards. An older host oracle skips just that case + /// rather than reddening the suite, the same bargain + /// `minimum_bash_major` strikes for the competitor corpus. + /// + /// The one case that needs it today is `--indent 0`: jq 1.7 and earlier + /// let it imply `-c`, jq 1.8 emits newlines without indentation + /// (jqlang/jq#3232). CI pins the oracle (`scripts/install-jq-ci.sh`), so + /// this only ever skips on a developer machine with an older jq. + async fn assert_matches_from_jq(min: (u32, u32), args: &[&str], input: &str) { + match real_jq_version() { + Some(found) if found >= min => assert_matches(args, input).await, + Some((major, minor)) => eprintln!( + "host jq {major}.{minor} predates {}.{}; skipping `jq {}`", + min.0, + min.1, + args.join(" ") + ), + None => eprintln!("real jq not present; skipping differential test"), + } } fn run_real_jq(args: &[&str], input: &str) -> Option<(String, i32)> { @@ -1640,7 +1710,39 @@ mod differential { #[tokio::test] async fn diff_indent_0() { - assert_matches(&["--indent", "0", "."], r#"{"a":1}"#).await; + assert_matches_from_jq((1, 8), &["--indent", "0", "."], r#"{"a":1}"#).await; + } + + #[tokio::test] + async fn diff_indent_0_nested() { + assert_matches_from_jq( + (1, 8), + &["--indent", "0", "."], + r#"{"a":[1,2],"b":{"c":3}}"#, + ) + .await; + } + + /// `-c`, `--tab` and `--indent N` all set the same thing, so the last one + /// on the command line wins. Each pair below disagrees with its reverse. + /// jq has behaved this way since at least 1.7; only what `--indent 0` + /// itself means changed in 1.8, which is why one case is gated. + #[tokio::test] + async fn diff_output_shape_flags_are_last_wins() { + let input = r#"{"a":[1,2]}"#; + for args in [ + ["-c", "--indent", "4", "."].as_slice(), + ["--indent", "4", "-c", "."].as_slice(), + ["--indent", "0", "-c", "."].as_slice(), + ["-c", "--tab", "."].as_slice(), + ["--tab", "-c", "."].as_slice(), + ["--tab", "--indent", "4", "."].as_slice(), + ["--indent", "4", "--tab", "."].as_slice(), + ["--indent", "0", "--indent", "4", "."].as_slice(), + ] { + assert_matches(args, input).await; + } + assert_matches_from_jq((1, 8), &["-c", "--indent", "0", "."], input).await; } #[tokio::test] diff --git a/examples/docs-grep-agent/uv.lock b/examples/docs-grep-agent/uv.lock index 37758769d..174b436aa 100644 --- a/examples/docs-grep-agent/uv.lock +++ b/examples/docs-grep-agent/uv.lock @@ -68,11 +68,11 @@ requires-dist = [ [[package]] name = "certifi" -version = "2026.7.22" +version = "2026.10.19" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a3/c2/24167ea9858356b47a87a50d39908bfdb72ceeefe0041586e704e5376b3a/certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55", size = 138112, upload-time = "2026-07-22T03:35:12.644Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a6/bd/1436c50cc46699de0230fbaacc1d1de7bfec413176099949e9d5112e88f2/certifi-2026.10.19.tar.gz", hash = "sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae", size = 137598, upload-time = "2026-10-20T02:24:16.604Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload-time = "2026-07-22T03:35:11.276Z" }, + { url = "https://files.pythonhosted.org/packages/c4/51/29046e48f693c54e5b7412216d447a1d8362c9653148d98c02cdf65fa733/certifi-2026.10.19-py3-none-any.whl", hash = "sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b", size = 136464, upload-time = "2026-10-20T02:24:15.072Z" }, ] [[package]] diff --git a/knowledge/operations/testing.md b/knowledge/operations/testing.md index f48f285d0..2c94bfc44 100644 --- a/knowledge/operations/testing.md +++ b/knowledge/operations/testing.md @@ -576,6 +576,53 @@ publishes one headline number, bashkit passes over real-bash passes. It is a measurement, not a gate: no floor, no CI job. See [Oils Spec Pass Rate](oils-spec.md). +## Pinned External Oracles + +A differential test that asserts byte-for-byte parity against a tool on +`PATH` is only as reproducible as that tool's version. Leave it unpinned and +the suite becomes a function of the runner image, which changes on GitHub's +schedule rather than ours. `ripgrep` (`scripts/install-ripgrep-ci.sh`, +`RG_VERSION`) and `jq` (`scripts/install-jq-ci.sh`, `JQ_VERSION`) are both +installed from a digest-pinned release in every job that runs their +differential tests — today `ci.yml`'s `test` job, `coverage.yml`, and +nightly's `asan` job. Both scripts verify a SHA-256 before the binary reaches +`PATH`, so a compromised release asset cannot become CI code execution, and +both prepend their install directory via `GITHUB_PATH` so the pin wins over +an image-preinstalled copy. + +The jq pin was added after the cost was paid: when `ubuntu-latest` began +rolling from 24.04 to 26.04 the two images shipped jq 1.7 and jq 1.8, which +disagree on `--indent 0` (1.7 implied `--compact-output`, 1.8 emits newlines +without indentation, jqlang/jq#3232). One parity case went red on whichever +half of the fleet disagreed with the implementation, with no change on our +side, and the two halves could not both be satisfied. + +**GNU coreutils is pinned by image, not by tool, and that is a stopgap.** The +`*_match_gnu` differential tests (`dd`, `join`, `od`, `pr`) require real GNU +coreutils and fail closed on Linux CI rather than skipping, which is why they +caught this. Ubuntu 26.04 installs uutils as the `coreutils` binary package, +so `dd --version` no longer says "GNU coreutils" and all four failed at once. +The jobs that run them — `ci.yml`'s `test`, `coverage.yml`'s `coverage`, +nightly's `asan` — are therefore pinned to `ubuntu-24.04`, the last image +shipping GNU coreutils as the default. That keeps the oracle at the 9.4 these +expectations were written against, but it is borrowed time: 24.04 runners +retire, and the pin stops those jobs exercising newer images. The durable fix +is to install the GNU oracle explicitly, as `rg` and `jq` are, so the image +stops deciding; it is deliberately not bundled with the unbreak-CI change +because moving the oracle's version can move the expectations with it, and +that deserves its own differential review. Note the tests already probe a +`g`-prefixed binary first (`gdd`, `god`, `gpr`) for Homebrew GNU on macOS — +`join` probes only `/opt/homebrew/bin/gjoin`, so it needs normalising before a +PATH-based GNU install would cover all four. + +Pinning the oracle in CI does not help a developer whose distro ships an +older build, so a case whose expectation only holds from some version upwards +is gated rather than asserted unconditionally: +`assert_matches_from_jq((1, 8), …)` skips just that case on an older host +oracle and prints why. This is the same bargain `minimum_bash_major` strikes +for the competitor corpus. Gate the individual case, never the whole suite — +a skipped suite is indistinguishable from a passing one. + ## SQLite CLI Differential Tests CSV comparisons explicitly set the host CLI row separator to LF. SQLite diff --git a/knowledge/runtimes/python-package.md b/knowledge/runtimes/python-package.md index 919a272e6..68880a6c0 100644 --- a/knowledge/runtimes/python-package.md +++ b/knowledge/runtimes/python-package.md @@ -398,6 +398,17 @@ per version), examples (wheel + `crates/bashkit-python/examples/` + build-wheel (maturin + twine check, produces the shared abi3 wheel artifact), python-check (branch-protection gate). +The version matrix runs on a pinned `ubuntu-24.04`, not `ubuntu-latest`, in +both `python.yml`'s `test` job and `publish-python.yml`'s `test-builds`. +Ubuntu 26.04 ships no hosted 3.9 build and `setup-python` has no fallback, so +the 3.9 leg failed outright ("The version '3.9' with architecture 'x64' was +not found for Ubuntu 26.04") as soon as `ubuntu-latest` began rolling onto +26.04. The package claims `>=3.9` through one cp39-abi3 wheel, so the matrix +needs an image that can host the whole declared range; shrinking the matrix +would stop testing a version the wheel still promises. Unpin when 3.9 support +is dropped. The two matrices are kept in step by `just check-workflow-parity`, +which compares version coverage and ignores runner labels. + ## Linting ruff (config in `crates/bashkit-python/pyproject.toml`; rules E/F/W/I/UP, diff --git a/scripts/install-jq-ci.sh b/scripts/install-jq-ci.sh new file mode 100755 index 000000000..982310f8b --- /dev/null +++ b/scripts/install-jq-ci.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash +set -euo pipefail + +# CI executes the installed `jq` as the oracle for the jq differential tests; +# pin the release binary's digest before it lands on PATH so a compromised +# release asset or redirect cannot become CI code execution. +# +# Pinning the version matters on its own here, not just the digest. The jq +# differential tests assert byte-for-byte parity against whatever `jq` is on +# PATH, so an unpinned oracle makes the suite a function of the runner image: +# jq 1.7 and 1.8 disagree on `--indent 0` (1.7 implied compact output, 1.8 +# emits newlines without indentation, jqlang/jq#3232), and when `ubuntu-latest` +# began rolling from 24.04 to 26.04 the two images shipped the two versions. +# That turned one parity case red on whichever half of the fleet disagreed with +# the implementation, with no change on our side. +version="${1:-1.8.1}" +os="$(uname -s)" +arch="$(uname -m)" + +case "${os}:${arch}" in + Linux:x86_64) + target="linux-amd64" + ;; + Linux:aarch64 | Linux:arm64) + target="linux-arm64" + ;; + Darwin:x86_64) + target="macos-amd64" + ;; + Darwin:aarch64 | Darwin:arm64) + target="macos-arm64" + ;; + *) + echo "unsupported platform for pinned jq install: ${os} ${arch}" >&2 + exit 1 + ;; +esac + +case "${version}:${target}" in + 1.8.1:linux-amd64) + expected_sha256="020468de7539ce70ef1bceaf7cde2e8c4f2ca6c3afb84642aabc5c97d9fc2a0d" + ;; + 1.8.1:linux-arm64) + expected_sha256="6bc62f25981328edd3cfcfe6fe51b073f2d7e7710d7ef7fcdac28d4e384fc3d4" + ;; + 1.8.1:macos-amd64) + expected_sha256="e80dbe0d2a2597e3c11c404f03337b981d74b4a8504b70586c354b7697a7c27f" + ;; + 1.8.1:macos-arm64) + expected_sha256="a9fe3ea2f86dfc72f6728417521ec9067b343277152b114f4e98d8cb0e263603" + ;; + *) + echo "missing pinned SHA-256 for jq ${version} ${target}" >&2 + exit 1 + ;; +esac + +sha256_file() { + local file="$1" + + if command -v sha256sum > /dev/null 2>&1; then + sha256sum "${file}" | awk '{print $1}' + elif command -v shasum > /dev/null 2>&1; then + shasum -a 256 "${file}" | awk '{print $1}' + else + echo "No SHA-256 tool found (need sha256sum or shasum)" >&2 + exit 1 + fi +} + +verify_sha256() { + local file="$1" + local expected="$2" + local actual + + actual="$(sha256_file "${file}")" + if [[ "${actual}" != "${expected}" ]]; then + echo "SHA-256 mismatch for ${file}" >&2 + echo "Expected: ${expected}" >&2 + echo "Actual: ${actual}" >&2 + exit 1 + fi +} + +binary="jq-${target}" +url="https://github.com/jqlang/jq/releases/download/jq-${version}/${binary}" +tmp="$(mktemp -d)" +trap 'rm -rf "${tmp}"' EXIT + +curl -fsSL "${url}" -o "${tmp}/jq" +verify_sha256 "${tmp}/jq" "${expected_sha256}" + +# Ahead of the image's own /usr/bin/jq on PATH: GitHub prepends each +# GITHUB_PATH entry, so the pinned build wins over a preinstalled one. +install_dir="${RUNNER_TOOL_CACHE:-${HOME}/.cache}/jq-${version}-${target}/bin" +mkdir -p "${install_dir}" +cp "${tmp}/jq" "${install_dir}/jq" +chmod +x "${install_dir}/jq" + +if [[ -n "${GITHUB_PATH:-}" ]]; then + echo "${install_dir}" >> "${GITHUB_PATH}" +fi + +"${install_dir}/jq" --version diff --git a/supply-chain/audits.toml b/supply-chain/audits.toml index a60354ff0..035ade0da 100644 --- a/supply-chain/audits.toml +++ b/supply-chain/audits.toml @@ -538,6 +538,12 @@ criteria = "safe-to-deploy" delta = "0.103.14 -> 0.103.15" notes = "Reviewed complete runtime delta: FIPS status reports underlying AWS-LC signature implementation status. Certificate parsing, verification algorithms and input bounds unchanged." +[[audits.rustls-webpki]] +who = "Mykhailo Chalyi " +criteria = "safe-to-deploy" +delta = "0.103.15 -> 0.103.16" +notes = "Reviewed complete runtime delta (the RUSTSEC-2026-0382 fix): path building now rejects any leaf, intermediate or trust anchor whose SPKI matches a static distrust list of eight emSign G3 SPKIs, surfaced as the new fail-closed Error::CertDistrusted. The distrust data is include_bytes! of DER SPKI blobs under src/data/distrusted. No unsafe code, no new dependencies, MSRV unchanged, and no change to signature verification or input bounds." + [[audits.serde_json]] who = "Mykhailo Chalyi " criteria = "safe-to-deploy" @@ -795,6 +801,12 @@ criteria = "safe-to-deploy" delta = "0.3.104 -> 0.3.105" notes = "Reviewed generated shared input slices for read/copy browser APIs, typed feature-gated OffscreenCanvas WebGL catch overloads, and deprecation metadata. No automatic browser API invocation." +[[audits.webpki-root-certs]] +who = "Mykhailo Chalyi " +criteria = "safe-to-deploy" +delta = "1.0.9 -> 1.0.10" +notes = "Reviewed complete delta (the RUSTSEC-2026-0383 fix): the bundled emSign ECC Root CA - G3 trust anchor is dropped from TLS_SERVER_ROOT_CERTS and nothing else in src/lib.rs changes. Package MSRV moves 1.70 -> 1.71, still below the pinned toolchain. Data-only change: no code, no unsafe, no new dependencies." + [[audits.wnaf]] who = "Mykhailo Chalyi " criteria = "safe-to-deploy"