From 9468882ee26238c102d59814a212d6c7f9c6be92 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Fri, 18 Sep 2026 12:19:37 -0400 Subject: [PATCH 1/3] refactor(rust): use the base64 and hex crates for the binary codecs `functions.rs` hand-rolled a standard-alphabet base64 encoder and a strict decoder (~58 lines, including a `clippy::cast_possible_truncation` opt-out for its shifts) and `cli/serialize.rs` hand-rolled hex. Both are replaced by the crates that cover them: `base64`'s `general_purpose::STANDARD` engine, which is byte-for-byte the configuration the `{"$bytes": ""}` wire format documents, and `hex::encode`. `base64` is an unconditional dependency because the function-worker path is core, not CLI; `hex` sits behind the `cli` feature. Both are zero-dependency and `base64` was already in the lockfile transitively. The colocated tests for the deleted functions go with them. The protocol tests one level up are untouched and still pass, including the malformed inputs the old decoder rejected. --- Cargo.lock | 8 ++ packages/rust/Cargo.toml | 5 + .../changelog.d/2026-09-18-codec-crates.md | 3 + packages/rust/src/cli/serialize.rs | 18 +-- packages/rust/src/functions.rs | 106 +----------------- 5 files changed, 25 insertions(+), 115 deletions(-) create mode 100644 packages/rust/changelog.d/2026-09-18-codec-crates.md diff --git a/Cargo.lock b/Cargo.lock index 8fd13602e..0aa28f386 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -556,6 +556,7 @@ version = "0.2.7" dependencies = [ "assert_cmd", "axum", + "base64 0.22.1", "blake3", "clap", "console", @@ -566,6 +567,7 @@ dependencies = [ "futures-executor", "futures-util", "globset", + "hex", "ignore", "indicatif", "libc", @@ -974,6 +976,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "http" version = "0.2.12" diff --git a/packages/rust/Cargo.toml b/packages/rust/Cargo.toml index 4d98353ce..47422d37e 100644 --- a/packages/rust/Cargo.toml +++ b/packages/rust/Cargo.toml @@ -46,6 +46,7 @@ cli = [ "dep:axum", "dep:clap", "dep:futures", + "dep:hex", "dep:reedline", "dep:tokio-stream", "tokio/rt-multi-thread", @@ -56,12 +57,16 @@ cli = [ [dependencies] axum = { version = "0.8", optional = true } +base64 = "0.22" blake3 = "1" clap = { version = "4", features = ["derive"], optional = true } console = "0.16.6" futures = { version = "0.3", optional = true } futures-channel.workspace = true globset = "0.4" +# Hex rendering of BLOB cells in the CLI's JSON output. CLI-only, so a +# library consumer never pulls it. +hex = { version = "0.4", optional = true } # Hierarchical `.gitignore` matching for path-table scans (the fd/ripgrep # semantics), without adopting its walker: the scan keeps `walkdir`. ignore = "0.4" diff --git a/packages/rust/changelog.d/2026-09-18-codec-crates.md b/packages/rust/changelog.d/2026-09-18-codec-crates.md new file mode 100644 index 000000000..d1a66a4bb --- /dev/null +++ b/packages/rust/changelog.d/2026-09-18-codec-crates.md @@ -0,0 +1,3 @@ +**Changed** + +- **The function worker's base64 codec and the CLI's hex codec are now the `base64` and `hex` crates.** The hand-rolled `base64_encode` / `base64_decode` in `functions.rs` and `hex_encode` in `cli/serialize.rs` are gone (~65 lines), along with the `clippy::cast_possible_truncation` opt-out the decoder's shifts needed. The wire format is unchanged: `{"$bytes": ""}` still uses the standard alphabet with required padding and no line wrapping, and BLOB cells still render as lowercase two-chars-per-byte hex with no separator — `base64::engine::general_purpose::STANDARD` is that configuration exactly. The decoder stays strict: a length that is not a multiple of four, a byte outside the alphabet, and padding anywhere but the close of the final chunk all still fail the query with `"$bytes" is not valid base64`. It is also strict on one input the hand-rolled decoder let through — a final character carrying non-canonical trailing bits (`"AQJ="`), which no conforming encoder emits. Costs `base64` as a dependency of the published crate (zero-dependency, `no_std`-capable, and already in the lockfile transitively, so one crate rather than a subtree); `hex` is likewise zero-dependency and sits behind the `cli` feature, so a library consumer never pulls it. (#1130) diff --git a/packages/rust/src/cli/serialize.rs b/packages/rust/src/cli/serialize.rs index 92a901f8b..c487e8c31 100644 --- a/packages/rust/src/cli/serialize.rs +++ b/packages/rust/src/cli/serialize.rs @@ -26,19 +26,10 @@ fn cell_to_json(value: &CellValue) -> Value { .map(Value::Number) .unwrap_or(Value::Null), CellValue::Text(s) => Value::String(s.clone()), - CellValue::Blob(bytes) => Value::String(hex_encode(bytes)), + CellValue::Blob(bytes) => Value::String(hex::encode(bytes)), } } -/// Hex-encode a byte slice, for `BLOB` SQLite values in JSON output. -fn hex_encode(bytes: &[u8]) -> String { - let mut out = String::with_capacity(bytes.len() * 2); - for b in bytes { - out.push_str(&format!("{b:02x}")); - } - out -} - pub(super) fn event_to_json(event: &RowEvent) -> String { event_to_value(event).to_string() } @@ -198,6 +189,8 @@ mod tests { fn blob_cell_becomes_hex_string() { let json = cell_to_json(&CellValue::Blob(vec![0xde, 0xad, 0xbe, 0xef])); assert_eq!(json.as_str(), Some("deadbeef")); + let padded = cell_to_json(&CellValue::Blob(vec![0x00, 0x0f, 0xff])); + assert_eq!(padded.as_str(), Some("000fff")); } #[test] @@ -205,9 +198,4 @@ mod tests { assert!(cell_to_json(&CellValue::Real(f64::NAN)).is_null()); assert!(cell_to_json(&CellValue::Real(f64::INFINITY)).is_null()); } - - #[test] - fn hex_encode_matches_lowercase_spec() { - assert_eq!(hex_encode(&[0x00, 0x0f, 0xff, 0xde, 0xad]), "000fffdead"); - } } diff --git a/packages/rust/src/functions.rs b/packages/rust/src/functions.rs index e0f0a237c..440a502db 100644 --- a/packages/rust/src/functions.rs +++ b/packages/rust/src/functions.rs @@ -38,6 +38,8 @@ use std::sync::mpsc::{Receiver, RecvTimeoutError, SyncSender}; use std::sync::{Arc, Mutex}; use std::time::Duration; +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD; use rusqlite::Connection; use rusqlite::functions::FunctionFlags; @@ -357,7 +359,7 @@ fn value_to_json(value: &Value) -> serde_json::Value { .map(serde_json::Value::Number) .unwrap_or(serde_json::Value::Null), Value::Text(s) => serde_json::Value::from(s.as_str()), - Value::Blob(bytes) => serde_json::json!({ "$bytes": base64_encode(bytes) }), + Value::Blob(bytes) => serde_json::json!({ "$bytes": STANDARD.encode(bytes) }), } } @@ -428,77 +430,15 @@ fn json_to_sql_value(value: &serde_json::Value) -> Result { let encoded = map["$bytes"] .as_str() .ok_or_else(|| "\"$bytes\" must be a base64 string".to_string())?; - let bytes = base64_decode(encoded) - .ok_or_else(|| "\"$bytes\" is not valid base64".to_string())?; + let bytes = STANDARD + .decode(encoded) + .map_err(|_| "\"$bytes\" is not valid base64".to_string())?; Ok(Value::Blob(bytes)) } other => Ok(Value::Text(other.to_string())), } } -const BASE64_ALPHABET: &[u8; 64] = - b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; - -/// Standard base64 with padding. Hand-rolled (~20 lines) rather than a new -/// dependency for one wire field. -fn base64_encode(bytes: &[u8]) -> String { - let mut out = String::with_capacity(bytes.len().div_ceil(3) * 4); - for chunk in bytes.chunks(3) { - let b0 = u32::from(chunk[0]); - let b1 = chunk.get(1).map(|b| u32::from(*b)); - let b2 = chunk.get(2).map(|b| u32::from(*b)); - let triple = (b0 << 16) | (b1.unwrap_or(0) << 8) | b2.unwrap_or(0); - let index = |shift: u32| BASE64_ALPHABET[(triple >> shift & 0x3f) as usize] as char; - out.push(index(18)); - out.push(index(12)); - out.push(if b1.is_some() { index(6) } else { '=' }); - out.push(if b2.is_some() { index(0) } else { '=' }); - } - out -} - -/// Decode standard base64 (padding required). `None` on any malformed input. -fn base64_decode(text: &str) -> Option> { - let bytes = text.as_bytes(); - if !bytes.len().is_multiple_of(4) { - return None; - } - let value_of = |b: u8| -> Option { - BASE64_ALPHABET - .iter() - .position(|c| *c == b) - .map(|i| u32::try_from(i).expect("index < 64")) - }; - let mut out = Vec::with_capacity(bytes.len() / 4 * 3); - for (i, chunk) in bytes.chunks(4).enumerate() { - let last = (i + 1) * 4 == bytes.len(); - let pad = chunk.iter().filter(|b| **b == b'=').count(); - // Padding may only close the final chunk, as its last one or two chars. - if pad > 0 && (!last || pad > 2 || chunk[..4 - pad].contains(&b'=')) { - return None; - } - let mut triple: u32 = 0; - for b in &chunk[..4 - pad] { - triple = (triple << 6) | value_of(*b)?; - } - triple <<= 6 * u32::try_from(pad).expect("pad <= 2"); - #[expect( - clippy::cast_possible_truncation, - reason = "each shift isolates one byte" - )] - { - out.push((triple >> 16) as u8); - if pad < 2 { - out.push((triple >> 8) as u8); - } - if pad < 1 { - out.push(triple as u8); - } - } - } - Some(out) -} - /// The production transport: the spawned worker process, its piped stdin, /// and a reader thread draining its stdout line-by-line into a channel (a /// blocking read has no timeout; `recv_timeout` on the channel does). @@ -902,40 +842,6 @@ mod tests { ); } - // --- base64 ------------------------------------------------------------ - - #[test] - fn base64_round_trips_every_padding_length() { - for bytes in [ - &b""[..], - &b"f"[..], - &b"fo"[..], - &b"foo"[..], - &b"foob"[..], - &[0u8, 255, 16, 3][..], - ] { - let encoded = base64_encode(bytes); - assert_eq!(base64_decode(&encoded).as_deref(), Some(bytes), "{encoded}"); - } - } - - #[test] - fn base64_known_vectors() { - assert_eq!(base64_encode(b"foobar"), "Zm9vYmFy"); - assert_eq!(base64_encode(b"foob"), "Zm9vYg=="); - assert_eq!(base64_encode(&[1, 2]), "AQI="); - assert_eq!(base64_decode("Zm9vYmFy").as_deref(), Some(&b"foobar"[..])); - } - - #[test] - fn base64_decode_rejects_malformed_input() { - assert_eq!(base64_decode("abc"), None); // not a multiple of 4 - assert_eq!(base64_decode("a!=="), None); // bad alphabet - assert_eq!(base64_decode("ab=c"), None); // padding inside a chunk - assert_eq!(base64_decode("ab==cd=="), None); // padding before the last chunk - assert_eq!(base64_decode("a==="), None); // over-padding - } - // --- defaults ----------------------------------------------------------- #[test] From 3b9d98d44a30dcce107b15037fa9b7ef5fcb088a Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Fri, 18 Sep 2026 12:20:36 -0400 Subject: [PATCH 2/3] e2e attestation for claude/1130-codec-crates --- .../python/e2e-attestations/claude-1130-codec-crates.json | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 packages/python/e2e-attestations/claude-1130-codec-crates.json diff --git a/packages/python/e2e-attestations/claude-1130-codec-crates.json b/packages/python/e2e-attestations/claude-1130-codec-crates.json new file mode 100644 index 000000000..b8ecb541d --- /dev/null +++ b/packages/python/e2e-attestations/claude-1130-codec-crates.json @@ -0,0 +1,7 @@ +{ + "command": "just test-e2e", + "ran_at": 1789748436, + "exit_code": 0, + "commit": "9468882ee26238c102d59814a212d6c7f9c6be92", + "branch": "claude/1130-codec-crates" +} From f36bf5c540f2162998d477fb845c8475cd713169 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Fri, 18 Sep 2026 12:21:48 -0400 Subject: [PATCH 3/3] e2e attestation for claude/1130-codec-crates --- packages/ts/e2e-attestations/claude-1130-codec-crates.json | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 packages/ts/e2e-attestations/claude-1130-codec-crates.json diff --git a/packages/ts/e2e-attestations/claude-1130-codec-crates.json b/packages/ts/e2e-attestations/claude-1130-codec-crates.json new file mode 100644 index 000000000..1762171c0 --- /dev/null +++ b/packages/ts/e2e-attestations/claude-1130-codec-crates.json @@ -0,0 +1,7 @@ +{ + "command": "pnpm test:e2e", + "ran_at": 1789748508, + "exit_code": 0, + "commit": "3b9d98d44a30dcce107b15037fa9b7ef5fcb088a", + "branch": "claude/1130-codec-crates" +}