From 2a7973301d87bd8cc685414a34de45b18a4d6814 Mon Sep 17 00:00:00 2001 From: fxwiegand Date: Tue, 21 Jul 2026 13:00:48 +0200 Subject: [PATCH] fix: Allow for full compatibility with JS library --- README.md | 28 ++++++++++++++++- src/packer.rs | 11 +++++++ src/unpacker.rs | 17 +++++++++-- tests/test_packer.rs | 72 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 125 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6fe885a..fd9725a 100644 --- a/README.md +++ b/README.md @@ -65,4 +65,30 @@ Original library written in JS is here: https://github.com/lennartcl/jsonm let mut unpacker = Unpacker::new(); let unpacked: Value = unpacker.unpack(&packed).unwrap(); // unpacked is Object({ "bar": 1, "foo": "1" }) - + +## Interoperability with the JavaScript implementation + +By default this crate memoizes whole objects: when every key and value of an object is +already in the dictionary, the object itself is added to the dictionary and later +occurrences are packed as a single dictionary reference. The original JavaScript +implementation does not memoize whole objects, so its dictionary index runs behind and +messages containing repeated objects get corrupted when unpacked on the other side. + +To exchange messages with the JavaScript implementation, disable object memoization on +both sides: + +```rust +let mut packer = Packer::new(); +let mut options = PackOptions::new(); +options.memoize_objects = false; +let packed = packer.pack(&json!([{"a": "x"}, {"a": "x"}]), &options).unwrap(); + +let mut unpacker = Unpacker::new(); +unpacker.set_memoize_objects(false); +let unpacked: Value = unpacker.unpack(&packed).unwrap(); +``` + +With `memoize_objects` disabled the packed output is byte-compatible with the JavaScript +`jsonm` library: its unpacker can consume messages packed by this crate, and messages +packed by the JavaScript packer can be consumed by this crate's unpacker. The packer and +unpacker must use the same setting, otherwise their dictionaries go out of sync. diff --git a/src/packer.rs b/src/packer.rs index 33ad7b1..294fa6d 100644 --- a/src/packer.rs +++ b/src/packer.rs @@ -20,6 +20,7 @@ const MAX_PACK_COMPLEX_OBJECT_SIZE: usize = 12; pub struct PackOptions { pub pack_string_depth: i32, pub no_sequence_id: bool, + pub memoize_objects: bool, } #[derive(Default, Debug)] @@ -33,6 +34,7 @@ impl PackOptions { PackOptions { pack_string_depth: -1, no_sequence_id: false, + memoize_objects: true, } } } @@ -46,6 +48,7 @@ pub struct Packer { memoised_index: u64, sequence_id: i64, max_dict_size: u64, + memoize_objects: bool, } #[derive(Debug, Clone)] @@ -75,6 +78,7 @@ impl Packer { sequence_id: -1, max_dict_size: 2000, memoised_index: MIN_DICT_INDEX, + memoize_objects: true, ..Default::default() } } @@ -85,6 +89,7 @@ impl Packer { T: Serialize, { let json_object = json!(object); + self.memoize_objects = options.memoize_objects; let result = self.pack_object_or_value(&json_object, options.pack_string_depth); if options.no_sequence_id { return result; @@ -195,6 +200,7 @@ impl Packer { let options = PackOptions { no_sequence_id: true, pack_string_depth: 0, + memoize_objects: self.memoize_objects, }; return self.pack_string(obj_str, &options); } @@ -245,6 +251,7 @@ impl Packer { &PackOptions { no_sequence_id: true, pack_string_depth: -1, + memoize_objects: self.memoize_objects, }, ) { Ok(packed_string) => packed_string, @@ -263,6 +270,10 @@ impl Packer { } fn try_pack_complex_object(&mut self, object: &Value, results: Vec) -> Value { + if !self.memoize_objects { + return json!(results); + } + if results.len() > MAX_PACK_COMPLEX_OBJECT_SIZE { return json!(results); } diff --git a/src/unpacker.rs b/src/unpacker.rs index 24e2b19..4a39900 100644 --- a/src/unpacker.rs +++ b/src/unpacker.rs @@ -23,6 +23,7 @@ pub struct Unpacker { dict_index: u64, sequence_id: i64, max_dict_size: u64, + memoize_objects: bool, pending_unpacks: Vec, } @@ -53,6 +54,7 @@ impl Unpacker { sequence_id: -1, max_dict_size: 2000, dict_index: MIN_DICT_INDEX, + memoize_objects: true, ..Default::default() } } @@ -162,7 +164,9 @@ impl Unpacker { // Handle empty packed array (empty object case) if packed_array.is_empty() { let empty_obj = json!({}); - self.add_to_dict(&empty_obj.to_string()); + if self.memoize_objects { + self.add_to_dict(&empty_obj.to_string()); + } return Ok(empty_obj); } @@ -243,7 +247,10 @@ impl Unpacker { } let json_result = json!(result); - if !contains_unmemoised && packed_array.len() <= MAX_PACK_COMPLEX_OBJECT_SIZE { + if self.memoize_objects + && !contains_unmemoised + && packed_array.len() <= MAX_PACK_COMPLEX_OBJECT_SIZE + { self.add_to_dict(&json_result.to_string()); } @@ -342,4 +349,10 @@ impl Unpacker { pub fn set_max_dict_size(&mut self, value: u64) { self.max_dict_size = value; } + + /// Set whether whole objects are memoized. Must match the `memoize_objects` + /// pack option used by the packer. Default - true. + pub fn set_memoize_objects(&mut self, value: bool) { + self.memoize_objects = value; + } } diff --git a/tests/test_packer.rs b/tests/test_packer.rs index ce6dfe2..a981157 100644 --- a/tests/test_packer.rs +++ b/tests/test_packer.rs @@ -887,6 +887,78 @@ fn it_handles_null_values_with_memoization() { assert_eq!(unpacked["record3"]["name"], json!("Charlie")); } +#[test] +fn it_memoizes_repeated_objects_by_default() { + let mut packer = Packer::new(); + let mut unpacker = Unpacker::new(); + + let options = PackOptions::new(); + let input = json!([{"a": "x"}, {"a": "x"}, {"a": "y"}, {"a": "y"}]); + let packed = packer.pack(&input, &options).unwrap(); + assert_eq!( + packed, + json!([TYPE_ARRAY, ["a", "x"], [3, 4], [3, "y"], [3, 6], 0]) + ); + let unpacked: Value = unpacker.unpack(&packed).unwrap(); + assert_eq!(unpacked, input); +} + +#[test] +fn it_matches_the_js_implementation_with_object_memoization_disabled() { + let mut packer = Packer::new(); + + let mut options = PackOptions::new(); + options.memoize_objects = false; + let input = json!([{"a": "x"}, {"a": "x"}, {"a": "y"}, {"a": "y"}]); + let packed = packer.pack(&input, &options).unwrap(); + assert_eq!( + packed, + json!([TYPE_ARRAY, ["a", "x"], [3, 4], [3, "y"], [3, 5], 0]) + ); +} + +#[test] +fn it_round_trips_with_object_memoization_disabled() { + let mut packer = Packer::new(); + let mut unpacker = Unpacker::new(); + unpacker.set_memoize_objects(false); + + let mut options = PackOptions::new(); + options.memoize_objects = false; + let input = json!([{"a": "x"}, {"a": "x"}, {"a": "y"}, {"a": "y"}]); + let packed = packer.pack(&input, &options).unwrap(); + let unpacked: Value = unpacker.unpack(&packed).unwrap(); + assert_eq!(unpacked, input); + + let packed = packer.pack(&input, &options).unwrap(); + let unpacked: Value = unpacker.unpack(&packed).unwrap(); + assert_eq!(unpacked, input); +} + +#[test] +fn it_round_trips_many_repeated_objects_with_object_memoization_disabled() { + let mut packer = Packer::new(); + let mut unpacker = Unpacker::new(); + unpacker.set_memoize_objects(false); + + let mut options = PackOptions::new(); + options.memoize_objects = false; + + let mut input = Vec::new(); + for i in 0..300 { + input.push(json!({ "name": format!("name-{}", i % 10), "value": i % 7 })); + } + let input = json!(input); + + let packed = packer.pack(&input, &options).unwrap(); + let unpacked: Value = unpacker.unpack(&packed).unwrap(); + assert_eq!(unpacked, input); + + let packed = packer.pack(&input, &options).unwrap(); + let unpacked: Value = unpacker.unpack(&packed).unwrap(); + assert_eq!(unpacked, input); +} + #[test] fn it_handles_boolean_values_with_memoization() { // This test ensures that boolean values are properly memoized