fix: Allow for full compatibility with JS library

README.md

@@ -65,4 +65,30 @@ Original library written in JS is here: https://github.com/lennartcl/jsonm
 
     let mut unpacker = Unpacker::new();
     let unpacked: Value = unpacker.unpack(&packed).unwrap(); // unpacked is Object({ "bar": 1, "foo": "1" })
-
+
+## Interoperability with the JavaScript implementation
+
+By default this crate memoizes whole objects: when every key and value of an object is
+already in the dictionary, the object itself is added to the dictionary and later
+occurrences are packed as a single dictionary reference. The original JavaScript
+implementation does not memoize whole objects, so its dictionary index runs behind and
+messages containing repeated objects get corrupted when unpacked on the other side.
+
+To exchange messages with the JavaScript implementation, disable object memoization on
+both sides:
+
+```rust
+let mut packer = Packer::new();
+let mut options = PackOptions::new();
+options.memoize_objects = false;
+let packed = packer.pack(&json!([{"a": "x"}, {"a": "x"}]), &options).unwrap();
+
+let mut unpacker = Unpacker::new();
+unpacker.set_memoize_objects(false);
+let unpacked: Value = unpacker.unpack(&packed).unwrap();
+```
+
+With `memoize_objects` disabled the packed output is byte-compatible with the JavaScript
+`jsonm` library: its unpacker can consume messages packed by this crate, and messages
+packed by the JavaScript packer can be consumed by this crate's unpacker. The packer and
+unpacker must use the same setting, otherwise their dictionaries go out of sync.

src/packer.rs

@@ -20,6 +20,7 @@ const MAX_PACK_COMPLEX_OBJECT_SIZE: usize = 12;
 pub struct PackOptions {
     pub pack_string_depth: i32,
     pub no_sequence_id: bool,
+    pub memoize_objects: bool,
 }
 
 #[derive(Default, Debug)]
@@ -33,6 +34,7 @@ impl PackOptions {
         PackOptions {
             pack_string_depth: -1,
             no_sequence_id: false,
+            memoize_objects: true,
         }
     }
 }
@@ -46,6 +48,7 @@ pub struct Packer {
     memoised_index: u64,
     sequence_id: i64,
     max_dict_size: u64,
+    memoize_objects: bool,
 }
 
 #[derive(Debug, Clone)]
@@ -75,6 +78,7 @@ impl Packer {
             sequence_id: -1,
             max_dict_size: 2000,
             memoised_index: MIN_DICT_INDEX,
+            memoize_objects: true,
             ..Default::default()
         }
     }
@@ -85,6 +89,7 @@ impl Packer {
         T: Serialize,
     {
         let json_object = json!(object);
+        self.memoize_objects = options.memoize_objects;
         let result = self.pack_object_or_value(&json_object, options.pack_string_depth);
         if options.no_sequence_id {
             return result;
@@ -195,6 +200,7 @@ impl Packer {
             let options = PackOptions {
                 no_sequence_id: true,
                 pack_string_depth: 0,
+                memoize_objects: self.memoize_objects,
             };
             return self.pack_string(obj_str, &options);
         }
@@ -245,6 +251,7 @@ impl Packer {
                         &PackOptions {
                             no_sequence_id: true,
                             pack_string_depth: -1,
+                            memoize_objects: self.memoize_objects,
                         },
                     ) {
                         Ok(packed_string) => packed_string,
@@ -263,6 +270,10 @@ impl Packer {
     }
 
     fn try_pack_complex_object(&mut self, object: &Value, results: Vec<Value>) -> Value {
+        if !self.memoize_objects {
+            return json!(results);
+        }
+
         if results.len() > MAX_PACK_COMPLEX_OBJECT_SIZE {
             return json!(results);
         }

src/unpacker.rs

@@ -23,6 +23,7 @@ pub struct Unpacker {
     dict_index: u64,
     sequence_id: i64,
     max_dict_size: u64,
+    memoize_objects: bool,
     pending_unpacks: Vec<i32>,
 }
 
@@ -53,6 +54,7 @@ impl Unpacker {
             sequence_id: -1,
             max_dict_size: 2000,
             dict_index: MIN_DICT_INDEX,
+            memoize_objects: true,
             ..Default::default()
         }
     }
@@ -162,7 +164,9 @@ impl Unpacker {
         // Handle empty packed array (empty object case)
         if packed_array.is_empty() {
             let empty_obj = json!({});
-            self.add_to_dict(&empty_obj.to_string());
+            if self.memoize_objects {
+                self.add_to_dict(&empty_obj.to_string());
+            }
             return Ok(empty_obj);
         }
 
@@ -243,7 +247,10 @@ impl Unpacker {
         }
 
         let json_result = json!(result);
-        if !contains_unmemoised && packed_array.len() <= MAX_PACK_COMPLEX_OBJECT_SIZE {
+        if self.memoize_objects
+            && !contains_unmemoised
+            && packed_array.len() <= MAX_PACK_COMPLEX_OBJECT_SIZE
+        {
             self.add_to_dict(&json_result.to_string());
         }
 
@@ -342,4 +349,10 @@ impl Unpacker {
     pub fn set_max_dict_size(&mut self, value: u64) {
         self.max_dict_size = value;
     }
+
+    /// Set whether whole objects are memoized. Must match the `memoize_objects`
+    /// pack option used by the packer. Default - true.
+    pub fn set_memoize_objects(&mut self, value: bool) {
+        self.memoize_objects = value;
+    }
 }

tests/test_packer.rs

@@ -887,6 +887,78 @@ fn it_handles_null_values_with_memoization() {
     assert_eq!(unpacked["record3"]["name"], json!("Charlie"));
 }
 
+#[test]
+fn it_memoizes_repeated_objects_by_default() {
+    let mut packer = Packer::new();
+    let mut unpacker = Unpacker::new();
+
+    let options = PackOptions::new();
+    let input = json!([{"a": "x"}, {"a": "x"}, {"a": "y"}, {"a": "y"}]);
+    let packed = packer.pack(&input, &options).unwrap();
+    assert_eq!(
+        packed,
+        json!([TYPE_ARRAY, ["a", "x"], [3, 4], [3, "y"], [3, 6], 0])
+    );
+    let unpacked: Value = unpacker.unpack(&packed).unwrap();
+    assert_eq!(unpacked, input);
+}
+
+#[test]
+fn it_matches_the_js_implementation_with_object_memoization_disabled() {
+    let mut packer = Packer::new();
+
+    let mut options = PackOptions::new();
+    options.memoize_objects = false;
+    let input = json!([{"a": "x"}, {"a": "x"}, {"a": "y"}, {"a": "y"}]);
+    let packed = packer.pack(&input, &options).unwrap();
+    assert_eq!(
+        packed,
+        json!([TYPE_ARRAY, ["a", "x"], [3, 4], [3, "y"], [3, 5], 0])
+    );
+}
+
+#[test]
+fn it_round_trips_with_object_memoization_disabled() {
+    let mut packer = Packer::new();
+    let mut unpacker = Unpacker::new();
+    unpacker.set_memoize_objects(false);
+
+    let mut options = PackOptions::new();
+    options.memoize_objects = false;
+    let input = json!([{"a": "x"}, {"a": "x"}, {"a": "y"}, {"a": "y"}]);
+    let packed = packer.pack(&input, &options).unwrap();
+    let unpacked: Value = unpacker.unpack(&packed).unwrap();
+    assert_eq!(unpacked, input);
+
+    let packed = packer.pack(&input, &options).unwrap();
+    let unpacked: Value = unpacker.unpack(&packed).unwrap();
+    assert_eq!(unpacked, input);
+}
+
+#[test]
+fn it_round_trips_many_repeated_objects_with_object_memoization_disabled() {
+    let mut packer = Packer::new();
+    let mut unpacker = Unpacker::new();
+    unpacker.set_memoize_objects(false);
+
+    let mut options = PackOptions::new();
+    options.memoize_objects = false;
+
+    let mut input = Vec::new();
+    for i in 0..300 {
+        input.push(json!({ "name": format!("name-{}", i % 10), "value": i % 7 }));
+    }
+    let input = json!(input);
+
+    let packed = packer.pack(&input, &options).unwrap();
+    let unpacked: Value = unpacker.unpack(&packed).unwrap();
+    assert_eq!(unpacked, input);
+
+    let packed = packer.pack(&input, &options).unwrap();
+    let unpacked: Value = unpacker.unpack(&packed).unwrap();
+    assert_eq!(unpacked, input);
+}
+
 #[test]
 fn it_handles_boolean_values_with_memoization() {
     // This test ensures that boolean values are properly memoized