Use relative pointers in block encoding

lib/encoding.js

@@ -15,8 +15,9 @@ exports.Block = Block1
 exports.encodeBlock = encodeBlock
 exports.decodeBlock = decodeBlock
 
-function encodeBlock(block) {
+function encodeBlock(block, seq) {
   if (block.type === 1) {
+    block = makeDeltasRelative(block, seq)
     return c.encode(Block1, block)
   }
 
@@ -38,7 +39,9 @@ function decodeBlock(buffer, seq) {
   state.start = 0
 
   if (type === 1) {
-    return Block1.decode(state)
+    let block = Block1.decode(state)
+    block = makeDeltasAbsolute(block, seq)
+    return block
   }
 
   if (type === 0) {
@@ -142,3 +145,55 @@ function toCohort(seq, pointers, cohorts) {
     }
   ]
 }
+
+function makeDeltasRelative(block, seq) {
+  if (block.tree) {
+    for (const tree of block.tree) {
+      // The deltas themselves are not owned by the block at this
+      // point and need to be copied on write.
+      tree.keys = tree.keys.map((d) => toRelativeDelta(seq, d))
+      tree.children = tree.children.map((d) => toRelativeDelta(seq, d))
+    }
+  }
+  if (block.cohorts) {
+    // cohorts array is owned by block (see write.js prepareCohorts)
+    // and can be mutated
+    for (let i = 0; i < block.cohorts.length; i++) {
+      // but the deltas array in each cohort is not owned by block
+      block.cohorts[i] = block.cohorts[i].map((d) => toRelativeDelta(seq, d))
+    }
+  }
+  return block
+}
+
+function makeDeltasAbsolute(block, seq) {
+  // This block is still owned by the encoding module at this point and
+  // everything is safe to mutate.
+  if (block.tree) {
+    for (const tree of block.tree) {
+      for (const d of tree.keys) toAbsoluteDelta(seq, d)
+      for (const d of tree.children) toAbsoluteDelta(seq, d)
+    }
+  }
+  if (block.cohorts) {
+    for (const cohort of block.cohorts) {
+      for (const d of cohort) toAbsoluteDelta(seq, d)
+    }
+  }
+  return block
+}
+
+function toRelativeDelta(seq, delta) {
+  if (!delta.pointer || delta.pointer.core !== 0) {
+    return delta
+  }
+  const pointer = { ...delta.pointer, seq: seq - delta.pointer.seq }
+  return { ...delta, pointer }
+}
+
+function toAbsoluteDelta(seq, delta) {
+  if (delta.pointer && delta.pointer.core === 0) {
+    delta.pointer.seq = seq - delta.pointer.seq
+  }
+  return delta
+}

lib/write.js

@@ -497,7 +497,7 @@ module.exports = class WriteBatch {
 
     for (let i = 0; i < blocks.length; i++) {
       blocks[i].checkpoint = context.checkpoint
-      buffers[i] = encodeBlock(blocks[i])
+      buffers[i] = encodeBlock(blocks[i], context.core.length + i)
     }
 
     if (this.closed) {

spec/hyperschema/index.js

@@ -36,7 +36,7 @@ const encoding0 = {
   }
 }
 
-// @bee/tree-pointer
+// @bee/relative-tree-pointer
 const encoding1 = {
   preencode(state, m) {
     state.end++ // flags are fixed size
@@ -65,7 +65,7 @@ const encoding1 = {
   }
 }
 
-// @bee/tree-pointer (inline)
+// @bee/relative-tree-pointer (inline)
 const encoding1_inline = {
   preencode(state, m) {
     if (m.core) c.uint.preencode(state, m.core)
@@ -527,7 +527,7 @@ function getEncoding(name) {
   switch (name) {
     case '@bee/tree-pointer-0':
       return encoding0
-    case '@bee/tree-pointer':
+    case '@bee/relative-tree-pointer':
       return encoding1
     case '@bee/tree-delta':
       return encoding2

spec/hyperschema/schema.json

@@ -28,7 +28,7 @@
       ]
     },
     {
-      "name": "tree-pointer",
+      "name": "relative-tree-pointer",
       "namespace": "bee",
       "compact": true,
       "flagsPosition": 0,
@@ -69,7 +69,7 @@
         {
           "name": "pointer",
           "inline": true,
-          "type": "@bee/tree-pointer",
+          "type": "@bee/relative-tree-pointer",
           "version": 1
         }
       ]

test/all.js

@@ -11,6 +11,7 @@ async function runTests() {
   await import('./diff.js')
   await import('./fuzz.js')
   await import('./undo.js')
+  await import('./compression.js')
 
   test.resume()
 }

test/compression.js

@@ -0,0 +1,83 @@
+const test = require('brittle')
+const b4a = require('b4a')
+const Bee = require('../')
+const Corestore = require('corestore')
+
+test('multi writer relative pointers', async function (t) {
+  const path = await t.tmp()
+  let store = new Corestore(path)
+  let a = new Bee(store.namespace('a'))
+  let b = new Bee(store.namespace('b'))
+
+  t.teardown(() => a.close())
+  t.teardown(() => b.close())
+  t.teardown(() => store.close())
+
+  await a.ready()
+  await b.ready()
+
+  async function reopen() {
+    await a.close()
+    await b.close()
+    await store.close()
+    store = new Corestore(path)
+    a = new Bee(store.namespace('a'))
+    b = new Bee(store.namespace('b'))
+    await a.ready()
+    await b.ready()
+  }
+
+  // Relative pointer for first entry is equivalent to absolute pointer
+  {
+    const k = b4a.from('1')
+    const w = a.write()
+
+    w.tryPut(k, k)
+    await w.flush()
+  }
+
+  // Introduce relative pointers that differ to the absolute pointers
+  {
+    const k = b4a.from('3')
+    const w = a.write()
+
+    w.tryPut(k, k)
+    await w.flush()
+  }
+  {
+    const k = b4a.from('4')
+    const w = a.write()
+
+    w.tryPut(k, k)
+    await w.flush()
+  }
+
+  // close and re-open dbs
+  await reopen()
+
+  // Build on top of remote hyperbee that has used a relative pointer
+  {
+    const k = b4a.from('2')
+    const w = b.write({ key: a.core.key, length: a.core.length })
+
+    w.tryPut(k, k)
+    await w.flush()
+  }
+
+  // close and re-open dbs
+  await reopen()
+
+  async function getKeys(hb) {
+    const keys = []
+    for await (const data of hb.createReadStream()) {
+      keys.push(data.key)
+    }
+    return keys
+  }
+
+  t.alike(await getKeys(a), [b4a.from('1'), b4a.from('3'), b4a.from('4')])
+
+  t.alike(await getKeys(b), [b4a.from('1'), b4a.from('2'), b4a.from('3'), b4a.from('4')])
+
+  t.pass('finished')
+})