livesync-bridge: stop restart-time vault deletion + make bulk replay resilient

services/obsidian-livesync/Dockerfile

@@ -25,13 +25,20 @@ RUN apt-get update \
 WORKDIR /app
 COPY --from=src /src /app
 COPY entrypoint.sh /usr/local/bin/livesync-bridge-entrypoint
+# Resilience patch for the vendored LiveSync core: guard the per-document getByMeta() inside the
+# watch try/catch so one corrupted/oversized doc can't stall the whole changes feed (see the script
+# header). Self-verifying — fails the build if the upstream pattern is gone.
+COPY patch-watch-resilience.ts /usr/local/lib/patch-watch-resilience.ts
 # Own /app + the Deno cache dir as uid 1000 so the runtime process (config write to /app/dat,
 # note writes to /app/data/notes) and the build-time cache all work as the vault-aligned uid.
 RUN chmod +x /usr/local/bin/livesync-bridge-entrypoint \
  && mkdir -p /app/data /app/dat /deno-dir \
  && chown -R 1000:1000 /app /deno-dir
 ENV DENO_DIR=/deno-dir
 USER 1000:1000
+# Apply the vendored-core resilience patch before caching, so the compiled/cached module is the
+# patched one. Runs as uid 1000 (owns /app after the chown above); aborts the build on drift.
+RUN deno run --allow-read --allow-write /usr/local/lib/patch-watch-resilience.ts
 # `deno install` may partial-fail (optional deps) — tolerate. `deno cache` is STRICT: a missing
 # module (e.g. an un-cloned submodule) must fail the build here, not at container start.
 RUN deno install || true

services/obsidian-livesync/entrypoint.sh

@@ -65,9 +65,53 @@ for db in _users _replicator _global_changes "${LIVESYNC_DATABASE}"; do
         "${COUCHDB_INTERNAL_URL}/${db}" >/dev/null 2>&1 || true
 done
 
+# Storage-mount readiness gate (data-loss guard). The storage peer materializes CouchDB into the
+# notes/ folder, which on Docker Desktop is a 9p bind that populates LAZILY: for the first seconds
+# after container start the folder enumerates incrementally, so any startup scan that runs against
+# it sees a PARTIAL tree and misreads the not-yet-visible files as deletions. Combined with the
+# bridge's from-scratch (in-memory) index, that manufactured a mass-delete storm that ate ~1.5k
+# vault files and re-uploaded the rest. Guard: when CouchDB is populated, do not start the sync
+# daemon until the on-disk file count has STABILIZED (unchanged across 3 consecutive 5s samples),
+# i.e. the 9p mount has finished enumerating. If it never stabilizes, exit rather than scan a
+# partial tree (under restart: unless-stopped this re-checks until the mount is fully ready).
+# Escape hatch for a deliberate fresh-device re-materialization: when an operator has
+# intentionally emptied notes/ to re-download the whole vault from CouchDB, the empty mount is
+# expected, not a not-ready 9p bind. LSB_ALLOW_EMPTY_STORAGE=1 skips the gate for that one run.
+COUCH_DOCS=$(curl -fsS -u "${COUCHDB_USER}:${COUCHDB_PASSWORD}" \
+    "${COUCHDB_INTERNAL_URL}/${LIVESYNC_DATABASE}" 2>/dev/null \
+    | grep -oE '"doc_count":[0-9]+' | cut -d: -f2)
+if [ "${LSB_ALLOW_EMPTY_STORAGE:-0}" = "1" ]; then
+    echo "livesync-bridge: LSB_ALLOW_EMPTY_STORAGE=1 - skipping the mount-readiness gate (deliberate re-materialization)."
+elif [ "${COUCH_DOCS:-0}" -gt 100 ]; then
+    prev=-1; stable=0; k=0
+    while [ "$stable" -lt 3 ]; do
+        n=$(find /app/data/notes -type f 2>/dev/null | wc -l)
+        if [ "$n" -gt 0 ] && [ "$n" -eq "$prev" ]; then
+            stable=$((stable + 1))
+        else
+            stable=0
+        fi
+        prev="$n"
+        k=$((k + 1))
+        if [ "$k" -gt 60 ]; then
+            echo "livesync-bridge: FATAL - notes/ file count not stabilizing (last=${n}, CouchDB has ${COUCH_DOCS} docs) after ~5m. 9p mount not fully ready; refusing to start to avoid a partial-scan delete storm." >&2
+            exit 1
+        fi
+        echo "livesync-bridge: waiting for notes/ 9p mount to finish populating (count=${n}, stable=${stable}/3)..."
+        sleep 5
+    done
+    echo "livesync-bridge: notes/ mount stabilized at ${prev} files; safe to start sync."
+fi
+
 # Render dat/config.json. The couchdb peer and the storage peer share group "notes", which is how
 # the bridge knows to mirror them. baseDir "" on the couchdb side = the whole LiveSync vault;
 # "data/notes/" on the storage side = /app/data/notes (the bind-mounted vault notes/ folder).
+# scanOfflineChanges is FALSE: the offline-changes reconciliation diffs a fresh in-memory index
+# against the disk and propagates the result as deletions — catastrophic against a lazily-mounted
+# 9p bind (it deleted ~1.5k files on a restart). Live sync does NOT need it: useChokidar (polling,
+# CHOKIDAR_USEPOLLING) carries host/AI edits -> CouchDB in ~3s, and device -> CouchDB -> file is
+# network-driven; only changes made while the bridge was DOWN are missed, and they re-sync on the
+# next touch — a benign staleness, never a deletion.
 # Generated secrets are base64url (JSON-safe: no " or \), so heredoc expansion can't break the JSON.
 mkdir -p /app/dat /app/data/notes
 cat > /app/dat/config.json <<JSON
@@ -89,7 +133,7 @@ cat > /app/dat/config.json <<JSON
       "name": "ordo-notes-storage",
       "group": "notes",
       "baseDir": "data/notes/",
-      "scanOfflineChanges": true,
+      "scanOfflineChanges": false,
       "useChokidar": true
     }
   ]

services/obsidian-livesync/patch-watch-resilience.ts

@@ -0,0 +1,51 @@
+// Build-time resilience patch for the vendored Self-hosted LiveSync core (the `lib/` submodule of
+// livesync-bridge, pinned by BRIDGE_REF in the Dockerfile).
+//
+// Two failure modes in DirectFileManipulatorV2.beginWatch() / followUpdates() let a single bad
+// document halt the whole CouchDB changes feed, so a bulk fetch-from-first replay (re-materializing
+// the vault CouchDB -> disk on a fresh index) silently stops partway and leaves the mirror short:
+//
+//   1. The per-document fetch runs OUTSIDE the try/catch that guards the sync callback:
+//        const docX = await this.getByMeta(doc);   // <-- unguarded
+//        try { await callback(docX, change.seq); } catch (ex) { ...log... }
+//      getByMeta() THROWS on a corrupted document ("Corrupted document: <path>"). The rejection
+//      escapes the handler and stalls the feed. (Observed: a 451 MB corrupted `.rtb`.)
+//   2. Even guarded, reassembling/decoding a very large binary is expensive enough to throw
+//      ("RangeError: string too long", a 34 MB PDF) or to monopolize the event loop.
+//
+// Fix (applied to BOTH watch handlers):
+//   - Skip documents over MAX_MATERIALIZE_BYTES *before* the expensive getByMeta() — such files
+//     cannot reliably materialize to the disk mirror anyway; they stay safe in CouchDB and on every
+//     Obsidian device, and the feed keeps going.
+//   - Move getByMeta() INSIDE the try/catch so any remaining bad document is logged and SKIPPED
+//     instead of stalling every document sequenced after it.
+//
+// Self-verifying: if the target pattern is gone (an upstream BRIDGE_REF bump changed the code),
+// the build FAILS here instead of shipping an unpatched, silently-stalling bridge.
+const target = "/app/lib/src/API/DirectFileManipulatorV2.ts";
+const MAX = 26214400; // 25 MiB — above this a doc is kept in CouchDB but not materialized to disk.
+const src = Deno.readTextFileSync(target);
+
+// Match the original (unpatched) shape in both beginWatch() and followUpdates():
+//   const docX = await this.getByMeta(doc);
+//   <p1>try {
+//   <p2>await callback(docX, change.seq);
+const pattern =
+  /const docX = await this\.getByMeta\(doc\);\n(\s*)try \{\n(\s*)await callback\(docX, change\.seq\);/g;
+const sites = (src.match(pattern) ?? []).length;
+if (sites < 1) {
+  console.error(
+    `[resilience-patch] target pattern not found in ${target} — the vendored LiveSync core changed ` +
+      `(BRIDGE_REF bump?). Re-verify beginWatch/followUpdates and update this patch.`,
+  );
+  Deno.exit(1);
+}
+
+const patched = src.replace(pattern, (_m, _p1: string, p2: string) =>
+  `try {\n` +
+  `${p2}if (((doc as any).size ?? 0) > ${MAX}) { Logger(\`WATCH: SKIP oversized (\${(doc as any).size} bytes): \${doc.path} — kept in CouchDB, not materialized to disk\`, LEVEL_INFO, "watch"); return; }\n` +
+  `${p2}const docX = await this.getByMeta(doc);\n` +
+  `${p2}await callback(docX, change.seq);`
+);
+Deno.writeTextFileSync(target, patched);
+console.log(`[resilience-patch] size-guarded + try/guarded getByMeta() at ${sites} watch site(s).`);