From 20d8b151510af43525a4b01b8ce818c1daf1fec2 Mon Sep 17 00:00:00 2001 From: Hermes Bot Date: Tue, 11 Aug 2026 17:46:05 -0400 Subject: [PATCH 1/2] livesync-bridge: stop restart-time vault deletion (mount-race + offline-scan) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A restart of the bridge mass-deleted the materialized notes/ vault. Root cause: the storage peer ran scanOfflineChanges against the notes/ 9p bind BEFORE Docker Desktop finished populating it. The initial scan read the partially/not-yet- mounted folder as thousands of offline deletions and propagated them (once saved only by an unrelated 9p I/O error; a later restart actually removed ~1.5k files). CouchDB was never at risk of loss here (it is the authoritative superset), but the disk mirror the AI stack reads was destroyed. Two root-cause safeguards: - scanOfflineChanges=false: the offline reconciliation is the destructive path and is unnecessary for a 24/7 bridge — useChokidar (CHOKIDAR_USEPOLLING) carries host/AI edits to CouchDB and device->CouchDB->file is network-driven. Only bridge-downtime changes are missed, and they re-sync on next touch (never a delete). CouchDB->disk full materialization still happens via the couchdb peer's fetch-from-first on a fresh index, which safely rebuilds the mirror. - Stable-populated mount gate: when CouchDB is populated, refuse to start the sync daemon until the on-disk file count has stabilized (unchanged across 3x5s), i.e. the 9p mount finished enumerating — so no startup scan ever sees a partial tree. Validated: on the fixed image the gate waits for the mount to stabilize, boots with zero Unlink/delete events, and the couchdb peer's fetch-from-first replay rebuilds notes/ from CouchDB with doc_del_count held at 1. Co-Authored-By: Claude Opus 4.8 --- services/obsidian-livesync/entrypoint.sh | 41 +++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/services/obsidian-livesync/entrypoint.sh b/services/obsidian-livesync/entrypoint.sh index 0532304..2801807 100644 --- a/services/obsidian-livesync/entrypoint.sh +++ b/services/obsidian-livesync/entrypoint.sh @@ -65,9 +65,48 @@ for db in _users _replicator _global_changes "${LIVESYNC_DATABASE}"; do "${COUCHDB_INTERNAL_URL}/${db}" >/dev/null 2>&1 || true done +# Storage-mount readiness gate (data-loss guard). The storage peer materializes CouchDB into the +# notes/ folder, which on Docker Desktop is a 9p bind that populates LAZILY: for the first seconds +# after container start the folder enumerates incrementally, so any startup scan that runs against +# it sees a PARTIAL tree and misreads the not-yet-visible files as deletions. Combined with the +# bridge's from-scratch (in-memory) index, that manufactured a mass-delete storm that ate ~1.5k +# vault files and re-uploaded the rest. Guard: when CouchDB is populated, do not start the sync +# daemon until the on-disk file count has STABILIZED (unchanged across 3 consecutive 5s samples), +# i.e. the 9p mount has finished enumerating. If it never stabilizes, exit rather than scan a +# partial tree (under restart: unless-stopped this re-checks until the mount is fully ready). +COUCH_DOCS=$(curl -fsS -u "${COUCHDB_USER}:${COUCHDB_PASSWORD}" \ + "${COUCHDB_INTERNAL_URL}/${LIVESYNC_DATABASE}" 2>/dev/null \ + | grep -oE '"doc_count":[0-9]+' | cut -d: -f2) +if [ "${COUCH_DOCS:-0}" -gt 100 ]; then + prev=-1; stable=0; k=0 + while [ "$stable" -lt 3 ]; do + n=$(find /app/data/notes -type f 2>/dev/null | wc -l) + if [ "$n" -gt 0 ] && [ "$n" -eq "$prev" ]; then + stable=$((stable + 1)) + else + stable=0 + fi + prev="$n" + k=$((k + 1)) + if [ "$k" -gt 60 ]; then + echo "livesync-bridge: FATAL - notes/ file count not stabilizing (last=${n}, CouchDB has ${COUCH_DOCS} docs) after ~5m. 9p mount not fully ready; refusing to start to avoid a partial-scan delete storm." >&2 + exit 1 + fi + echo "livesync-bridge: waiting for notes/ 9p mount to finish populating (count=${n}, stable=${stable}/3)..." + sleep 5 + done + echo "livesync-bridge: notes/ mount stabilized at ${prev} files; safe to start sync." +fi + # Render dat/config.json. The couchdb peer and the storage peer share group "notes", which is how # the bridge knows to mirror them. baseDir "" on the couchdb side = the whole LiveSync vault; # "data/notes/" on the storage side = /app/data/notes (the bind-mounted vault notes/ folder). +# scanOfflineChanges is FALSE: the offline-changes reconciliation diffs a fresh in-memory index +# against the disk and propagates the result as deletions — catastrophic against a lazily-mounted +# 9p bind (it deleted ~1.5k files on a restart). Live sync does NOT need it: useChokidar (polling, +# CHOKIDAR_USEPOLLING) carries host/AI edits -> CouchDB in ~3s, and device -> CouchDB -> file is +# network-driven; only changes made while the bridge was DOWN are missed, and they re-sync on the +# next touch — a benign staleness, never a deletion. # Generated secrets are base64url (JSON-safe: no " or \), so heredoc expansion can't break the JSON. mkdir -p /app/dat /app/data/notes cat > /app/dat/config.json < /app/dat/config.json < Date: Tue, 11 Aug 2026 20:37:23 -0400 Subject: [PATCH 2/2] livesync-bridge: make bulk CouchDB->disk replay resilient + add re-materialize escape hatch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Recovering the notes/ mirror after the deletion incident surfaced a second class of bug: the bridge's bulk fetch-from-first replay (rebuilding the whole vault from CouchDB on a fresh index) would silently stall partway, leaving the mirror short. Two root causes, both in the vendored LiveSync core's watch handlers: - A per-document fetch (getByMeta) ran OUTSIDE the try/catch guarding the sync callback, so a corrupted doc ("Corrupted document", a 451 MB .rtb) or an oversized one ("RangeError: string too long", a 34 MB PDF) rejected the changes handler and stalled the feed — every document after it was skipped. patch-watch-resilience.ts (applied at build time, self-verifying — the build fails if the upstream pattern is gone) now, in both beginWatch() and followUpdates(): * skips documents over 25 MiB BEFORE the expensive getByMeta() — such files can't reliably materialize to the disk mirror anyway and stay safe in CouchDB; * moves getByMeta() INSIDE the try/catch so any remaining bad doc is logged and skipped instead of halting the feed. entrypoint.sh: LSB_ALLOW_EMPTY_STORAGE=1 escape hatch to skip the mount-readiness gate for a deliberate fresh-device re-materialization (intentionally-emptied notes/). Validated: with polling contention removed, a full CouchDB->disk rebuild now completes to the full ~5,260 files; the 451 MB and 34 MB monsters are cleanly skipped (kept in CouchDB), not stalling the sync. Co-Authored-By: Claude Opus 4.8 --- services/obsidian-livesync/Dockerfile | 7 +++ services/obsidian-livesync/entrypoint.sh | 7 ++- .../patch-watch-resilience.ts | 51 +++++++++++++++++++ 3 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 services/obsidian-livesync/patch-watch-resilience.ts diff --git a/services/obsidian-livesync/Dockerfile b/services/obsidian-livesync/Dockerfile index b7a65b4..0a5d899 100644 --- a/services/obsidian-livesync/Dockerfile +++ b/services/obsidian-livesync/Dockerfile @@ -25,6 +25,10 @@ RUN apt-get update \ WORKDIR /app COPY --from=src /src /app COPY entrypoint.sh /usr/local/bin/livesync-bridge-entrypoint +# Resilience patch for the vendored LiveSync core: guard the per-document getByMeta() inside the +# watch try/catch so one corrupted/oversized doc can't stall the whole changes feed (see the script +# header). Self-verifying — fails the build if the upstream pattern is gone. +COPY patch-watch-resilience.ts /usr/local/lib/patch-watch-resilience.ts # Own /app + the Deno cache dir as uid 1000 so the runtime process (config write to /app/dat, # note writes to /app/data/notes) and the build-time cache all work as the vault-aligned uid. RUN chmod +x /usr/local/bin/livesync-bridge-entrypoint \ @@ -32,6 +36,9 @@ RUN chmod +x /usr/local/bin/livesync-bridge-entrypoint \ && chown -R 1000:1000 /app /deno-dir ENV DENO_DIR=/deno-dir USER 1000:1000 +# Apply the vendored-core resilience patch before caching, so the compiled/cached module is the +# patched one. Runs as uid 1000 (owns /app after the chown above); aborts the build on drift. +RUN deno run --allow-read --allow-write /usr/local/lib/patch-watch-resilience.ts # `deno install` may partial-fail (optional deps) — tolerate. `deno cache` is STRICT: a missing # module (e.g. an un-cloned submodule) must fail the build here, not at container start. RUN deno install || true diff --git a/services/obsidian-livesync/entrypoint.sh b/services/obsidian-livesync/entrypoint.sh index 2801807..70148e9 100644 --- a/services/obsidian-livesync/entrypoint.sh +++ b/services/obsidian-livesync/entrypoint.sh @@ -74,10 +74,15 @@ done # daemon until the on-disk file count has STABILIZED (unchanged across 3 consecutive 5s samples), # i.e. the 9p mount has finished enumerating. If it never stabilizes, exit rather than scan a # partial tree (under restart: unless-stopped this re-checks until the mount is fully ready). +# Escape hatch for a deliberate fresh-device re-materialization: when an operator has +# intentionally emptied notes/ to re-download the whole vault from CouchDB, the empty mount is +# expected, not a not-ready 9p bind. LSB_ALLOW_EMPTY_STORAGE=1 skips the gate for that one run. COUCH_DOCS=$(curl -fsS -u "${COUCHDB_USER}:${COUCHDB_PASSWORD}" \ "${COUCHDB_INTERNAL_URL}/${LIVESYNC_DATABASE}" 2>/dev/null \ | grep -oE '"doc_count":[0-9]+' | cut -d: -f2) -if [ "${COUCH_DOCS:-0}" -gt 100 ]; then +if [ "${LSB_ALLOW_EMPTY_STORAGE:-0}" = "1" ]; then + echo "livesync-bridge: LSB_ALLOW_EMPTY_STORAGE=1 - skipping the mount-readiness gate (deliberate re-materialization)." +elif [ "${COUCH_DOCS:-0}" -gt 100 ]; then prev=-1; stable=0; k=0 while [ "$stable" -lt 3 ]; do n=$(find /app/data/notes -type f 2>/dev/null | wc -l) diff --git a/services/obsidian-livesync/patch-watch-resilience.ts b/services/obsidian-livesync/patch-watch-resilience.ts new file mode 100644 index 0000000..a053936 --- /dev/null +++ b/services/obsidian-livesync/patch-watch-resilience.ts @@ -0,0 +1,51 @@ +// Build-time resilience patch for the vendored Self-hosted LiveSync core (the `lib/` submodule of +// livesync-bridge, pinned by BRIDGE_REF in the Dockerfile). +// +// Two failure modes in DirectFileManipulatorV2.beginWatch() / followUpdates() let a single bad +// document halt the whole CouchDB changes feed, so a bulk fetch-from-first replay (re-materializing +// the vault CouchDB -> disk on a fresh index) silently stops partway and leaves the mirror short: +// +// 1. The per-document fetch runs OUTSIDE the try/catch that guards the sync callback: +// const docX = await this.getByMeta(doc); // <-- unguarded +// try { await callback(docX, change.seq); } catch (ex) { ...log... } +// getByMeta() THROWS on a corrupted document ("Corrupted document: "). The rejection +// escapes the handler and stalls the feed. (Observed: a 451 MB corrupted `.rtb`.) +// 2. Even guarded, reassembling/decoding a very large binary is expensive enough to throw +// ("RangeError: string too long", a 34 MB PDF) or to monopolize the event loop. +// +// Fix (applied to BOTH watch handlers): +// - Skip documents over MAX_MATERIALIZE_BYTES *before* the expensive getByMeta() — such files +// cannot reliably materialize to the disk mirror anyway; they stay safe in CouchDB and on every +// Obsidian device, and the feed keeps going. +// - Move getByMeta() INSIDE the try/catch so any remaining bad document is logged and SKIPPED +// instead of stalling every document sequenced after it. +// +// Self-verifying: if the target pattern is gone (an upstream BRIDGE_REF bump changed the code), +// the build FAILS here instead of shipping an unpatched, silently-stalling bridge. +const target = "/app/lib/src/API/DirectFileManipulatorV2.ts"; +const MAX = 26214400; // 25 MiB — above this a doc is kept in CouchDB but not materialized to disk. +const src = Deno.readTextFileSync(target); + +// Match the original (unpatched) shape in both beginWatch() and followUpdates(): +// const docX = await this.getByMeta(doc); +// try { +// await callback(docX, change.seq); +const pattern = + /const docX = await this\.getByMeta\(doc\);\n(\s*)try \{\n(\s*)await callback\(docX, change\.seq\);/g; +const sites = (src.match(pattern) ?? []).length; +if (sites < 1) { + console.error( + `[resilience-patch] target pattern not found in ${target} — the vendored LiveSync core changed ` + + `(BRIDGE_REF bump?). Re-verify beginWatch/followUpdates and update this patch.`, + ); + Deno.exit(1); +} + +const patched = src.replace(pattern, (_m, _p1: string, p2: string) => + `try {\n` + + `${p2}if (((doc as any).size ?? 0) > ${MAX}) { Logger(\`WATCH: SKIP oversized (\${(doc as any).size} bytes): \${doc.path} — kept in CouchDB, not materialized to disk\`, LEVEL_INFO, "watch"); return; }\n` + + `${p2}const docX = await this.getByMeta(doc);\n` + + `${p2}await callback(docX, change.seq);` +); +Deno.writeTextFileSync(target, patched); +console.log(`[resilience-patch] size-guarded + try/guarded getByMeta() at ${sites} watch site(s).`);