diff --git a/.gitattributes b/.gitattributes index 372a347..bcd0677 100644 --- a/.gitattributes +++ b/.gitattributes @@ -8,6 +8,9 @@ scripts/llamacpp/run-llama-server.sh text eol=lf # nginx config is COPYed into the codebase-memory-ui image; keep LF so nginx # doesn't choke on CRLF after a Windows checkout. services/codebase-memory-ui/nginx.conf text eol=lf +# hermes-agent patches are applied with `git apply` inside the Linux image +# build; keep LF so a Windows checkout can't corrupt the hunks. +services/hermes/*.diff text eol=lf # SOPS-encrypted files must keep LF endings or `sops` chokes parsing the # embedded timestamp metadata. The failure surfaces as diff --git a/services/hermes/Dockerfile b/services/hermes/Dockerfile index 6fa3a76..d3580c9 100644 --- a/services/hermes/Dockerfile +++ b/services/hermes/Dockerfile @@ -166,6 +166,27 @@ RUN cd /opt/hermes-agent \ && chown hermes:hermes agent/context_compressor.py tests/agent/test_context_compressor.py \ && echo "pressure-user-condense patch applied" +# PATCH: bound the checkpoint manager's pre-snapshot tree walk. +# With checkpoints enabled, any terminal command matching the destructive +# heuristic (a bare `2>/dev/null` qualifies via the `>` redirect regex) +# triggers _dir_file_count(): an unbounded rglob of the process cwd. Our +# /workspace/data is a Docker Desktop 9p bind (~50-95 stats/s), so each walk +# toward the 50k-file cap took ~9 min — and new_turn() clears the dedup set, +# so it repeated EVERY turn (2026-08-17 session ed681a: ~20 stalls x ~530s, +# confirmed by py-spy mid-stall in _dir_file_count; same pathology all of +# 2026-08-16). The scan always ended in ">50k files, skip" logged at DEBUG, +# so no checkpoint was ever produced. The patch (a) wall-clock-bounds the +# walk (HERMES_CHECKPOINT_SCAN_SECONDS, default 10s), (b) remembers the +# too-big verdict across turns, (c) logs the skip once at INFO. Upstream +# v0.20.3 is still unbounded — re-check on pin bumps. +# --check first: the build fails loudly if a future pin bump breaks the patch. +COPY checkpoint-scan-bounds.diff /tmp/checkpoint-scan-bounds.diff +RUN cd /opt/hermes-agent \ + && git apply --recount --check /tmp/checkpoint-scan-bounds.diff \ + && git apply --recount /tmp/checkpoint-scan-bounds.diff \ + && chown hermes:hermes tools/checkpoint_manager.py \ + && echo "checkpoint-scan-bounds patch applied" + USER hermes WORKDIR /opt/hermes-agent diff --git a/services/hermes/checkpoint-scan-bounds.diff b/services/hermes/checkpoint-scan-bounds.diff new file mode 100644 index 0000000..958aec9 --- /dev/null +++ b/services/hermes/checkpoint-scan-bounds.diff @@ -0,0 +1,72 @@ +diff --git a/tools/checkpoint_manager.py b/tools/checkpoint_manager.py +index df3666f..5efd444 100644 +--- a/tools/checkpoint_manager.py ++++ b/tools/checkpoint_manager.py +@@ -147,6 +147,12 @@ _GIT_TIMEOUT: int = max(10, min(60, env_int("HERMES_CHECKPOINT_TIMEOUT", 30))) + # Max files to snapshot — skip huge directories to avoid slowdowns. + _MAX_FILES = 50_000 + ++# Hard wall-clock bound on the pre-checkpoint tree walk. On slow filesystems ++# (network shares, Docker Desktop 9p/gRPC-FUSE binds) an unbounded rglob can ++# take many minutes per call; a tree we cannot even count quickly is a tree we ++# should not try to snapshot. ++_MAX_SCAN_SECONDS: int = max(1, min(120, env_int("HERMES_CHECKPOINT_SCAN_SECONDS", 10))) ++ + # Valid git commit hash pattern: 4–40 hex chars (short or full SHA-1/SHA-256). + _COMMIT_HASH_RE = re.compile(r'^[0-9a-fA-F]{4,64}$') + +@@ -639,13 +645,18 @@ def _pre_v2_shadow_repos(base: Path) -> List[Dict]: + + + def _dir_file_count(path: str) -> int: +- """Quick file count estimate (stops early if over _MAX_FILES).""" ++ """Quick file count estimate (stops early if over _MAX_FILES or too slow).""" + count = 0 ++ deadline = time.monotonic() + _MAX_SCAN_SECONDS + try: + for _ in Path(path).rglob("*"): + count += 1 + if count > _MAX_FILES: + return count ++ if count % 256 == 0 and time.monotonic() > deadline: ++ # A walk this slow means a filesystem where the snapshot itself ++ # would be even slower — treat it exactly like an oversized tree. ++ return _MAX_FILES + 1 + except (PermissionError, OSError): + pass + return count +@@ -732,6 +743,10 @@ class CheckpointManager: + self.max_total_size_mb = max(0, int(max_total_size_mb)) + self.max_file_size_mb = max(0, int(max_file_size_mb)) + self._checkpointed_dirs: Set[str] = set() ++ # Dirs whose size guard failed (too many files / unscannably slow). ++ # Deliberately NOT cleared by new_turn(): a huge tree stays huge, and ++ # re-walking it every turn cost minutes per tool call on slow mounts. ++ self._oversized_dirs: Set[str] = set() + self._git_available: Optional[bool] = None # lazy probe + + # ------------------------------------------------------------------ +@@ -772,6 +787,9 @@ class CheckpointManager: + if abs_dir in self._checkpointed_dirs: + return False + ++ if abs_dir in self._oversized_dirs: ++ return False ++ + self._checkpointed_dirs.add(abs_dir) + + try: +@@ -1008,7 +1026,12 @@ class CheckpointManager: + + # Quick size guard — don't try to snapshot enormous directories + if _dir_file_count(working_dir) > _MAX_FILES: +- logger.debug("Checkpoint skipped: >%d files in %s", _MAX_FILES, working_dir) ++ self._oversized_dirs.add(working_dir) ++ logger.info( ++ "Checkpoints disabled for %s this session: >%d files or scan " ++ "slower than %ds (tree too large to snapshot)", ++ working_dir, _MAX_FILES, _MAX_SCAN_SECONDS, ++ ) + return False + + dir_hash = _project_hash(working_dir)