From 9eac25ee8fa413de390655c75ecc7349d043f977 Mon Sep 17 00:00:00 2001
From: Helen Kwok <107752876+helenkwok@users.noreply.github.com>
Date: Fri, 18 Sep 2026 09:20:03 +0930
Subject: [PATCH] convert.py: the default trunk path never wrote the Engram
 index

A DeepSeek-V4.1 conversion without --reclaim writes the 110 GB of Engram
tables and none of engram.json, so the container it reports as finished
cannot be opened at all. build_engram_meta() was called on the --reclaim
path only, and called there twice.

engram_open() was the one -2 in the load path that printed nothing, and
DS41 is the only architecture that can reach it, so the failure was
"open: malformed container" with no subject, on a container whose files
were all present and byte-exact. Name the file instead.

The CI guard checks the invariant per block rather than by counting call
sites, because the duplicated call is precisely what made the totals
match while the default path wrote nothing: two build_engram(), two
build_engram_meta(), and a broken tree. A counting guard passes on it.

Found converting deepseek-ai/DeepSeek-V4.1-Flash (475 GB staged, 320 GB
container) on an EPYC 7713; the repair needs no re-conversion, only
ds41_engram.build() run by hand against the source config's text_config.

Refs #71
---
 .github/workflows/ci.yml | 39 +++++++++++++++++++++++++++++++++++++++
 src/model.c              | 11 ++++++++++-
 tools/convert.py         |  2 +-
 3 files changed, 50 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index f97f64601..373caf4f4 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -324,6 +324,45 @@ jobs:
           fi
           echo "ok: all source files carry the header"
 
+      # convert.py has two trunk paths and only the --reclaim one called
+      # build_engram_meta(), so a default DeepSeek-V4.1 conversion wrote the
+      # 110 GB of Engram tables and none of the index that addresses them.
+      # The container then failed to open with five words and no subject,
+      # because engram_open() was also the one -2 in the load path that
+      # printed nothing (#71) — six and a half hours of conversion to find
+      # out. No test that lacks real weights can see either half, so the
+      # invariant is checked on the source instead.
+      #
+      # It is checked per BLOCK and not by counting call sites, because the
+      # bug shipped alongside a duplicated build_engram_meta() on the other
+      # path: two tables, two indexes, totals equal, and the default path
+      # still wrote nothing. A counting guard passes on the broken tree.
+      - name: every block writing Engram tables also writes the index
+        run: |
+          python3 - <<'EOF'
+          import ast, sys
+          tree = ast.parse(open("tools/convert.py").read())
+          def names(stmt):
+              return {n.func.id for n in ast.walk(stmt)
+                      if isinstance(n, ast.Call) and isinstance(n.func, ast.Name)}
+          bad = []
+          for node in ast.walk(tree):
+              for field in ("body", "orelse", "finalbody"):
+                  block = getattr(node, field, None)
+                  if not isinstance(block, list):
+                      continue
+                  called = set().union(*(names(s) for s in block)) if block else set()
+                  if "build_engram" in called and "build_engram_meta" not in called:
+                      bad.append(min(s.lineno for s in block))
+          for line in sorted(set(bad)):
+              print(f"FAIL: block at tools/convert.py:{line} calls build_engram() "
+                    f"but not build_engram_meta()")
+          if bad:
+              print("A container whose Engram tables have no engram.json cannot be opened.")
+              sys.exit(1)
+          print("ok: every block that writes Engram tables also writes the index")
+          EOF
+
   # The cross-build job proves the code compiles for Windows and the run job
   # proves those binaries execute. Neither builds *on* Windows, so neither
   # ever ran the Makefile's platform detection, the shell harness, or the
diff --git a/src/model.c b/src/model.c
index 5412e237d..3c944f0eb 100644
--- a/src/model.c
+++ b/src/model.c
@@ -1963,7 +1963,16 @@ static int engram_open(waste_model *m, const char *dir, const js_doc *d)
 
     snprintf(path, sizeof path, "%s/engram.json", dir);
     char *es = slurp(path, NULL);
-    if (!es) return -2;
+    /* Name the file. This was the one -2 in the whole load path that printed
+     * nothing, and DeepSeek-V4.1 is the only architecture that can reach it,
+     * so a container missing its Engram index failed with five words and no
+     * subject: "open: malformed container". Every other WASTE_E_FORMAT site
+     * says what it did not like. */
+    if (!es) {
+        fprintf(stderr, "waste: %s is missing — the Engram tables are there "
+                        "but the index that addresses them is not\n", path);
+        return -2;
+    }
     js_doc ed;
     if (js_parse(&ed, es) < 0) { free(es); return -2; }
 
diff --git a/tools/convert.py b/tools/convert.py
index d64eee09c..dd539155a 100644
--- a/tools/convert.py
+++ b/tools/convert.py
@@ -1619,7 +1619,6 @@ def ename(L, e, tag):
         if ds41:
             engram = build_engram(st, args.out, cfg, args.engram_bits)
             build_engram_meta(args.src, cfg, args.out)
-            build_engram_meta(args.src, cfg, args.out)
         reclaim(debt, args.reclaim, ShardDebt.TRUNK, "trunk")
 
         # A resumed conversion is holding the shards of every layer an
@@ -1989,6 +1988,7 @@ def reclaim_layer(L, size):
             return 1
         if ds41:
             engram = build_engram(st, args.out, cfg, args.engram_bits)
+            build_engram_meta(args.src, cfg, args.out)
     trunk_path = os.path.join(args.out, "trunk.bin")
     trunk_tmp = trunk_path + ".tmp"