fix(ci): merge a stacked PR against the newest commit it holds of the landed PR below it

tools/maintainer/train/arch.py

@@ -2,8 +2,9 @@
 
 renumber MB MAIN   at the PR head: the `## 7X.` sections the PR adds (relative to merge-base MB,
                    compared by heading text) take the suffixes right after MAIN's last one, and every
-                   `§7X` in lines the PR added (any file) follows it. Edits files in place; prints
-                   the mapping, or nothing when no section moves.
+                   `§7X` in lines the PR added (any file) follows it, as it follows a section MB
+                   has that MAIN gives another suffix (matched by title). Edits files in place;
+                   prints the mapping, or nothing when no section moves.
 resolve PATH       during a merge: each conflict hunk whose base side is empty (both sides appended
                    at the same place) becomes theirs, a blank line, then ours, less every `## `
                    section of ours that main's file already holds word for word (a PR stacked on
@@ -72,13 +73,26 @@ def title(h):
         new_heads = kept
     mine = sorted({suffix_ for suffix_, _ in new_heads}, key=ordinal)
     new_lines = {h for _, h in new_heads}
-    if not mine:
-        return
     last = max((ordinal(v) for v in on_main.values()), default=0)
-    mapping = {k: suffix(last + i + 1) for i, k in enumerate(mine) if k != suffix(last + i + 1)}
+    own = {k: suffix(last + i + 1) for i, k in enumerate(mine) if k != suffix(last + i + 1)}
+    # A PR stacked on one that has landed cites that PR's sections by the letters they had at the
+    # merge base; main gave them new ones when it landed. Those sections are matched by title, when
+    # the title is unique on both sides, and the PR's lines follow them too.
+    def by_title(text):
+        seen = {}
+        for m in heading.finditer(text):
+            seen.setdefault(title(m.group(0)), []).append(m.group(1))
+        return {t: s[0] for t, s in seen.items() if len(s) == 1}
+    then, now = by_title(git('show', f'{mb}:ARCHITECTURE.md')), by_title(git('show', f'{main}:ARCHITECTURE.md'))
+    moved = {s: now[t] for t, s in then.items() if t in now and now[t] != s and s not in own}
+    mapping = {**moved, **own}
     if not mapping:
         return
     pattern = re.compile(r'(§7|^## 7)(' + '|'.join(sorted(mapping, key=len, reverse=True)) + r')(?![a-z])')
+    used = set()
+    def swap(m):
+        used.add(m.group(2))
+        return m.group(1) + mapping[m.group(2)]
     for path, lines in added.items():
         # A heading line moves only if it is one of the PR's new sections; an edited heading keeps
         # its suffix even when it shares it with a new one.
@@ -87,10 +101,13 @@ def title(h):
             continue
         with open(path, encoding='utf-8') as fh:
             text = fh.read().split('\n')
-        text = [pattern.sub(lambda m: m.group(1) + mapping[m.group(2)], l) if l in hits else l for l in text]
+        text = [pattern.sub(swap, l) if l in hits else l for l in text]
         with open(path, 'w', encoding='utf-8') as fh:
             fh.write('\n'.join(text))
-    print(' '.join(f'7{k}->7{v}' for k, v in mapping.items()))
+    # Only the moves that rewrote a line are reported: an empty report means nothing changed.
+    done = [f'7{k}->7{v}' for k, v in own.items() if k in used] + [f'§7{k}->§7{v}' for k, v in moved.items() if k in used]
+    if done:
+        print(' '.join(done))
 
 
 def sections(lines):

tools/maintainer/train/refresh.sh

@@ -4,9 +4,11 @@
 #
 #   tools/maintainer/train/refresh.sh <pr>
 #
-# Merges origin/main into the PR branch (never a force push) in the train's worktree. Before the
-# merge, an ARCHITECTURE.md `## 7x.` section the PR adds whose letter main has since taken is moved
-# to the next free letter, with every §7x in lines the PR added (arch.py). Conflicts are resolved
+# Merges origin/main into the PR branch (never a force push) in the train's worktree. A PR stacked
+# on one that has since landed merges against the newest commit of that PR it holds rather than the
+# usual merge base. Before the merge, an ARCHITECTURE.md `## 7x.` section the PR adds whose letter
+# main has since taken is moved to the next free letter, with every §7x in lines the PR added, and
+# those lines follow any section main renumbered when the PR below landed (arch.py). Conflicts are resolved
 # only in generated or tabular files: tools/corpus/baseline.tsv, tools/oracle/baseline.tsv and
 # docs/spec-coverage.md take main's, and so does the coverage readme_status.py renders into
 # README.md and STATUS.md when every conflicting line is rendered text (readme3.py); the PR's CI run
@@ -36,15 +38,50 @@ git merge --abort > /dev/null 2>&1
 git reset -q --hard
 git checkout -q --detach "origin/$B" || exit 2
 base=$(git rev-parse HEAD)
-echo "$base" > "$OUT/pr-head.sha"; git merge-base origin/main HEAD > "$OUT/merge-base.sha"
-map=$(python3 -I "$TRAIN_HERE/arch.py" renumber "$(git merge-base origin/main HEAD)" origin/main 2> "$OUT/renumber.log") || { say "REFUSED: section renumber failed: $(cat "$OUT/renumber.log")"; exit 4; }
-if [ -n "$map" ]; then
+# A PR stacked on one that has since landed holds that PR's commits but not main's squash of it, and
+# the train's own commits on the landed PR (its merges of main, a renumbered section, baselines) then
+# conflict with the stacked copies of the same lines. Such a PR merges against the newest commit of
+# the landed PR it holds, the base the lanes chose by hand: main's side is then exactly what main
+# changed since, the train's commits included, and the PR's side only its own work.
+# Such a base must already hold every main commit the branch holds: when the branch merged main
+# after taking the landed PR's commits, that history would drop out of the merge and main's later
+# changes to it could be undone, so the branch merges as it always has.
+mb=$(git merge-base origin/main HEAD); mball=$(git merge-base --all origin/main HEAD); stack=""; under=""
+for n in $(git log --format=%s "$mb..origin/main" | sed -n 's/.*(#\([0-9][0-9]*\))$/\1/p'); do
+  h=$(gh pr view "$n" -R "$TRAIN_REPO" --json headRefOid --jq .headRefOid 2> /dev/null) || continue
+  git cat-file -e "$h^{commit}" 2> /dev/null || git fetch -q origin "refs/pull/$n/head" 2> /dev/null || continue
+  for c in $(git merge-base --all "$h" HEAD 2> /dev/null); do
+    git merge-base --is-ancestor "$c" origin/main && continue
+    for m in $mball; do git merge-base --is-ancestor "$m" "$c" || continue 2; done
+    if [ -z "$stack" ] || [ "$(git rev-list --count "$c..HEAD")" -lt "$(git rev-list --count "$stack..HEAD")" ]; then stack=$c; under=$n; fi
+  done
+done
+mb=${stack:-$mb}
+echo "$base" > "$OUT/pr-head.sha"; echo "$mb" > "$OUT/merge-base.sha"
+[ -n "$stack" ] && say "stacked on #$under: merging against ${stack:0:9}, the newest commit of #$under this branch holds"
+map=$(python3 -I "$TRAIN_HERE/arch.py" renumber "$mb" origin/main 2> "$OUT/renumber.log") || { say "REFUSED: section renumber failed: $(cat "$OUT/renumber.log")"; exit 4; }
+if [ -n "$map" ] && ! git diff --quiet; then
   git commit -qam "docs(architecture): renumber the new section ($map) after the sections main added" || exit 1
   say "renumbered ARCHITECTURE.md section $map"
 fi
 
 conflicted=0
-if ! git merge --no-edit -q origin/main > "$OUT/merge.log" 2>&1; then
+merge_main() {
+  [ -z "$stack" ] && { git merge --no-edit -q origin/main; return; }
+  git merge-base --is-ancestor origin/main HEAD && return 0
+  # git merge cannot take a base; merge-recursive can, and leaves the same index and conflict
+  # stages, so MERGE_HEAD makes the commit below a merge of origin/main as git merge would.
+  git rev-parse origin/main > "$(git rev-parse --git-path MERGE_HEAD)"
+  echo "Merge origin/main into $B against ${stack:0:9}, the newest commit of #$under it holds" > "$(git rev-parse --git-path MERGE_MSG)"
+  git merge-recursive "$stack" -- HEAD origin/main; rc=$?
+  [ $rc = 0 ] && { git commit -q --no-edit; return; }
+  [ $rc = 1 ] && return 1
+  rm -f "$(git rev-parse --git-path MERGE_HEAD)" "$(git rev-parse --git-path MERGE_MSG)"; git reset -q --hard
+  echo "[refresh #$P] merge-recursive against ${stack:0:9} failed (exit $rc); see $OUT/merge.log" >&3
+  exit 2
+}
+exec 3>&1
+if ! merge_main > "$OUT/merge.log" 2>&1; then
   conflicted=1
   for f in $(git diff --name-only --diff-filter=U); do
     case $f in