From e8948895be5ab2bdfc209a81100824f03a22502f Mon Sep 17 00:00:00 2001 From: Fernando Date: Mon, 24 Aug 2026 01:43:46 -0400 Subject: [PATCH 01/35] feat(anim): exclude non-human gaits from t2m training + posture eval tool (#837) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The curated V2 library carries 41 clips (27%) whose gait is deliberately NON-HUMAN: Half-Life/Quaternius zombies shamble with a bent spine and a dragging, splayed stride (incl. 5 walks + 1 run), and produce characters (Avacado, FruitLoop) are legless blobs whose canonical limbs are only placeholders. Flow matching samples the distribution it is given, so those clips surface on ordinary human prompts. prep-t2m-v6.py gains --exclude-sources (regex, default zombie|avacado|avocado|fruit|banana|melon|undead|ghoul), applied uniformly to all three sources — corpus, folded-in library takes, and CMU — through one excluded() closure that counts and reports the drops. Pass '' to disable. Also adds scripts/eval-t2m-posture.py: runs a t2m ONNX and reports the posture metrics that actually gate shipping (spineY / chestY / headY / worst-arm hang / foot fwd-side stride ratio) as BOTH best-of-N and mean, optionally beside the training-data reference. The v6/v6.1 numbers were tracked by hand in EVAL_NOTES.md, so there was no repeatable way to compare two models; the mean-vs-best split also makes the known walk SAMPLE VARIANCE visible (shipped v6.1 measures best-of-6 fwd/side 3.21 but mean 1.76 — which is why MotionGenerator ranks 16 candidates). Co-Authored-By: Claude Opus 5 (1M context) --- scripts/eval-t2m-posture.py | 169 ++++++++++++++++++++++++++++++++++++ scripts/prep-t2m-v6.py | 35 +++++++- 2 files changed, 202 insertions(+), 2 deletions(-) create mode 100644 scripts/eval-t2m-posture.py diff --git a/scripts/eval-t2m-posture.py b/scripts/eval-t2m-posture.py new file mode 100644 index 000000000..72154702b --- /dev/null +++ b/scripts/eval-t2m-posture.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 +# ruff: noqa: E702, E741 +"""Score a t2m ONNX model on the POSTURE metrics that gate shipping (#837). + +ONE-TIME OFFLINE dev tool — NOT shipped. + +The v6/v6.1 quality bar was tracked by hand in ~/t2m_v6/EVAL_NOTES.md. This +script recomputes those exact numbers so any two models (and the training +data itself) are directly comparable: + + spineY mean world up-component of the spine aim (v6.1 shipped: 0.99) + headY ... of the neck/head aim (v6.1 shipped: 0.90) + armY signed up-component per upper arm, WORST (v6.1 shipped: -0.98) + fwd/side foot travel ratio, locomotion only (v6.1 shipped: 2.29, + broken v6: 1.2, bar 2.0) + +Metrics are computed on the model's own canonical output (the same quantity +prep-t2m-v6.py gates the training windows on), so "model vs data" is an +apples-to-apples read. Reports best-of-N per action the way the shipped +MotionGenerator picks a candidate, plus the mean, so sample VARIANCE (the +v6 walk failure mode) is visible rather than hidden by a lucky draw. + +Usage: + python3 scripts/eval-t2m-posture.py --model ~/t2m_v62/flow/t2m.onnx \ + --vocab ~/t2m_v62/flow/t2m-vocab.json [--data ~/t2m_v62/t2m_v62.npz] \ + [--actions walk,run,jump] [--samples 16] +""" +import argparse +import importlib.util +import json +import os + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) + + +def load_module(name, fname): + spec = importlib.util.spec_from_file_location(name, os.path.join(HERE, fname)) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +prep5 = load_module("prep5", "prep-t2m-v5.py") +prep6 = load_module("prep6", "prep-t2m-v6.py") + +J = 22 +D_CANON = prep5.D_CANON +LOCOMOTION = prep6.LOCOMOTION + + +def aim_y(w, roles): + """Mean world up-component of the first VALID role's canonical aim.""" + for r in roles: + return prep6.mean_dir_y(w, r) + return float("nan") + + +def score_window(action, w): + """Posture metrics for one [T,J,4] canonical window.""" + spine = aim_y(w, (0,)) + chest = aim_y(w, (2,)) + head = aim_y(w, (4,)) + arms = [prep6.mean_dir_y(w, r) for r in (7, 11)] + out = { + "spineY": spine, + "chestY": chest, + "headY": head, + "armY_worst": max(arms), # arms hang => strongly negative; worst = least negative + "armY_L": arms[0], + "armY_R": arms[1], + } + if action in LOCOMOTION: + out["fwd_side"] = prep6.foot_travel_ratio(w) + return out + + +def quat_from_motion(motion): + """MotionGenerator output [T,220] -> canonical quats [T,J,4] (x,y,z,w).""" + m = np.asarray(motion, np.float32).reshape(motion.shape[0], J, 10) + q = m[:, :, 3:7] + return q / (np.linalg.norm(q, axis=-1, keepdims=True) + 1e-12) + + +def fmt(d): + keys = ["spineY", "chestY", "headY", "armY_worst", "fwd_side"] + return " ".join(f"{k}={d[k]:+.3f}" for k in keys if k in d) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--model", required=True) + ap.add_argument("--vocab", default="") + ap.add_argument("--data", default="", help="npz cache — also score the DATA") + ap.add_argument("--actions", default="", help="comma list; default = all vocab") + ap.add_argument("--samples", type=int, default=16) + ap.add_argument("--seed", type=int, default=0) + a = ap.parse_args() + + vocab_path = a.vocab or os.path.join(os.path.dirname(a.model), "t2m-vocab.json") + vj = json.load(open(os.path.expanduser(vocab_path))) + vocab = vj["vocab"] if isinstance(vj, dict) and "vocab" in vj else vj + if isinstance(vocab, dict): + vocab = vocab.get("actions", []) + print(f"vocab({len(vocab)}): {vocab}") + + import onnxruntime as ort + so = ort.SessionOptions() + so.log_severity_level = 3 + sess = ort.InferenceSession(os.path.expanduser(a.model), so, + providers=["CPUExecutionProvider"]) + inp = {i.name: i.shape for i in sess.get_inputs()} + print("model inputs:", inp) + tok_name = next(n for n in inp if "tok" in n.lower()) + seed_name = next((n for n in inp if "seed" in n.lower()), None) + zdim = None + if seed_name is not None: + zdim = int(inp[seed_name][-1]) + + actions = [s for s in a.actions.split(",") if s] or list(vocab) + rng = np.random.default_rng(a.seed) + + # ---- data reference (what the model is trying to match) ---- + if a.data: + z = np.load(os.path.expanduser(a.data), allow_pickle=True) + mo, tk = z["mo"], z["tk"] + dvocab = [str(s) for s in z["vocab"]] + print("\n=== TRAINING DATA (reference) ===") + for act in actions: + if act not in dvocab: + continue + idx = np.nonzero(tk[:, dvocab.index(act)])[0] + if not len(idx): + continue + pick = idx[:64] + ss = [score_window(act, mo[i]) for i in pick] + agg = {k: float(np.mean([s[k] for s in ss])) for k in ss[0]} + print(f" {act:10s} n={len(idx):6d} {fmt(agg)}") + + # ---- model ---- + print(f"\n=== MODEL ({a.samples} samples/action) ===") + print(f"{'action':10s} {'best-of-N':>44s} | {'mean':>44s}") + for act in actions: + if act not in vocab: + print(f" {act:10s} NOT IN VOCAB") + continue + t = np.zeros((1, len(vocab)), np.float32) + t[0, vocab.index(act)] = 1.0 + scores = [] + for _ in range(a.samples): + feeds = {tok_name: t} + if seed_name is not None: + feeds[seed_name] = (rng.standard_normal((1, zdim)) * 0.5).astype(np.float32) + out = sess.run(None, feeds)[0][0] + scores.append(score_window(act, quat_from_motion(out))) + # rank the way the shipped scorer does: upright + arms hanging + def rank(s): + r = s["spineY"] + s["headY"] - s["armY_worst"] + if "fwd_side" in s: + r += min(s["fwd_side"], 4.0) + return r + best = max(scores, key=rank) + mean = {k: float(np.mean([s[k] for s in scores])) for k in scores[0]} + print(f" {act:10s} {fmt(best)} | {fmt(mean)}") + + +if __name__ == "__main__": + main() diff --git a/scripts/prep-t2m-v6.py b/scripts/prep-t2m-v6.py index 85417c678..057b25af9 100644 --- a/scripts/prep-t2m-v6.py +++ b/scripts/prep-t2m-v6.py @@ -32,6 +32,7 @@ import importlib.util import json import os +import re import sys import numpy as np @@ -60,6 +61,15 @@ def load_module(name, fname): LOCOMOTION = {"walk", "run", "march"} HORIZONTAL_OK = {"death", "crawl", "roll", "swim", "fall", "sleep", "sit"} +# Source-exclusion (v6.2): some corpus characters carry a deliberately +# NON-HUMAN gait that the flow model faithfully reproduces on human prompts. +# Zombies shamble (bent spine, dragging/splayed stride, arms forward) and were +# 27% of the curated library incl. 5 walks + 1 run; "fruit"/produce characters +# (avocado etc.) are legless blobs whose canonical limbs are placeholders. +# Both poison locomotion. Matched case-insensitively against the clip's source +# string (asset title + animation name). +DEFAULT_EXCLUDE = r"zombie|avacado|avocado|fruit|banana|melon|undead|ghoul" + def qrot(q, v): qv = q[..., :3] @@ -166,8 +176,21 @@ def main(): ap.add_argument("--T", type=int, default=60) ap.add_argument("--min-roles", type=int, default=12) ap.add_argument("--min-action-windows", type=int, default=16) + ap.add_argument("--exclude-sources", default=DEFAULT_EXCLUDE, + help="regex; clips whose source matches are DROPPED " + "(default: non-human gaits — zombies, produce). " + "Pass '' to disable.") a = ap.parse_args() + excl = re.compile(a.exclude_sources, re.I) if a.exclude_sources else None + dropped_src = [0] + + def excluded(src): + if excl is None or not src or not excl.search(str(src)): + return False + dropped_src[0] += 1 + return True + T = a.T mo, msk, acts = [], [], [] gated = [0] @@ -202,8 +225,10 @@ def windows(action, cq, valid): if a.corpus: n = 0 - for action, cq, valid, _src in prep5.corpus_clips( + for action, cq, valid, src in prep5.corpus_clips( os.path.expanduser(a.corpus), a.min_roles): + if excluded(src): + continue windows(action, cq, valid) n += 1 print(f"corpus: {n} clips → {len(mo)} windows (cum)") @@ -214,6 +239,8 @@ def windows(action, cq, valid): rw, rd = c.get("restWorld"), c.get("restDir") if not rw or not rd: continue + if excluded(c.get("source", "")): + continue cq, valid = prep5.canonicalize( np.asarray(c["quats"], np.float32), rw, rd) windows(c["action"], cq, valid) @@ -221,12 +248,16 @@ def windows(action, cq, valid): print(f"library: {n} takes → {len(mo)} windows (cum)") if a.bvh and a.index: n = 0 - for action, cq, valid, _src in prep5.cmu_clips(a.bvh, a.index): + for action, cq, valid, src in prep5.cmu_clips(a.bvh, a.index): + if excluded(src): + continue windows(action, cq, valid) n += 1 print(f"cmu: {n} trials → {len(mo)} windows (cum)") print(f"quality gates dropped {gated[0]} base windows") + print(f"source exclusion dropped {dropped_src[0]} clips " + f"(pattern: {a.exclude_sources or None})") if not mo: sys.exit("no windows") from collections import Counter From a02c145c327a9c2722a6948d49a41ac4f6d0b9d4 Mon Sep 17 00:00:00 2001 From: Fernando Date: Mon, 24 Aug 2026 09:04:44 -0400 Subject: [PATCH 02/35] fix(anim): t2m training throughput collapse from per-batch MPS sync (#837) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A 400-epoch run degraded from 84s/epoch to ~16min/epoch (11x) around ep84 and parked the process in uninterruptible wait (ps stat "UN") at ~1% CPU. sample(1) put 1925 of 1981 stacks in at::native::_local_scalar_dense_mps -> at::native::mps::mps_copy_ -> MPSStream::copy_and_sync -> -[_MTLCommandBuffer waitUntilCompleted] i.e. the per-batch `tot += loss.item()`. `.item()` is a GPU->CPU sync that drains the entire MPS queue, ~97x per epoch here; over a long run the MPS allocator degrades until each drain stalls. Accumulate the epoch loss on device (`tot += loss.detach()`) and sync ONCE per epoch when printing. Note the process is both victim and cause of the memory pressure that accompanies the stall — free memory fell to ~146MB with 597k compressor pages while stalled and recovered to ~8.8GB the moment it was stopped, so "low free memory" is a symptom to look past, not the root cause. Also adds scripts/export-t2m-from-ckpt.py: the trainer only exports ONNX after its final epoch, leaving a ~9h run unobservable. This rebuilds the sampler-unrolled ONNX from any /ckpt.pt (inferring dim/layers from the checkpoint's own tensor shapes) so a run in progress can be scored with eval-t2m-posture.py at milestones — which is how the ep75 read that caught walk lagging was taken. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/export-t2m-from-ckpt.py | 107 ++++++++++++++++++++++++++++++++ scripts/train-t2m-flow-v5.py | 14 ++++- 2 files changed, 118 insertions(+), 3 deletions(-) create mode 100644 scripts/export-t2m-from-ckpt.py diff --git a/scripts/export-t2m-from-ckpt.py b/scripts/export-t2m-from-ckpt.py new file mode 100644 index 000000000..64bad3c8f --- /dev/null +++ b/scripts/export-t2m-from-ckpt.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +# ruff: noqa: E702, E741 +"""Export a t2m ONNX from a MID-RUN training checkpoint (#837). + +ONE-TIME OFFLINE dev tool — NOT shipped. + +train-t2m-flow-v5.py only exports ONNX after its LAST epoch, so a 400-epoch +(~9h) run is unobservable until it finishes. This reuses the trainer's own +FlowDiT/Sampler definitions and export block against `/ckpt.pt`, so a +run in progress can be scored with eval-t2m-posture.py at any milestone +(the v6 notes show walk quality still climbing at ep200 → ep400, which is +exactly what you want to watch rather than assume). + +Reads the arch dims from the checkpoint's own tensor shapes, so it does not +need to be told --dim/--layers. + +Usage: + python3 scripts/export-t2m-from-ckpt.py --ckpt ~/t2m_v62/flow/ckpt.pt \ + --data ~/t2m_v62/t2m_v62.npz --out ~/t2m_v62/eval_ep100 [--steps 24] +""" +import argparse +import importlib.util +import json +import os + +import numpy as np +import torch + +HERE = os.path.dirname(os.path.abspath(__file__)) + + +def load_module(name, fname): + spec = importlib.util.spec_from_file_location(name, os.path.join(HERE, fname)) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +tr = load_module("tr", "train-t2m-flow-v5.py") + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--ckpt", required=True) + ap.add_argument("--data", required=True, help="npz (for vocab + canonRestDir)") + ap.add_argument("--out", required=True) + ap.add_argument("--steps", type=int, default=24) + ap.add_argument("--guidance", type=float, default=1.0) + a = ap.parse_args() + + z = np.load(os.path.expanduser(a.data), allow_pickle=True) + vocab = [str(s) for s in z["vocab"]] + fps = int(z["fps"]) if "fps" in z else 30 + canon_rd = z["canonRestDir"] + T = int(z["mo"].shape[1]) + V = len(vocab) + + ck = torch.load(os.path.expanduser(a.ckpt), map_location="cpu", + weights_only=True) + sd = ck.get("net", ck) + epoch = ck.get("epoch", "?") + + # infer arch from tensor shapes rather than trusting flags + dim = None + for k, v in sd.items(): + if k.endswith("inp.weight") or ("inp" in k and v.ndim == 2): + dim = v.shape[0] + break + if dim is None: + dim = next(v.shape[-1] for v in sd.values() if v.ndim == 2) + layers = 1 + max( + (int(k.split(".")[1]) for k in sd if k.startswith("blocks.")), + default=0) + print(f"ckpt epoch={epoch} dim={dim} layers={layers} T={T} V={V}") + + net = tr.FlowDiT(V, T, dim=dim, layers=layers) + net.load_state_dict(sd) + net.eval() + + samp = tr.Sampler(net, V, T, a.steps, a.guidance).eval() + C6 = tr.C6 + J = tr.J + Z = T * C6 + tokens = torch.zeros(1, V); tokens[0, 0] = 1.0 + seed = torch.randn(1, Z) * 0.5 + + os.makedirs(os.path.expanduser(a.out), exist_ok=True) + onnx_path = os.path.join(os.path.expanduser(a.out), "t2m.onnx") + torch.onnx.export(samp, (tokens, seed), onnx_path, + input_names=["tokens", "seed"], + output_names=["motion"], opset_version=17, + dynamo=False) + vj = { + "vocab": vocab, "Z": Z, "T": T, "C": J * 10, "J": J, + "fps": fps, "frame": "world", "version": f"v5-flow-ep{epoch}", + "flowSteps": a.steps, + "restWorld": [[0.0, 0.0, 0.0, 1.0]] * J, + "restDir": [[float(v) for v in row] for row in canon_rd], + } + with open(os.path.join(os.path.expanduser(a.out), "t2m-vocab.json"), "w") as f: + json.dump(vj, f) + print(f"exported {onnx_path} ({os.path.getsize(onnx_path)/1e6:.1f} MB) " + f"+ vocab (epoch {epoch})") + + +if __name__ == "__main__": + main() diff --git a/scripts/train-t2m-flow-v5.py b/scripts/train-t2m-flow-v5.py index ba6c145e8..4b58d67d7 100644 --- a/scripts/train-t2m-flow-v5.py +++ b/scripts/train-t2m-flow-v5.py @@ -238,7 +238,14 @@ def main(): print(f"resumed from epoch {start_ep}", flush=True) for ep in range(start_ep, a.epochs): - tot, nb = 0.0, 0 + # Accumulate the epoch loss ON DEVICE. A per-batch `loss.item()` is a + # GPU->CPU sync (`_local_scalar_dense_mps` -> `waitUntilCompleted`) + # that drains the whole MPS queue ~97x/epoch; on long runs the MPS + # allocator degrades until each sync parks the process in + # uninterruptible wait and throughput collapses ~11x (observed at + # ep84: 84s/epoch -> ~16min/epoch). One sync per EPOCH instead. + tot = torch.zeros((), device=dev) + nb = 0 for xb, mb, tb in dl: xb, mb, tb = xb.to(dev), mb.to(dev), tb.to(dev) x0 = torch.randn_like(xb) @@ -257,8 +264,9 @@ def main(): torch.nn.utils.clip_grad_norm_(net.parameters(), 1.0) opt.step() sched.step() - tot += loss.item(); nb += 1 - print(f"ep {ep + 1}/{a.epochs} loss {tot / max(1, nb):.4f}", flush=True) + tot += loss.detach(); nb += 1 + print(f"ep {ep + 1}/{a.epochs} " + f"loss {tot.item() / max(1, nb):.4f}", flush=True) torch.save({"net": net.state_dict(), "opt": opt.state_dict(), "sched": sched.state_dict(), "epoch": ep}, ckpt_path) From 0bee3b785ed400bc7a680af4a32417f94e43defd Mon Sep 17 00:00:00 2001 From: Fernando Date: Mon, 24 Aug 2026 13:07:59 -0400 Subject: [PATCH 03/35] feat(anim): --balance-power to stop inverse-frequency sampling starving walk (#837) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pure inverse-frequency class balancing draws every action equally often, which on the v6.2 cache means the actions that matter get starved: walk 7688 windows 30.85% of data -> 4.35% of draws (0.14x) jump 3408 13.67% -> 4.35% (0.32x) confession 16 0.06% -> 4.35% (67.7x) So the model spends as much capacity memorising a 16-window curiosity as on 7688 walk windows. Measured consequence: walk foot fwd/side plateaued at ~2.6 from ep167 to ep247 (+0.04 over 40 epochs) while the training loss kept falling 0.0525 -> 0.0500 — the model was converging on everything except the action the metric cares about. For reference the shipped v6.1 measures 3.588 best-of-12 on the same metric. --balance-power tempers the exponent: 1.0 keeps the old inverse-frequency behaviour (default, so existing runs are unchanged), 0.5 is sqrt-tempered (walk 16.09%, run still 2.77%, confession 0.73%), 0.0 is the raw distribution. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/train-t2m-flow-v5.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/scripts/train-t2m-flow-v5.py b/scripts/train-t2m-flow-v5.py index 4b58d67d7..79558fa0d 100644 --- a/scripts/train-t2m-flow-v5.py +++ b/scripts/train-t2m-flow-v5.py @@ -188,6 +188,10 @@ def main(): ap.add_argument("--guidance", type=float, default=1.0, help="CFG scale baked into the exported sampler") ap.add_argument("--device", default="mps") + ap.add_argument("--balance-power", type=float, default=1.0, + help="class-balance exponent: 1.0 = inverse-frequency " + "(equal per action), 0.5 = sqrt-tempered (favours " + "the data-rich actions like walk), 0.0 = raw") ap.add_argument("--resume", action="store_true", help="resume from /ckpt.pt (long runs survive " "sleep/restarts)") @@ -209,9 +213,18 @@ def main(): m6 = torch.from_numpy(msk).repeat_interleave(D6, -1) # [N,C6] tok = torch.from_numpy(tk) - # class-balanced sampling — walk is 64% of windows (v4 lesson) + # Class-balanced sampling — walk dominates the window count (v4 lesson). + # `--balance-power` tempers it: 1.0 = pure inverse-frequency (every action + # drawn equally often), 0.5 = sqrt-tempered, 0.0 = raw distribution. + # + # Pure inverse-frequency badly starves the actions that matter most. On the + # v6.2 cache walk is 30.85% of windows but only 4.35% of draws (0.14x its + # share) while 16-window curiosities like `confession` get 67x oversampling + # — equal capacity spent memorising 16 windows as on 7688 walk windows. + # Measured effect: walk fwd/side plateaued at ~2.6 by ep167 while the loss + # kept falling. sqrt-tempering gives walk 16.1% and still leaves run 2.8%. freq = tk.sum(0) - w = (tk @ (1.0 / np.maximum(freq, 1.0))).astype(np.float64) + w = (tk @ (1.0 / np.maximum(freq, 1.0) ** a.balance_power)).astype(np.float64) sampler = torch.utils.data.WeightedRandomSampler( torch.from_numpy(w), num_samples=N, replacement=True) ds = torch.utils.data.TensorDataset(x1, m6, tok) From 405a5b8b84fa57f0facd0b099be8d084c6a56030 Mon Sep 17 00:00:00 2001 From: Fernando Date: Mon, 24 Aug 2026 17:06:20 -0400 Subject: [PATCH 04/35] feat(anim): torsoUp + ankleSpread posture metrics for t2m eval (#837) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The posture eval only measured per-joint aim directions, which are blind to two failure modes that show up in renders: - whole-body PITCH about the root: the spine can aim "up" within its own canonical frame while the entire body is rotated toward horizontal, so spineY stays ~0.97 on a body that renders near-horizontal. torsoUp FK's the hip->head axis and takes its world up-component instead, and it is weighted 3x in the best-of-N rank so a pitched draw can never win. - COLLAPSED legs: a draw whose legs fuse into one mass still scores fine on every aim metric. ankleSpread FK's both ankles (roles 17/21) and reports their mean separation. Both use the same forward kinematics the retarget uses, so they measure what actually renders. Reference values on the v6.2 walk data: torsoUp 0.998, ankleSpread 1.120. These were added while chasing an apparent "diving" render, which they then disproved: both models measure torsoUp ~0.99 and on-distribution ankleSpread, and re-rendering the same clip from a side azimuth showed it upright with separated legs. The dive was a projection artefact of viewing a +Z-facing character head-on from behind at elevation 0 — for stride verification use `--elevation 0 --start-azimuth 90`, never the default rear azimuth. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/eval-t2m-posture.py | 61 +++++++++++++++++++++++++++++++++++-- 1 file changed, 59 insertions(+), 2 deletions(-) diff --git a/scripts/eval-t2m-posture.py b/scripts/eval-t2m-posture.py index 72154702b..64b368fcd 100644 --- a/scripts/eval-t2m-posture.py +++ b/scripts/eval-t2m-posture.py @@ -57,6 +57,59 @@ def aim_y(w, roles): return float("nan") +def torso_world_up(w): + """World up-component of the HIP->HEAD axis, via the same FK the retarget + uses. This is what catches a whole-body forward PITCH about the root. + + `spineY` (the per-joint aim's up-component) does NOT: the spine can aim + "up" within its own canonical frame while the entire body is rotated + toward horizontal. A render-verified diving sample scored spineY 0.97 + yet was pitched near-horizontal, which is why this term exists. + + Returns the mean over frames of dot(normalize(head - hip), +Y). + 1.0 = perfectly upright, 0.0 = horizontal, <0 = inverted/head-down. + """ + T = len(w) + PAR = prep6.PAR + + def pos(role): + p = np.zeros((T, 3), np.float32) + r = role + while PAR[r] >= 0: + p = p + prep6.qrot(w[:, PAR[r]], np.broadcast_to(D_CANON[r], (T, 3))) + r = PAR[r] + return p + + axis = pos(5) - pos(0) # head - hip + n = np.linalg.norm(axis, axis=-1, keepdims=True) + axis = axis / (n + 1e-12) + return float(axis[:, 1].mean()) + + +def ankle_spread(w): + """Mean world distance between the two ankles (roles 17/21), in unit bone + lengths, over the window. + + Catches the COLLAPSED-LEGS failure that the aim-based metrics miss: a + render-verified bad draw showed both legs fused into one tapering mass + while torsoUp/spineY/fwd_side all still looked healthy. A real stride + separates the ankles for most of the cycle. + """ + T = len(w) + PAR = prep6.PAR + + def pos(role): + p = np.zeros((T, 3), np.float32) + r = role + while PAR[r] >= 0: + p = p + prep6.qrot(w[:, PAR[r]], np.broadcast_to(D_CANON[r], (T, 3))) + r = PAR[r] + return p + + d = np.linalg.norm(pos(17) - pos(21), axis=-1) + return float(d.mean()) + + def score_window(action, w): """Posture metrics for one [T,J,4] canonical window.""" spine = aim_y(w, (0,)) @@ -64,6 +117,8 @@ def score_window(action, w): head = aim_y(w, (4,)) arms = [prep6.mean_dir_y(w, r) for r in (7, 11)] out = { + "torsoUp": torso_world_up(w), + "ankleSpread": ankle_spread(w), "spineY": spine, "chestY": chest, "headY": head, @@ -84,7 +139,8 @@ def quat_from_motion(motion): def fmt(d): - keys = ["spineY", "chestY", "headY", "armY_worst", "fwd_side"] + keys = ["torsoUp", "ankleSpread", "spineY", "headY", "armY_worst", + "fwd_side"] return " ".join(f"{k}={d[k]:+.3f}" for k in keys if k in d) @@ -156,7 +212,8 @@ def main(): scores.append(score_window(act, quat_from_motion(out))) # rank the way the shipped scorer does: upright + arms hanging def rank(s): - r = s["spineY"] + s["headY"] - s["armY_worst"] + # torsoUp dominates: a pitched-forward body must never win best-of-N + r = 3.0 * s["torsoUp"] + s["spineY"] + s["headY"] - s["armY_worst"] if "fwd_side" in s: r += min(s["fwd_side"], 4.0) return r From 877fd3a4f85088f87f82f2e3158e9d303d77c2b3 Mon Sep 17 00:00:00 2001 From: Fernando Date: Mon, 24 Aug 2026 18:33:17 -0400 Subject: [PATCH 05/35] feat(anim): contralateral gait-phase loss for t2m orientation correctness (#837) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit User report: generated clips render with the character facing the camera but the limbs moving as if it faced away — on BOTH Mixamo and UniRig skeletons. Measured cause. A correct walk pairs the LEFT leg forward with the RIGHT arm forward, so corr(Lankle_z, Rwrist_z) > 0. On the v6.2 cache: training data +0.625 contralateral / -0.578 ipsilateral, 92% correct sign v6.2-sqrt ep400 +0.324 / -0.363, only 75% correct v6.1 shipped +0.212 / -0.207, only 67% correct So the data has clean gait but both models degrade it: ~1 in 4 draws swings the arms on the wrong side, which is exactly what "moving as if facing backwards" looks like. Nothing in the flow-matching objective constrained the pairing. --phase-weight adds a hinge on that correlation, applied to LOCOMOTION rows only (--phase-actions, default walk,run,march) and computed on the clean sample reconstructed from the predicted velocity (x1 = x_t + (1-t)v), through differentiable canonical FK that mirrors prep-t2m-v6/eval-t2m-posture exactly — so the supervised quantity is the one the metrics and the retarget read. The term averages both body sides, so it cannot prefer one chirality. Default 0 = off, existing runs unchanged. Validated before use: the term scores the real data +0.51 walk / +0.61 run / +0.43 march and ~0.000 on `sit` (correctly neutral for non-locomotion), and backprops finite gradients through d6_to_quat. Two other candidate fixes were investigated and REJECTED on measurement, not opinion: - mirror augmentation is NOT the culprit: mirrored windows preserve phase (+0.664 -> +0.639, sign flips in 2%). - flipping the D_CANON foot direction to -Z (making toe-forward agree with the shoulder-derived -Z forward) helps walk (89->94% correct) but hurts run (98->97%) and badly hurts march (96->72%); an initial 93->100% reading was a 60-window small-sample artefact. D_CANON is self-consistent as a convention since the model output and restDir come from the same table. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/train-t2m-flow-v5.py | 108 +++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/scripts/train-t2m-flow-v5.py b/scripts/train-t2m-flow-v5.py index 79558fa0d..2454589f2 100644 --- a/scripts/train-t2m-flow-v5.py +++ b/scripts/train-t2m-flow-v5.py @@ -93,6 +93,79 @@ def d6_to_quat(d): return F.normalize(q, dim=-1, eps=1e-6) +# ---------- gait-phase supervision (#837 orientation correctness) ---------- +# Canonical parent chain + rest directions, mirroring prep-t2m-v5/v6 exactly. +PAR_CANON = [-1, 0, 1, 2, 3, 4, 2, 6, 7, 8, 2, 10, 11, 12, + 0, 14, 15, 16, 0, 18, 19, 20] +DIR_CANON = [ + [0, 1, 0], [0, 1, 0], [0, 1, 0], [0, 1, 0], [0, 1, 0], [0, 1, 0], + [-1, 0, 0], [-1, 0, 0], [-1, 0, 0], [-1, 0, 0], + [1, 0, 0], [1, 0, 0], [1, 0, 0], [1, 0, 0], + [0, -1, 0], [0, -1, 0], [0, -1, 0], [0, 0, 1], + [0, -1, 0], [0, -1, 0], [0, -1, 0], [0, 0, 1], +] +# roles whose Z trajectory defines the gait phase +L_ANKLE, R_ANKLE, L_WRIST, R_WRIST = 17, 21, 9, 13 + + +def qrot_t(q, v): + """Rotate v[...,3] by quat q[...,4] (x,y,z,w). Differentiable.""" + qv, qw = q[..., :3], q[..., 3:] + uv = torch.cross(qv, v, dim=-1) + uuv = torch.cross(qv, uv, dim=-1) + return v + 2.0 * (qw * uv + uuv) + + +def fk_pos(q, role, dirs): + """World position of `role` by walking the canonical parent chain. + + q: [B,T,J,4]. Mirrors the numpy FK in prep-t2m-v6/eval-t2m-posture, so the + quantity supervised here is the same one the metrics and the retarget read. + """ + p = torch.zeros(q.shape[0], q.shape[1], 3, device=q.device, dtype=q.dtype) + r = role + while PAR_CANON[r] >= 0: + d = dirs[r].expand(q.shape[0], q.shape[1], 3) + p = p + qrot_t(q[:, :, PAR_CANON[r]], d) + r = PAR_CANON[r] + return p + + +def _centred_z(p): + z = p[..., 2] + return z - z.mean(dim=1, keepdim=True) + + +def gait_phase_corr(q, dirs): + """Per-sample contralateral gait correlation, in [-1, 1]. + + A correct walk pairs LEFT leg forward with RIGHT arm forward, so + corr(Lankle_z, Rwrist_z) > 0 and corr(Lankle_z, Lwrist_z) < 0. The + training data measures +0.63 / -0.58 (92% correct sign) but the v6.1 and + v6.2 models emit only 67% / 75% correct — samples with the arms swinging + on the wrong side, which renders as "limbs moving as if facing backwards" + (user-reported). Nothing in the flow-matching loss constrained this, so + this term supervises it directly. + + Returns the mean of (contralateral - ipsilateral)/2, averaged over both + body sides so the term is itself mirror-symmetric. + """ + la = _centred_z(fk_pos(q, L_ANKLE, dirs)) + ra = _centred_z(fk_pos(q, R_ANKLE, dirs)) + lw = _centred_z(fk_pos(q, L_WRIST, dirs)) + rw = _centred_z(fk_pos(q, R_WRIST, dirs)) + + def corr(x, y): + xs = x / (x.std(dim=1, keepdim=True) + 1e-4) + ys = y / (y.std(dim=1, keepdim=True) + 1e-4) + return (xs * ys).mean(dim=1) + + # both sides, so the objective cannot prefer one chirality + contra = 0.5 * (corr(la, rw) + corr(ra, lw)) + ipsi = 0.5 * (corr(la, lw) + corr(ra, rw)) + return 0.5 * (contra - ipsi) + + # ---------- model ---------- class Block(nn.Module): def __init__(self, dim, heads): @@ -192,6 +265,13 @@ def main(): help="class-balance exponent: 1.0 = inverse-frequency " "(equal per action), 0.5 = sqrt-tempered (favours " "the data-rich actions like walk), 0.0 = raw") + ap.add_argument("--phase-weight", type=float, default=0.0, + help="weight of the contralateral gait-phase loss " + "(#837 orientation correctness). 0 = off. Applied to " + "LOCOMOTION actions only, on the reconstructed clean " + "sample. 0.05-0.2 is a sane range.") + ap.add_argument("--phase-actions", default="walk,run,march", + help="comma list the phase loss applies to") ap.add_argument("--resume", action="store_true", help="resume from /ckpt.pt (long runs survive " "sleep/restarts)") @@ -213,6 +293,18 @@ def main(): m6 = torch.from_numpy(msk).repeat_interleave(D6, -1) # [N,C6] tok = torch.from_numpy(tk) + # gait-phase loss plumbing (#837): a per-action gate + the canonical rest + # directions as a device tensor for the differentiable FK. + dirs_t = torch.tensor(DIR_CANON, dtype=torch.float32, device=dev) + phase_acts = {w for w in a.phase_actions.split(",") if w} + loco_idx = [i for i, w in enumerate(vocab) if w in phase_acts] + loco_mask_v = torch.zeros(V, device=dev) + for i in loco_idx: + loco_mask_v[i] = 1.0 + if a.phase_weight > 0: + print(f"gait-phase loss ON (w={a.phase_weight}) for " + f"{[vocab[i] for i in loco_idx]}", flush=True) + # Class-balanced sampling — walk dominates the window count (v4 lesson). # `--balance-power` tempers it: 1.0 = pure inverse-frequency (every action # drawn equally often), 0.5 = sqrt-tempered, 0.0 = raw distribution. @@ -264,6 +356,7 @@ def main(): x0 = torch.randn_like(xb) # classifier-free guidance: drop the action condition 10% of the # time so the sampler can extrapolate cond vs uncond at export. + tb_raw = tb drop = (torch.rand(tb.shape[0], device=dev) < 0.1).float() tb = tb * (1.0 - drop)[:, None] t = torch.rand(xb.shape[0], device=dev) @@ -272,6 +365,21 @@ def main(): tgt = xb - x0 mask = mb[:, None, :] # [B,1,C6] loss = ((v - tgt) ** 2 * mask).sum() / mask.sum() / T + if a.phase_weight > 0: + # Flow matching predicts a VELOCITY, so reconstruct the clean + # sample the model implies at this t before measuring gait: + # x_t = (1-t)x0 + t*x1 and v* = x1 - x0 => x1 = x_t + (1-t)v + x1_hat = xt + (1.0 - t[:, None, None]) * v + q_hat = d6_to_quat(x1_hat.reshape(-1, T, J, D6)) + ph = gait_phase_corr(q_hat, dirs_t) # [B], want -> +1 + # gate to locomotion rows only (tb is the CFG-dropped token, so + # use the pre-drop labels for the gate) + g = (tb_raw * loco_mask_v).sum(-1).clamp(0, 1) + denom = g.sum().clamp_min(1.0) + # hinge: only penalise below a firm-but-not-saturating target, + # so well-phased samples are left alone + loss = loss + a.phase_weight * ( + (g * (0.6 - ph).clamp_min(0.0)).sum() / denom) opt.zero_grad(set_to_none=True) loss.backward() torch.nn.utils.clip_grad_norm_(net.parameters(), 1.0) From d11f17d68020ae8c845225282a85274e1ed0d7eb Mon Sep 17 00:00:00 2001 From: Fernando Date: Mon, 24 Aug 2026 20:08:08 -0400 Subject: [PATCH 06/35] =?UTF-8?q?fix(anim):=20canonical=20chirality=20was?= =?UTF-8?q?=20inverted=20=E2=80=94=20t2m=20locomotion=20ran=20backwards=20?= =?UTF-8?q?(#837)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit User report, after the gait-phase loss landed: "run and march are still backwards, but good — they are just moving backwards", and "the walk got all twisted". Two separate bugs, both measured. 1. D_CANON had the LEFT arm chain (roles 6-9) on -X and the RIGHT (10-13) on +X — inverted. Both the file's own docstring ("left arm +X, right arm -X") and AnimationMerger.cpp ("canonical axes X=left, Y=up, Z=forward" / "canonical left joints expect +X") specify left = +X, so the table contradicted its own spec in two places. Consequence: every clip's shoulder-derived forward pointed -Z while the runtime treats clips as +Z-facing, so detectBackwardFacing compensated with a whole-clip 180 yaw. That fixed the BODY's facing but not the limbs, which is exactly the reported symptom. Stance-foot travel vs the body's own forward, on the v6.2 cache: before walk -0.209 (40% forward) run -0.785 (7%) march -0.334 (32%) after walk +0.209 (60% forward) run +0.785 (93%) march +0.334 (68%) An exact sign inversion — the signature of a swapped-chirality frame. The trainer's DIR_CANON copy is synced and asserted equal. 2. The phase loss found a DEGENERATE shortcut: rather than fix limb timing it rotated the torso ~180 deg so the existing swing reads as contralateral. Measured hip->chest yaw at --phase-weight 0.15: 170 deg, vs 6.3 deg in the data and 6.7 deg in v6.2 — the visible "twisted walk". --twist-weight adds a hinge on that yaw (free below ~20 deg, linear above), closing the escape route so the phase reward can only be earned by real timing. Validated: zero penalty on real data (3.2 deg), 2.74 on a synthetic 180-deg twist. Both fixes change the canonical frame, so the cache must be rebuilt. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/prep-t2m-v5.py | 14 +++++++++++- scripts/train-t2m-flow-v5.py | 44 +++++++++++++++++++++++++++++++++--- 2 files changed, 54 insertions(+), 4 deletions(-) diff --git a/scripts/prep-t2m-v5.py b/scripts/prep-t2m-v5.py index e7a387f25..8243e9da5 100644 --- a/scripts/prep-t2m-v5.py +++ b/scripts/prep-t2m-v5.py @@ -79,10 +79,22 @@ def load_module(name, fname): # FIXED canonical T-pose bone directions D (canonical axes: +Y up, +Z fwd, # +X left — the CMU frame convention every dump is normalized into). +# CHIRALITY FIX (#837): roles 6-9 are the LEFT arm chain and 10-13 the RIGHT, +# and the documented convention above (and AnimationMerger.cpp: "canonical axes +# X=left, Y=up, Z=forward" / "canonical left joints expect +X") puts left on +X. +# The table previously had left on -X and right on +X — inverted. That made every +# clip's shoulder-derived forward point -Z while the runtime treats clips as +# +Z-facing, so `detectBackwardFacing` compensated with a whole-clip 180 yaw: +# the BODY then faced the right way but the LIMBS still moved backwards +# (user-reported: "faces the camera, limbs move as if facing back"). +# Measured on the v6.2 cache, stance-foot travel vs the body's own forward: +# before walk -0.209 (40% forward) run -0.785 (7%) march -0.334 (32%) +# after walk +0.209 (60% forward) run +0.785 (93%) march +0.334 (68%) +# An exact sign inversion — the signature of a swapped-chirality frame. D_CANON = np.array([ [0, 1, 0], [0, 1, 0], [0, 1, 0], [0, 1, 0], [0, 1, 0], [0, 1, 0], - [-1, 0, 0], [-1, 0, 0], [-1, 0, 0], [-1, 0, 0], [1, 0, 0], [1, 0, 0], [1, 0, 0], [1, 0, 0], + [-1, 0, 0], [-1, 0, 0], [-1, 0, 0], [-1, 0, 0], [0, -1, 0], [0, -1, 0], [0, -1, 0], [0, 0, 1], [0, -1, 0], [0, -1, 0], [0, -1, 0], [0, 0, 1], ], np.float32) diff --git a/scripts/train-t2m-flow-v5.py b/scripts/train-t2m-flow-v5.py index 2454589f2..6833f1428 100644 --- a/scripts/train-t2m-flow-v5.py +++ b/scripts/train-t2m-flow-v5.py @@ -97,10 +97,10 @@ def d6_to_quat(d): # Canonical parent chain + rest directions, mirroring prep-t2m-v5/v6 exactly. PAR_CANON = [-1, 0, 1, 2, 3, 4, 2, 6, 7, 8, 2, 10, 11, 12, 0, 14, 15, 16, 0, 18, 19, 20] -DIR_CANON = [ +DIR_CANON = [ # keep in sync with prep-t2m-v5.D_CANON [0, 1, 0], [0, 1, 0], [0, 1, 0], [0, 1, 0], [0, 1, 0], [0, 1, 0], - [-1, 0, 0], [-1, 0, 0], [-1, 0, 0], [-1, 0, 0], - [1, 0, 0], [1, 0, 0], [1, 0, 0], [1, 0, 0], + [1, 0, 0], [1, 0, 0], [1, 0, 0], [1, 0, 0], # LEFT arm = +X (#837) + [-1, 0, 0], [-1, 0, 0], [-1, 0, 0], [-1, 0, 0], # RIGHT arm = -X [0, -1, 0], [0, -1, 0], [0, -1, 0], [0, 0, 1], [0, -1, 0], [0, -1, 0], [0, -1, 0], [0, 0, 1], ] @@ -136,6 +136,34 @@ def _centred_z(p): return z - z.mean(dim=1, keepdim=True) +def spine_twist_penalty(q): + """Mean |hip->chest yaw| in radians — an anatomy guard for the phase loss. + + The phase term alone is satisfiable by a DEGENERATE shortcut: rotate the + torso ~180 deg so the existing swing reads as contralateral. Measured at + --phase-weight 0.15 without this guard: hip->chest twist went to 170 deg + (data 6.3 deg, v6.2 6.7 deg) — the user-visible "walk got all twisted". + Penalising the yaw closes that escape route so the only way to earn the + phase reward is to actually fix limb timing. + """ + def conj(a): + return torch.cat([-a[..., :3], a[..., 3:]], -1) + + def mul(a, b): + ax, ay, az, aw = a.unbind(-1) + bx, by, bz, bw = b.unbind(-1) + return torch.stack([aw * bx + ax * bw + ay * bz - az * by, + aw * by - ax * bz + ay * bw + az * bx, + aw * bz + ax * by - ay * bx + az * bw, + aw * bw - ax * bx - ay * by - az * bz], -1) + + rel = mul(conj(q[:, :, 0]), q[:, :, 2]) # hip -> chest + yaw = 2.0 * torch.atan2(rel[..., 1], rel[..., 3]) + # wrap to [-pi, pi] so a 180 deg cheat is maximally penalised + yaw = torch.atan2(torch.sin(yaw), torch.cos(yaw)) + return yaw.abs().mean(dim=1) + + def gait_phase_corr(q, dirs): """Per-sample contralateral gait correlation, in [-1, 1]. @@ -270,6 +298,10 @@ def main(): "(#837 orientation correctness). 0 = off. Applied to " "LOCOMOTION actions only, on the reconstructed clean " "sample. 0.05-0.2 is a sane range.") + ap.add_argument("--twist-weight", type=float, default=0.5, + help="weight of the hip->chest yaw penalty that stops the " + "phase loss from cheating by rotating the torso 180 " + "deg. Only active when --phase-weight > 0.") ap.add_argument("--phase-actions", default="walk,run,march", help="comma list the phase loss applies to") ap.add_argument("--resume", action="store_true", @@ -378,8 +410,14 @@ def main(): denom = g.sum().clamp_min(1.0) # hinge: only penalise below a firm-but-not-saturating target, # so well-phased samples are left alone + # anatomy guard: free below ~20 deg of hip->chest yaw (the + # data sits at ~6 deg), then linearly penalised. + tw = spine_twist_penalty(q_hat) + tw_pen = (tw - 0.35).clamp_min(0.0) loss = loss + a.phase_weight * ( (g * (0.6 - ph).clamp_min(0.0)).sum() / denom) + loss = loss + a.twist_weight * ( + (g * tw_pen).sum() / denom) opt.zero_grad(set_to_none=True) loss.backward() torch.nn.utils.clip_grad_norm_(net.parameters(), 1.0) From f02d30e58fca0f2debd51732c2501a60eb410ed1 Mon Sep 17 00:00:00 2001 From: Fernando Date: Tue, 25 Aug 2026 02:38:11 -0400 Subject: [PATCH 07/35] fix(anim): mirror aug manufactured a backward-locomotion mode (#837) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Travel direction was not converging: measured dot(travel, body-forward) on the chirality-fixed run went 58%/42%/50% forward at ep80 to 56%/25%/31% at ep184 — WORSE with more training, so it was never an under-training problem. Cause: the cache is genuinely BIMODAL in travel direction. Walk has a real backward mode (2792 of 7608 windows, mean -0.810, vs the forward mode's +0.738). Flow matching samples the conditional, so the model walked backward roughly half the time — the user-reported symptom that survived the D_CANON chirality fix. Two sources, both fixed: 1. mirror() reflected quats as (x, -y, -z, w), which flips the FORWARD (Z) axis as well as the lateral one — so every augmented window travelled the opposite way and the augmentation manufactured a backward twin for free. The correct sagittal (left<->right) mirror is (-x, y, z, -w) + the L/R role swap: it inverts only the lateral axis and preserves forward/up. Verified on 40 forward-travelling walk windows: mirrored travel stays forward in 100% of cases (was 0%), contralateral phase preserved (90%), quats unit-norm to 1.2e-07, and it is a genuine mirror not a no-op — mirror-left tracks original-right (0.358 vs 0.773), lateral X flips (+0.179 -> -0.179) while forward Z is preserved (+0.562 -> +0.562). 2. Some SOURCE clips really do walk backward. window_quality() now gates locomotion windows on travel_forward() >= MIN_TRAVEL_FORWARD (0.15), the same stance-foot metric, so the residual mode is dropped rather than learned. Requires another cache rebuild + retrain from scratch. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/prep-t2m-v6.py | 76 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 74 insertions(+), 2 deletions(-) diff --git a/scripts/prep-t2m-v6.py b/scripts/prep-t2m-v6.py index 057b25af9..7629ee3a0 100644 --- a/scripts/prep-t2m-v6.py +++ b/scripts/prep-t2m-v6.py @@ -59,6 +59,8 @@ def load_module(name, fname): MIRROR_PERM[a], MIRROR_PERM[b] = b, a LOCOMOTION = {"walk", "run", "march"} +# minimum dot(travel, body-forward) for a locomotion window to be kept +MIN_TRAVEL_FORWARD = 0.15 HORIZONTAL_OK = {"death", "crawl", "roll", "swim", "fall", "sleep", "sit"} # Source-exclusion (v6.2): some corpus characters carry a deliberately @@ -111,6 +113,51 @@ def pos(role): return fwd / (side + 1e-6) +def travel_forward(w): + """dot(direction of travel, body's own forward). >0 = moving forward. + + Travel is inferred from the STANCE foot: a planted foot drifts backward + relative to a body moving forward, so travel = -velocity of the slower + (planted) foot per frame. Forward comes from the shoulder line + (left - right) x up. + + Some source clips genuinely walk BACKWARD, and they formed a real second + mode in the cache (37% of walk windows, mean -0.810). Flow matching then + samples both modes, so the model walked backward about half the time + (#837, user-reported "moving backwards"). Gating on this collapses the + conditional to one mode. + """ + lsh, rsh = fk_pos(w, 7), fk_pos(w, 11) + side = lsh - rsh + side = side / (np.linalg.norm(side, axis=-1, keepdims=True) + 1e-9) + up = np.broadcast_to(np.array([0, 1, 0], np.float32), side.shape) + f = np.cross(side, up) + f = (f / (np.linalg.norm(f, axis=-1, keepdims=True) + 1e-9)).mean(0) + la, ra = fk_pos(w, 17), fk_pos(w, 21) + vl, vr = np.diff(la, axis=0), np.diff(ra, axis=0) + sl, sr = np.linalg.norm(vl, axis=-1), np.linalg.norm(vr, axis=-1) + v = np.where((sl < sr)[:, None], vl, vr) + t = -v.mean(0) + n = np.linalg.norm(t) + if n < 1e-6: + return 0.0 + fn = np.linalg.norm(f) + if fn < 1e-6: + return 0.0 + return float(np.dot(t / n, f / fn)) + + +def fk_pos(w, role): + """World position of `role` over the window via the canonical chain.""" + T = len(w) + p = np.zeros((T, 3), np.float32) + r = role + while PAR[r] >= 0: + p = p + qrot(w[:, PAR[r]], np.broadcast_to(D_CANON[r], (T, 3))) + r = PAR[r] + return p + + def window_quality(action, w, valid): """True when the window meets the library curation bar.""" # energy band — mean joint rotation speed (rad/frame) @@ -141,12 +188,37 @@ def window_quality(action, w, valid): # majority and walked sideways. Require a clean front-to-back stride. if valid[17] and valid[21] and foot_travel_ratio(w) < 2.0: return False + # Travel-direction gate (v6.4): drop windows that move BACKWARD + # relative to the body's own forward. See travel_forward(). + if valid[7] and valid[11] and valid[17] and valid[21]: + if travel_forward(w) < MIN_TRAVEL_FORWARD: + return False return True def mirror(w, valid): - """Sagittal mirror: reflect each quat (x,-y,-z,w) and swap L/R roles.""" - m = w * np.array([1, -1, -1, 1], np.float32) + """Sagittal (left<->right) mirror: reflect about the X=0 plane and swap + L/R roles. + + A reflection about the plane with unit normal n maps a quaternion + (v, w) -> (-(v - 2(v.n)n), w) — i.e. negate the two components + PERPENDICULAR to n and keep the one along it. The sagittal plane's normal + is X (canonical X = left), so the correct sagittal mirror negates Y and Z + and KEEPS X... which is what this did. But that reflection also flips the + FORWARD (Z) axis, so every mirrored window travelled the opposite way and + the augmentation manufactured a backward-locomotion mode for free + (measured: 37% of walk windows travel backward, a real mode at -0.810). + Flow matching then samples both modes and the model walks backwards about + half the time (#837, user-reported). + + A true left<->right mirror must flip only the LATERAL axis: negate X and W + (equivalently reflect about the YZ plane, normal Z... see below) while + preserving the forward and up axes. For quats under a reflection about the + plane normal to X, the improper transform is applied as q -> (-x, y, z, -w) + combined with the L/R role swap, which preserves handedness of the + forward/up frame and therefore the direction of travel. + """ + m = w * np.array([-1, 1, 1, -1], np.float32) return m[:, MIRROR_PERM], valid[MIRROR_PERM] From 273795c642d1e27ef2bfb2db6c2bacddb125d62b Mon Sep 17 00:00:00 2001 From: Fernando Date: Tue, 25 Aug 2026 03:34:48 -0400 Subject: [PATCH 08/35] =?UTF-8?q?feat(anim):=20travel-direction=20loss=20?= =?UTF-8?q?=E2=80=94=20clean=20data=20alone=20did=20not=20fix=20backwards?= =?UTF-8?q?=20gait=20(#837)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Even with a 100%-forward, unimodal training set (v6.5 cache, after the mirror fix + travel gate), the model still emitted ~50% forward travel at ep40: walk travel -0.009 (50% fwd) run +0.111 (62%) march +0.017 (56%) Ruled out a measurement artefact first: the model's stance-foot drift magnitude is 73% of the data's (0.0337 vs 0.0460), so the feet really are moving and the direction is genuinely ambiguous, not noise-dominated. Flow matching has no term tying the emitted gait to a direction of travel, and the previous run showed this does NOT self-correct — travel got WORSE from ep80 to ep184 (run 42% -> 25%). So supervise it directly rather than spend another 7h rediscovering that. --travel-weight hinges dot(travel, body-forward) toward +0.5 on locomotion rows only, using the same stance-foot definition as prep-t2m-v6.travel_forward and the eval, through the differentiable canonical FK. Validated before use: scores real data +0.62 walk / +0.80 run / +0.82 march at 100% forward with small hinge penalties, punishes a time-REVERSED walk (-0.342, hinge 0.84), emits finite gradients (norm 7.0), and scores `sit` at +0.023 with a large hinge — which is exactly why the term is gated to locomotion actions. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/train-t2m-flow-v5.py | 42 ++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/scripts/train-t2m-flow-v5.py b/scripts/train-t2m-flow-v5.py index 6833f1428..a1424680f 100644 --- a/scripts/train-t2m-flow-v5.py +++ b/scripts/train-t2m-flow-v5.py @@ -136,6 +136,39 @@ def _centred_z(p): return z - z.mean(dim=1, keepdim=True) +def travel_forward_torch(q, dirs): + """dot(direction of travel, body forward), differentiable, per sample. + + Same quantity as prep-t2m-v6.travel_forward and the eval: travel is + inferred from the STANCE (slower) foot, which drifts backward while the + body moves forward. Forward is (Lshoulder - Rshoulder) x up. + + Even with a 100%-forward training set the model sat at ~50% forward + (measured at ep40 with foot motion at 73% of the data's magnitude, so this + is a real direction ambiguity and not measurement noise). Flow matching has + no term tying the emitted gait to a travel direction, so this supervises it + directly — the same reasoning as the gait-phase hinge. + """ + lsh = fk_pos(q, 7, dirs) + rsh = fk_pos(q, 11, dirs) + side = F.normalize(lsh - rsh, dim=-1, eps=1e-6) + up = torch.zeros_like(side) + up[..., 1] = 1.0 + fwd = F.normalize(torch.cross(side, up, dim=-1), dim=-1, eps=1e-6) + fwd = F.normalize(fwd.mean(dim=1), dim=-1, eps=1e-6) # [B,3] + + la = fk_pos(q, 17, dirs) + ra = fk_pos(q, 21, dirs) + vl = la[:, 1:] - la[:, :-1] + vr = ra[:, 1:] - ra[:, :-1] + sl = vl.norm(dim=-1, keepdim=True) + sr = vr.norm(dim=-1, keepdim=True) + v = torch.where(sl < sr, vl, vr) # stance foot + travel = -v.mean(dim=1) # [B,3] + travel = F.normalize(travel, dim=-1, eps=1e-6) + return (travel * fwd).sum(-1) # [B] in [-1,1] + + def spine_twist_penalty(q): """Mean |hip->chest yaw| in radians — an anatomy guard for the phase loss. @@ -298,6 +331,10 @@ def main(): "(#837 orientation correctness). 0 = off. Applied to " "LOCOMOTION actions only, on the reconstructed clean " "sample. 0.05-0.2 is a sane range.") + ap.add_argument("--travel-weight", type=float, default=0.0, + help="weight of the travel-DIRECTION hinge: the body must " + "move along its own forward axis (#837). Locomotion " + "rows only. 0 = off; 0.1-0.3 is a sane range.") ap.add_argument("--twist-weight", type=float, default=0.5, help="weight of the hip->chest yaw penalty that stops the " "phase loss from cheating by rotating the torso 180 " @@ -418,6 +455,11 @@ def main(): (g * (0.6 - ph).clamp_min(0.0)).sum() / denom) loss = loss + a.twist_weight * ( (g * tw_pen).sum() / denom) + if a.travel_weight > 0: + tv = travel_forward_torch(q_hat, dirs_t) + # hinge toward the data's level (~+0.75); no reward above it + loss = loss + a.travel_weight * ( + (g * (0.5 - tv).clamp_min(0.0)).sum() / denom) opt.zero_grad(set_to_none=True) loss.backward() torch.nn.utils.clip_grad_norm_(net.parameters(), 1.0) From 6b57addb2f8730e62ec81802d23b2303ad469dee Mon Sep 17 00:00:00 2001 From: Fernando Date: Tue, 25 Aug 2026 21:40:46 -0400 Subject: [PATCH 09/35] =?UTF-8?q?fix(anim):=20revert=20the=20bogus=20chira?= =?UTF-8?q?lity=20"fix"=20=E2=80=94=20it=20inverted=20a=20correct=20table?= =?UTF-8?q?=20(#837)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit d11f17d6 swapped D_CANON roles 6-9 with 10-13 on the premise that 6-9 were the LEFT arm chain. They are not. AnimationMerger.cpp kParentCanon documents: 6-9 rcollar, rshoulder, relbow, rhand <- RIGHT 10-13 lcollar, lshoulder, lelbow, lhand <- LEFT 14-17 rbuttock, rhip, rknee, rfoot <- RIGHT 18-21 lbuttock, lhip, lknee, lfoot <- LEFT confirmed independently by compensateCanonicalHandedness(), which reads `lx = worldXForCanon(19)` (LEFT leg) against `rx = worldXForCanon(15)` (RIGHT). The "X=left" note in the axis comment describes the AXIS convention, not the role order — I misread it as the latter. So roles 6-9 (RIGHT) on -X was already correct, and the swap inverted it. Cost: a ~120 deg shoulder-vs-hip yaw in the cache where real gait counter-rotates 10-20 deg — the anatomically twisted torso the user reported from the GUI. Reverting drops it to ~59 deg on the same windows. The travel-direction sign flip that appeared to justify the swap was real but was a SYMPTOM of measuring travel with a mirrored frame, not evidence that the role order was wrong. Both D_CANON and the trainer's DIR_CANON copy are reverted and asserted equal. The v6.5 cache and model were built with the broken table and are discarded. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/prep-t2m-v5.py | 27 ++++++++++++++------------- scripts/train-t2m-flow-v5.py | 4 ++-- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/scripts/prep-t2m-v5.py b/scripts/prep-t2m-v5.py index 8243e9da5..ae0ae6a90 100644 --- a/scripts/prep-t2m-v5.py +++ b/scripts/prep-t2m-v5.py @@ -79,22 +79,23 @@ def load_module(name, fname): # FIXED canonical T-pose bone directions D (canonical axes: +Y up, +Z fwd, # +X left — the CMU frame convention every dump is normalized into). -# CHIRALITY FIX (#837): roles 6-9 are the LEFT arm chain and 10-13 the RIGHT, -# and the documented convention above (and AnimationMerger.cpp: "canonical axes -# X=left, Y=up, Z=forward" / "canonical left joints expect +X") puts left on +X. -# The table previously had left on -X and right on +X — inverted. That made every -# clip's shoulder-derived forward point -Z while the runtime treats clips as -# +Z-facing, so `detectBackwardFacing` compensated with a whole-clip 180 yaw: -# the BODY then faced the right way but the LIMBS still moved backwards -# (user-reported: "faces the camera, limbs move as if facing back"). -# Measured on the v6.2 cache, stance-foot travel vs the body's own forward: -# before walk -0.209 (40% forward) run -0.785 (7%) march -0.334 (32%) -# after walk +0.209 (60% forward) run +0.785 (93%) march +0.334 (68%) -# An exact sign inversion — the signature of a swapped-chirality frame. +# ROLE ORDER (AnimationMerger.cpp kParentCanon comment — do NOT re-derive this +# from the "X=left" axis note, which describes the AXES, not the role order): +# 0-5 hip, abdomen, chest, neck, neck1, head +# 6-9 RIGHT collar, shoulder, elbow, hand <-- RIGHT side, so -X +# 10-13 LEFT collar, shoulder, elbow, hand <-- LEFT side, so +X +# 14-17 RIGHT buttock, hip, knee, foot +# 18-21 LEFT buttock, hip, knee, foot +# Confirmed independently by compensateCanonicalHandedness(), which reads +# `lx = worldXForCanon(19)` (LEFT leg) vs `rx = worldXForCanon(15)` (RIGHT leg). +# A previous "chirality fix" swapped 6-9 with 10-13 on the false premise that +# 6-9 were the left arm; that INVERTED an already-correct table and produced a +# ~119 deg shoulder-vs-hip yaw in the cache (real gait counter-rotates 10-20), +# i.e. the anatomically twisted torso the user saw. Reverted. D_CANON = np.array([ [0, 1, 0], [0, 1, 0], [0, 1, 0], [0, 1, 0], [0, 1, 0], [0, 1, 0], - [1, 0, 0], [1, 0, 0], [1, 0, 0], [1, 0, 0], [-1, 0, 0], [-1, 0, 0], [-1, 0, 0], [-1, 0, 0], + [1, 0, 0], [1, 0, 0], [1, 0, 0], [1, 0, 0], [0, -1, 0], [0, -1, 0], [0, -1, 0], [0, 0, 1], [0, -1, 0], [0, -1, 0], [0, -1, 0], [0, 0, 1], ], np.float32) diff --git a/scripts/train-t2m-flow-v5.py b/scripts/train-t2m-flow-v5.py index a1424680f..12b8b1910 100644 --- a/scripts/train-t2m-flow-v5.py +++ b/scripts/train-t2m-flow-v5.py @@ -99,8 +99,8 @@ def d6_to_quat(d): 0, 14, 15, 16, 0, 18, 19, 20] DIR_CANON = [ # keep in sync with prep-t2m-v5.D_CANON [0, 1, 0], [0, 1, 0], [0, 1, 0], [0, 1, 0], [0, 1, 0], [0, 1, 0], - [1, 0, 0], [1, 0, 0], [1, 0, 0], [1, 0, 0], # LEFT arm = +X (#837) - [-1, 0, 0], [-1, 0, 0], [-1, 0, 0], [-1, 0, 0], # RIGHT arm = -X + [-1, 0, 0], [-1, 0, 0], [-1, 0, 0], [-1, 0, 0], # roles 6-9 = RIGHT arm + [1, 0, 0], [1, 0, 0], [1, 0, 0], [1, 0, 0], # roles 10-13 = LEFT arm [0, -1, 0], [0, -1, 0], [0, -1, 0], [0, 0, 1], [0, -1, 0], [0, -1, 0], [0, -1, 0], [0, 0, 1], ] From de5fe2223f00921599f8c1b6a66deadae7e4f597 Mon Sep 17 00:00:00 2001 From: Fernando Date: Wed, 26 Aug 2026 09:06:21 -0400 Subject: [PATCH 10/35] =?UTF-8?q?feat(anim):=20amplitude=20ceiling=20?= =?UTF-8?q?=E2=80=94=20the=20phase/travel=20hinges=20over-drove=20the=20ga?= =?UTF-8?q?it=20(#837)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit At ep30 on the corrected v66 data the signs were finally right (travel +0.475 at 92% forward, contra +0.647 at 100%) but the close-up RENDER showed a knee-to-chest exaggerated march, not a walk. The metrics agreed once I looked at excursion: stride 3.46 and armSwing 3.73 against real Mixamo walk values of 0.765 and 1.718 — a ~4.5x over-drive. Both the phase and travel hinges REWARD motion, and nothing bounded the amplitude. --amp-weight penalises only the EXCESS above generous ceilings (stride 1.6, swing 3.0, roughly 2x the real-walk values), so ordinary motion is untouched. Validated before use: real Walk.fbx scores 0.0000, the v66 training walk 0.0016 (8% of windows above the ceiling), the over-driven ep30 model 2.5577. Real Running.fbx scores 0.347 — correctly reflecting that a run legitimately has a larger excursion than a walk, which is why the ceilings are set above walk values rather than at them. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/train-t2m-flow-v5.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/scripts/train-t2m-flow-v5.py b/scripts/train-t2m-flow-v5.py index 12b8b1910..59b680c5e 100644 --- a/scripts/train-t2m-flow-v5.py +++ b/scripts/train-t2m-flow-v5.py @@ -169,6 +169,25 @@ def travel_forward_torch(q, dirs): return (travel * fwd).sum(-1) # [B] in [-1,1] +def amplitude_excess(q, dirs): + """How far the ankle/wrist fore-aft excursion EXCEEDS human range. + + The phase + travel hinges reward motion and nothing bounded it, so the + model over-drove the limbs: measured stride 3.46 and armSwing 3.73 against + real Mixamo walk values of 0.765 and 1.718 - rendering as a knee-to-chest + exaggerated march instead of a walk. This penalises only the EXCESS above + generous ceilings, so normal motion is untouched. + """ + def pk(role): + z = fk_pos(q, role, dirs)[..., 2] + return z.max(dim=1).values - z.min(dim=1).values + + stride = 0.5 * (pk(17) + pk(21)) + swing = 0.5 * (pk(9) + pk(13)) + # ceilings ~2x the real-walk values, so only true over-drive is punished + return (stride - 1.6).clamp_min(0.0) + (swing - 3.0).clamp_min(0.0) + + def spine_twist_penalty(q): """Mean |hip->chest yaw| in radians — an anatomy guard for the phase loss. @@ -335,6 +354,11 @@ def main(): help="weight of the travel-DIRECTION hinge: the body must " "move along its own forward axis (#837). Locomotion " "rows only. 0 = off; 0.1-0.3 is a sane range.") + ap.add_argument("--amp-weight", type=float, default=0.0, + help="penalise limb excursion ABOVE human range (#837): " + "the phase/travel hinges reward motion and otherwise " + "over-drive the legs into a knee-to-chest march. " + "0 = off; 0.3 is a sane start.") ap.add_argument("--twist-weight", type=float, default=0.5, help="weight of the hip->chest yaw penalty that stops the " "phase loss from cheating by rotating the torso 180 " @@ -455,6 +479,10 @@ def main(): (g * (0.6 - ph).clamp_min(0.0)).sum() / denom) loss = loss + a.twist_weight * ( (g * tw_pen).sum() / denom) + if a.amp_weight > 0: + amp = amplitude_excess(q_hat, dirs_t) + loss = loss + a.amp_weight * ( + (g * amp).sum() / denom) if a.travel_weight > 0: tv = travel_forward_torch(q_hat, dirs_t) # hinge toward the data's level (~+0.75); no reward above it From 2b579e0601bc08f9dd344e88fe2eb4cfaa2a68b2 Mon Sep 17 00:00:00 2001 From: Fernando Date: Wed, 26 Aug 2026 10:55:30 -0400 Subject: [PATCH 11/35] =?UTF-8?q?fix(anim):=20jitter=20penalty=20=E2=80=94?= =?UTF-8?q?=20the=20model=20emitted=20noise,=20not=20motion=20(#837)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit THE defect that made every amplitude metric lie, and the reason renders looked static while the numbers looked healthy. Round-trip proof: the ep90 model's canonical ankle "scissor" measured 0.955 — LARGER than the real Mixamo walk's 0.652 — yet after retarget and re-extraction it was 0.241, and 8 rendered frames showed the legs barely moving. Measuring joint SPEED instead of amplitude explained it: v66 training walk 0.0557 rad/frame (band 0.032-0.099) ep90 model output 0.2661 rad/frame (6x the data mean) ep90 worst joint 1.4165 rad/frame (~81 deg/frame, physically impossible; ~20x the data's worst joint) So the model was producing high-frequency jitter. The retarget's smoothing damped it to 0.041, leaving no coherent stride — while every metric I had built (stride, armSwing, ankleSpread) reads amplitude, which jitter inflates. That is why travel/contra/armSwing/stride all hit target at ep90 on a broken render. --jitter-weight penalises mean per-joint angular speed above 0.12 rad/frame plus the worst single joint above 0.40, applied to ALL actions since jitter is never wanted. Validated: real Walk.fbx and the walk/jump training windows all score 0.0000 (dance 0.0059, 10% of windows), while the ep90 model scores 0.6134. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/train-t2m-flow-v5.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/scripts/train-t2m-flow-v5.py b/scripts/train-t2m-flow-v5.py index 59b680c5e..49306d2d5 100644 --- a/scripts/train-t2m-flow-v5.py +++ b/scripts/train-t2m-flow-v5.py @@ -188,6 +188,28 @@ def pk(role): return (stride - 1.6).clamp_min(0.0) + (swing - 3.0).clamp_min(0.0) +def jitter_excess(q): + """Per-frame angular speed ABOVE the training band = high-frequency jitter. + + THE defect that made every amplitude metric lie. The ep90 model emitted a + mean joint speed of 0.266 rad/frame against the data's 0.0557 (band + 0.032-0.099), and its worst joint (role 21) ran at 1.416 rad/frame — about + 81 deg per frame, physically impossible. The retarget's smoothing then + damped it to 0.041, so the RENDER was nearly static while the canonical + ankle "scissor" measured LARGER than the real walk: the metrics were + reading jitter amplitude, not stride. + + Penalise the mean per-joint speed above a ceiling just past the data band, + plus the worst single joint, which is where the impossible spikes live. + """ + dot = (q[:, 1:] * q[:, :-1]).sum(-1).abs().clamp(0.0, 1.0) + speed = 2.0 * torch.acos(dot.clamp(max=1.0 - 1e-7)) # [B,T-1,J] + per_joint = speed.mean(dim=1) # [B,J] + mean_excess = (per_joint.mean(dim=-1) - 0.12).clamp_min(0.0) + worst_excess = (per_joint.max(dim=-1).values - 0.40).clamp_min(0.0) + return mean_excess + worst_excess + + def spine_twist_penalty(q): """Mean |hip->chest yaw| in radians — an anatomy guard for the phase loss. @@ -354,6 +376,10 @@ def main(): help="weight of the travel-DIRECTION hinge: the body must " "move along its own forward axis (#837). Locomotion " "rows only. 0 = off; 0.1-0.3 is a sane range.") + ap.add_argument("--jitter-weight", type=float, default=0.0, + help="penalise per-frame angular speed above the data band " + "(#837). Applied to ALL actions — jitter is never " + "wanted. 0 = off; 1.0 is a sane start.") ap.add_argument("--amp-weight", type=float, default=0.0, help="penalise limb excursion ABOVE human range (#837): " "the phase/travel hinges reward motion and otherwise " @@ -458,6 +484,10 @@ def main(): tgt = xb - x0 mask = mb[:, None, :] # [B,1,C6] loss = ((v - tgt) ** 2 * mask).sum() / mask.sum() / T + if a.jitter_weight > 0: + q_all = d6_to_quat( + (xt + (1.0 - t[:, None, None]) * v).reshape(-1, T, J, D6)) + loss = loss + a.jitter_weight * jitter_excess(q_all).mean() if a.phase_weight > 0: # Flow matching predicts a VELOCITY, so reconstruct the clean # sample the model implies at this t before measuring gait: From 84609ded6bc9be33de8e05c9f05aced70f9acb99 Mon Sep 17 00:00:00 2001 From: Fernando Date: Wed, 26 Aug 2026 11:31:36 -0400 Subject: [PATCH 12/35] fix(anim): make the speed guard a two-sided BAND, not a ceiling (#837) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ceiling-only jitter penalty removed the noise and the motion with it. At ep30 the mean joint speed fell 0.353 -> 0.227 and the render went smooth but nearly STATIC — legs barely separating, arms pinned — with contra regressing 0.345 -> 0.284 while every other metric improved. Jitter and stride were entangled: the amplitude metrics had been counting the noise, so suppressing it removed what they were measuring. Pushing the speed below a ceiling is therefore not enough; the objective has to pull it TOWARD the data band (walk mean 0.056, real Walk.fbx 0.059). jitter_excess() now penalises both sides — above 0.10 and, at 4x weight, below 0.045 — plus the worst joint above 0.45. Validated: real Walk.fbx 0.0000, data walk 0.0006 (4% of windows), run 0.0144 and jump 0.0274 (legitimately more variable), and a synthetically FROZEN clip scores 0.1761, confirming too-slow is now penalised as well as too-fast. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/train-t2m-flow-v5.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/scripts/train-t2m-flow-v5.py b/scripts/train-t2m-flow-v5.py index 49306d2d5..39a095236 100644 --- a/scripts/train-t2m-flow-v5.py +++ b/scripts/train-t2m-flow-v5.py @@ -205,8 +205,15 @@ def jitter_excess(q): dot = (q[:, 1:] * q[:, :-1]).sum(-1).abs().clamp(0.0, 1.0) speed = 2.0 * torch.acos(dot.clamp(max=1.0 - 1e-7)) # [B,T-1,J] per_joint = speed.mean(dim=1) # [B,J] - mean_excess = (per_joint.mean(dim=-1) - 0.12).clamp_min(0.0) - worst_excess = (per_joint.max(dim=-1).values - 0.40).clamp_min(0.0) + # TWO-SIDED band, not a ceiling. A pure ceiling removes the jitter AND the + # motion with it: at ep30 speed fell 0.353 -> 0.227 and the render went + # smooth but nearly STATIC (legs barely separating, contra dropped + # 0.345 -> 0.284). Jitter and stride were entangled, so the objective has + # to pull the speed TOWARD the data band (walk mean 0.056, real Walk.fbx + # 0.059) rather than merely below a ceiling. + m = per_joint.mean(dim=-1) + mean_excess = (m - 0.10).clamp_min(0.0) + (0.045 - m).clamp_min(0.0) * 4.0 + worst_excess = (per_joint.max(dim=-1).values - 0.45).clamp_min(0.0) return mean_excess + worst_excess From b09507fb7f5551b2f156d15310a195e1196e3bc7 Mon Sep 17 00:00:00 2001 From: Fernando Date: Wed, 26 Aug 2026 14:52:23 -0400 Subject: [PATCH 13/35] fix(anim): amplitude term was one-sided and its ceiling was below the data (#837) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two calibration faults in amplitude_excess, both found by measuring the guard contributions instead of guessing which weight to change next. 1. NOTHING DEFENDED AMPLITUDE FROM BELOW. At v6.7 ep45 the jitter term was 80% of the guard loss (0.1446 of 0.181) while amp contributed exactly 0.0000, and armSwing/stride fell 1.481/1.126 -> 0.894/0.690 — below the real walk's 1.718/0.765. The limbs were collateral damage of jitter pushing speed down, with no term objecting. amplitude_excess is now two-sided, floors weighted 2x. 2. THE CEILING WAS BELOW THE DATA. The stride ceiling of 1.6 sat under the all-action p95 of 2.969 (p99 3.326), so it had been clipping legitimate march, climb and kick motion the whole time. Ceilings raised to 3.5/3.7, just above the data p99. Catching genuine over-drive is the jitter band's job, not this term's. Floors are calibrated to WALK's data p5 (stride 0.741, swing 0.809) and gated to walk rows only: march's stride p5 is 1.174, so a global floor penalised 81% of real march windows. Note the floors follow the TRAINING data, not the reference Mixamo clip, whose swing (1.718) sits well above this corpus's median (1.052). Validated: real Walk.fbx 0.0000; training data walk 0.0052, run 0.0048, march 0.0266; a frozen clip 2.9000; and the collapsed v6.7 ep45 model is now penalised (0.0982) where before it scored 0.0000. v6.8 raises amp-weight 0.1 -> 0.4 now that the term is correctly two-sided. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/train-t2m-flow-v5.py | 41 ++++++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/scripts/train-t2m-flow-v5.py b/scripts/train-t2m-flow-v5.py index 39a095236..7d518baa3 100644 --- a/scripts/train-t2m-flow-v5.py +++ b/scripts/train-t2m-flow-v5.py @@ -169,7 +169,7 @@ def travel_forward_torch(q, dirs): return (travel * fwd).sum(-1) # [B] in [-1,1] -def amplitude_excess(q, dirs): +def amplitude_excess(q, dirs, lo_scale=None): """How far the ankle/wrist fore-aft excursion EXCEEDS human range. The phase + travel hinges reward motion and nothing bounded it, so the @@ -184,8 +184,29 @@ def pk(role): stride = 0.5 * (pk(17) + pk(21)) swing = 0.5 * (pk(9) + pk(13)) - # ceilings ~2x the real-walk values, so only true over-drive is punished - return (stride - 1.6).clamp_min(0.0) + (swing - 3.0).clamp_min(0.0) + # TWO-SIDED, like the speed band. A ceiling-only amplitude term contributes + # exactly 0.0 once the model is under it, so nothing defends the limbs from + # COLLAPSING: measured at v6.7 ep45 the jitter term was 80% of the guard + # loss (0.1446/0.181) while amp was 0.0000, and armSwing/stride fell + # 1.481/1.126 -> 0.894/0.690, i.e. BELOW the real walk's 1.718/0.765. The + # limbs were collateral damage of jitter pushing speed down. Real-walk + # values anchor the floors (stride 0.765, swing 1.718); floors are set just + # under them and weighted 2x so shrinking is punished harder than growing. + # Ceilings sit above the ALL-ACTION p99 (stride 3.33, swing 3.54) so real + # data is never penalised. The original 1.6 stride ceiling was BELOW the + # data's p95 of 2.969 — it was clipping legitimate motion across march, + # climb and kick, which contributed to the amplitude collapse. + over = (stride - 3.5).clamp_min(0.0) + (swing - 3.7).clamp_min(0.0) + # Floors calibrated to the TRAINING data's 5th percentile (walk stride 0.741, + # swing 0.809) — NOT to the reference Mixamo clip, whose swing (1.718) sits + # well above this corpus's median (1.052). Floors above the data would + # penalise legitimate windows: at swing floor 1.50, 67% of real walk windows + # scored a penalty. + st_floor = 0.70 * (1.0 if lo_scale is None else lo_scale) + sw_floor = 0.75 * (1.0 if lo_scale is None else lo_scale) + under = ((st_floor - stride).clamp_min(0.0) + + (sw_floor - swing).clamp_min(0.0)) * 2.0 + return over + under def jitter_excess(q): @@ -517,7 +538,19 @@ def main(): loss = loss + a.twist_weight * ( (g * tw_pen).sum() / denom) if a.amp_weight > 0: - amp = amplitude_excess(q_hat, dirs_t) + # Floors are calibrated to WALK's data p5; march/run have + # legitimately larger excursion (march stride p5 1.174 vs + # walk 0.741), so a global floor penalised 81% of real march + # windows. Apply the floor only to walk rows; the others + # keep the over-drive ceiling. + walk_i = vocab.index("walk") if "walk" in vocab else -1 + if walk_i >= 0: + gw = tb_raw[:, walk_i].clamp(0, 1) + else: + gw = torch.zeros_like(g) + amp_ceil = amplitude_excess(q_hat, dirs_t, lo_scale=0.0) + amp_full = amplitude_excess(q_hat, dirs_t) + amp = amp_ceil + gw * (amp_full - amp_ceil) loss = loss + a.amp_weight * ( (g * amp).sum() / denom) if a.travel_weight > 0: From 96e45f4437295bf2f36694385b731c5e6c30aceb Mon Sep 17 00:00:00 2001 From: Fernando Date: Wed, 26 Aug 2026 16:37:21 -0400 Subject: [PATCH 14/35] fix(anim): the data gates encoded WALK assumptions and starved run/march (#837) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit v6.8's walk render is good, but run and march came out too gentle to tell apart from a walk. Instrumenting the prep pipeline's drop counts per filter (rather than reasoning about it) found three gates inherited from the walk-focused v6.1 work, each rejecting legitimate non-walk motion: 1. ENERGY CEILING (the big one). A single 0.11 rad/frame bound, but the real Mixamo clips measure walk 0.059 and **run 0.187** — a genuine run FAILS the gate. Five of the twelve curated `run` clips died on it (0.13-0.22), leaving run to train on 24-36 windows. The ceiling is now action-aware: 0.26 for FAST_ACTIONS (run/march/jump/kick/punch/boxing/attack/throw/dance), 0.11 retained for walk. Run windows 36 -> 76, dance 2548 -> 4560. 2. STRIDE-DIRECTIONALITY GATE vs MARCH. The fwd/side >= 2.0 gate catches splayed sideways walking, but marching lifts the knees IN PLACE, so its fore/aft travel is legitimately low — CMU march windows have a median ratio of 1.56 and only 36% clear 2.0, so the gate discarded 64% of the data. March is now exempt; walk keeps the gate. 3. SHORT-CLIP FLOOR. `nF < T//2` returned before the cycle-repeat logic could run, and 11 of 12 curated run clips are 20-27 frames (game run cycles are short loops). 10 of those 11 loop cleanly (cyc <= 0.24, most exactly 0.000), so clips >= 16 frames that loop are now admitted and repeated; non-loops are still rejected because their repeat would visibly jump. Also relaxes the travel gate to > 0 for non-walk actions (walk keeps 0.15). Co-Authored-By: Claude Opus 5 (1M context) --- scripts/prep-t2m-v6.py | 44 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/scripts/prep-t2m-v6.py b/scripts/prep-t2m-v6.py index 7629ee3a0..64297e019 100644 --- a/scripts/prep-t2m-v6.py +++ b/scripts/prep-t2m-v6.py @@ -59,6 +59,9 @@ def load_module(name, fname): MIRROR_PERM[a], MIRROR_PERM[b] = b, a LOCOMOTION = {"walk", "run", "march"} +# actions whose real-world joint speed exceeds the walk-tuned energy ceiling +FAST_ACTIONS = {"run", "march", "jump", "kick", "punch", "boxing", "attack", + "throw", "dance"} # minimum dot(travel, body-forward) for a locomotion window to be kept MIN_TRAVEL_FORWARD = 0.15 HORIZONTAL_OK = {"death", "crawl", "roll", "swim", "fall", "sleep", "sit"} @@ -160,10 +163,16 @@ def fk_pos(w, role): def window_quality(action, w, valid): """True when the window meets the library curation bar.""" - # energy band — mean joint rotation speed (rad/frame) + # Energy band — mean joint rotation speed (rad/frame). The upper bound is + # ACTION-AWARE: a real Mixamo run measures 0.187 and a real walk 0.059, so + # the single 0.11 ceiling (tuned on walk-dominated data) systematically + # excluded genuine running — 5 of the 12 curated `run` clips died on it + # (0.13-0.22), which is why run trained on 24-36 windows and rendered as a + # gentle walk. Fast actions get headroom; walk keeps the tight bound. dq = np.abs((w[1:] * w[:-1]).sum(-1)).clip(0, 1) e = float((2 * np.arccos(dq)).mean()) - if not (0.004 <= e <= 0.11): + hi = 0.26 if action in FAST_ACTIONS else 0.11 + if not (0.004 <= e <= hi): return False if action in HORIZONTAL_OK: return True @@ -186,12 +195,26 @@ def window_quality(action, w, valid): # sideways. 59% of raw CMU walk windows are splayed/side-stepping # (measured fwd/side < 1.5) — the model faithfully learned that # majority and walked sideways. Require a clean front-to-back stride. - if valid[17] and valid[21] and foot_travel_ratio(w) < 2.0: + # Stride-directionality gate: catches splayed/sideways walking. EXEMPT + # march — marching lifts the knees in place, so its fore/aft travel is + # legitimately low (CMU march windows: median ratio 1.56, only 36% clear + # 2.0), and applying a walk gate to it discarded 64% of the data and + # left march with 136 windows. + if (action != "march" and valid[17] and valid[21] + and foot_travel_ratio(w) < 2.0): return False # Travel-direction gate (v6.4): drop windows that move BACKWARD # relative to the body's own forward. See travel_forward(). + # + # The threshold is RELAXED for the data-poor actions. At 0.15 the gate + # left run with 24 windows and march 132 (from 192/308), and the ep45 + # renders showed run and march too gentle to distinguish from a walk — + # the sampler cannot manufacture variety from 24 windows. Walk has 2704 + # and can afford the strict gate; for run/march, merely NOT going + # backward (> 0) is the useful signal. if valid[7] and valid[11] and valid[17] and valid[21]: - if travel_forward(w) < MIN_TRAVEL_FORWARD: + floor = MIN_TRAVEL_FORWARD if action == "walk" else 0.0 + if travel_forward(w) < floor: return False return True @@ -274,8 +297,19 @@ def add(action, w, valid): def windows(action, cq, valid): nF = cq.shape[0] + # Admit clips shorter than T/2 WHEN THEY LOOP. Game run cycles are short + # loops: 11 of the 12 curated `run` clips are 20-27 frames, and the old + # T//2 (30-frame) floor discarded them before the cycle-repeat below + # could use them — leaving run with 24 training windows and a render too + # gentle to tell apart from a walk. Measured: 10 of those 11 loop + # cleanly (cyc <= 0.24, most exactly 0.000), so repeating them is sound. + # Non-looping short clips are still rejected (their repeat would jump). if nF < T // 2: - return + if nF < 16: + return + dq0 = np.abs((cq[-1] * cq[0]).sum(-1)).clip(0, 1) + if float((2 * np.arccos(dq0)).mean()) >= 0.25: + return if nF < T: dq = np.abs((cq[-1] * cq[0]).sum(-1)).clip(0, 1) cyc = float((2 * np.arccos(dq)).mean()) < 0.25 From 55d812a3b40951e9e8c08590cfb7a2502fe5d931 Mon Sep 17 00:00:00 2001 From: Fernando Date: Wed, 26 Aug 2026 17:32:16 -0400 Subject: [PATCH 15/35] fix(anim): make the speed band action-aware, calibrated from measured p95 (#837) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After the run/march data recovery, v7.2's run render finally had genuine knee lift and arm swing (v6.8's run was indistinguishable from its walk) — but the WALK render regressed: arms tucked, minimal leg separation. Diagnosed by measurement, and my first hypothesis was wrong. Sampling shares barely moved (walk 10.56% -> 9.81%), so it was not sampling dilution. What did change: the cache's overall energy rose 0.0429 -> 0.0550 (p90 0.078 -> 0.113) while WALK-only energy stayed at 0.0510. The faster fast-action windows pull the shared model and walk inherits the speed — a conditioning problem, and the jitter band was action-blind. jitter_excess() now takes per-sample hi/lo, and the bands come from the data's measured per-action mean-speed medians on the v72 cache: run 0.110, dance 0.095, punch 0.064, walk 0.055, march 0.050, jump 0.042 So only run and dance get the raised ceiling (0.22 vs walk's 0.10). Notably march is NOT fast by this metric — it is close to walk — and an earlier cut that lumped march in with run and raised its FLOOR to 0.10 penalised 96% of real march windows. The floor is now a uniform low 0.035. Validated: data walk 0.0000, run 0.0073 (20% of windows), march 0.0069 (19%), dance 0.0489; a frozen clip still scores 0.1361. Real Mixamo running drops from 0.1425 on the walk band to 0.0553 on the fast band. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/train-t2m-flow-v5.py | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/scripts/train-t2m-flow-v5.py b/scripts/train-t2m-flow-v5.py index 7d518baa3..a1a9adcca 100644 --- a/scripts/train-t2m-flow-v5.py +++ b/scripts/train-t2m-flow-v5.py @@ -209,7 +209,7 @@ def pk(role): return over + under -def jitter_excess(q): +def jitter_excess(q, hi=None, lo=None): """Per-frame angular speed ABOVE the training band = high-frequency jitter. THE defect that made every amplitude metric lie. The ep90 model emitted a @@ -233,7 +233,15 @@ def jitter_excess(q): # to pull the speed TOWARD the data band (walk mean 0.056, real Walk.fbx # 0.059) rather than merely below a ceiling. m = per_joint.mean(dim=-1) - mean_excess = (m - 0.10).clamp_min(0.0) + (0.045 - m).clamp_min(0.0) * 4.0 + # The band is PER-SAMPLE so it can be action-aware. A single band is + # action-blind, and after the run/march data recovery the cache's overall + # energy rose 0.0429 -> 0.0550 (p90 0.078 -> 0.113) while WALK-only energy + # stayed at 0.0510 — the faster fast-action windows pull the shared model + # and walk inherits the speed, which is what degraded the v7.2 walk render. + hi_t = 0.10 if hi is None else hi + lo_t = 0.045 if lo is None else lo + mean_excess = ((m - hi_t).clamp_min(0.0) + + (lo_t - m).clamp_min(0.0) * 4.0) worst_excess = (per_joint.max(dim=-1).values - 0.45).clamp_min(0.0) return mean_excess + worst_excess @@ -448,6 +456,16 @@ def main(): loco_mask_v = torch.zeros(V, device=dev) for i in loco_idx: loco_mask_v[i] = 1.0 + # actions whose real joint speed exceeds the walk band (mirrors + # prep-t2m-v6.FAST_ACTIONS) + # Measured mean-speed medians on the v72 cache: run 0.110, dance 0.095, + # punch 0.064, walk 0.055, march 0.050, jump 0.042. Only run and dance + # genuinely need a raised ceiling; march/jump are NOT fast by this metric. + fast_names = {"run", "dance"} + fast_mask_v = torch.zeros(V, device=dev) + for i, w in enumerate(vocab): + if w in fast_names: + fast_mask_v[i] = 1.0 if a.phase_weight > 0: print(f"gait-phase loss ON (w={a.phase_weight}) for " f"{[vocab[i] for i in loco_idx]}", flush=True) @@ -515,7 +533,19 @@ def main(): if a.jitter_weight > 0: q_all = d6_to_quat( (xt + (1.0 - t[:, None, None]) * v).reshape(-1, T, J, D6)) - loss = loss + a.jitter_weight * jitter_excess(q_all).mean() + # per-sample band: fast actions may legitimately move ~2.5x a + # walk (real Mixamo run 0.187 vs walk 0.059) + # Bands are set from the DATA's per-action p95, measured on the + # v72 cache. Note march's mean speed (median 0.050) is close to + # WALK's (0.055) — marching in place is not fast by this measure, + # so lumping it with run and raising its FLOOR to 0.10 penalised + # 96% of real march windows. Only genuinely fast actions (run, + # dance) get the raised ceiling, and the floor stays low for all. + is_fast = (tb_raw * fast_mask_v).sum(-1).clamp(0, 1) + hi = 0.10 + is_fast * 0.12 # walk 0.10, fast 0.22 + lo = torch.full_like(hi, 0.035) # low floor for every action + loss = loss + a.jitter_weight * jitter_excess( + q_all, hi=hi, lo=lo).mean() if a.phase_weight > 0: # Flow matching predicts a VELOCITY, so reconstruct the clean # sample the model implies at this t before measuring gait: From 8157d7509c9286887c69c67f4666d98f5720df1d Mon Sep 17 00:00:00 2001 From: Fernando Date: Wed, 26 Aug 2026 18:24:59 -0400 Subject: [PATCH 16/35] =?UTF-8?q?feat(anim):=20periodicity=20gate=20?= =?UTF-8?q?=E2=80=94=20the=20model=20had=20no=20gait=20CYCLE=20(#837)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit User render feedback: "the model is not moving, it keeps just trembling in weird positions". Every metric I had built said the model was healthy, so the metrics were the problem. Measuring PERIODICITY — autocorrelation of the ankle-scissor signal at lag >= 8 — explains it: real Mixamo Walk.fbx +0.968 (near-perfect cycle) Rumba's own mixamo clip +0.712 generated_march +0.187 (no cycle) The exported clip's joint speed was 0.0574, squarely inside the real-motion band, so the motion had the right MAGNITUDE with no cyclic STRUCTURE. That is what trembling in place is. Every prior metric (travel, contra, twist, amplitude, speed) is an average or a correlation that non-cyclic twitching satisfies just as well — none of them can see a cycle. The data is the deeper cause: walk's median periodicity is only 0.446 with 26% of windows above 0.6, and the model reproduced about half of that (0.26). It was never shown a clean gait. 814 strongly-periodic locomotion windows do exist (walk 692, run 43, march 79), so window_quality() now requires >= 0.6 for locomotion. Cache: walk 2704 -> 584, run 76 -> 44, march 340 -> 52 — much smaller but actual gait, and cleanliness over volume is what unlocked every previous step here. Also adds joint_hinge_signs() for the requested knee/elbow backward-bend guard, but leaves it UNGATED and documents why: two sign conventions both rejected real Mixamo clips (details in the docstring), and an unsigned bound cannot separate "bent 75 deg forward" from "bent 75 deg backward". A correct guard needs each rig's bind-pose hinge axis, which the canonical frame does not carry. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/prep-t2m-v6.py | 112 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) diff --git a/scripts/prep-t2m-v6.py b/scripts/prep-t2m-v6.py index 64297e019..fdc23fc1c 100644 --- a/scripts/prep-t2m-v6.py +++ b/scripts/prep-t2m-v6.py @@ -64,6 +64,13 @@ def load_module(name, fname): "throw", "dance"} # minimum dot(travel, body-forward) for a locomotion window to be kept MIN_TRAVEL_FORWARD = 0.15 +# minimum ankle-scissor autocorrelation for a locomotion window (see +# gait_periodicity); real Mixamo Walk.fbx scores 0.968 +MIN_PERIODICITY = 0.6 +# max tolerated BACKWARD bend (degrees) for the knee/elbow hinges; small +# positive slack absorbs canonicalisation noise near full extension +MAX_HYPEREXTEND_KNEE = 20.0 +MAX_HYPEREXTEND_ELBOW = 35.0 HORIZONTAL_OK = {"death", "crawl", "roll", "swim", "fall", "sleep", "sit"} # Source-exclusion (v6.2): some corpus characters carry a deliberately @@ -161,6 +168,105 @@ def fk_pos(w, role): return p +def gait_periodicity(w): + """Strongest autocorrelation of the ankle-scissor signal at lag >= 8. + + THE quality the other gates cannot see. A real gait is strongly cyclic — + real Mixamo Walk.fbx scores 0.968 — but every other metric here (energy, + stride ratio, travel, uprightness) is an average or a correlation that + NON-cyclic twitching satisfies just as well. The v6.8 model measured + healthy on all of them yet rendered as trembling in place, and its + periodicity was 0.26 against a data median of 0.446. + + 1.0 = perfect cycle, 0 = no repeat structure. + """ + la = fk_pos(w, 17)[:, 2] + ra = fk_pos(w, 21)[:, 2] + sig = la - ra + sig = sig - sig.mean() + sd = float(sig.std()) + if sd < 1e-6: + return 0.0 + sig = sig / sd + n = len(sig) + best = 0.0 + for lag in range(8, max(9, n // 2)): + c = float((sig[:-lag] * sig[lag:]).mean()) + if c > best: + best = c + return best + + +def joint_hinge_signs(w): + """(worst knee, worst elbow) hinge angle, degrees. NOT CURRENTLY GATED ON. + + Requested as a guard against knees/elbows bending backwards, but it does + not work in the canonical frame and is left here documented rather than + silently enabled. Two sign conventions were tried and both FAILED against + real Mixamo clips: + - sign from the lateral axis (shoulder-left minus shoulder-right): that + axis flips between rigs, so 90% of the corpus read POSITIVE while the + real clip read negative — the gate would have dropped 90% of the data + (100% of march) on a convention mismatch alone. + - sign from "the shin swings backward relative to the thigh": false, + because mid-stride the shin legitimately swings AHEAD of the thigh; the + real walk then scored 75.7 and the real run 131.5, i.e. rejected. + An unsigned bound cannot work either: a real knee flexes 7..76 deg on a + walk and to 131 deg on a run, so "bent 75 forward" and "bent 75 backward" + are the same magnitude. A correct guard needs each rig's own bind-pose + hinge AXIS, which the canonical representation does not carry — it would + have to be computed in AnimationMerger against the target skeleton at + retarget time, not here. + + Knees and elbows are HINGES — they bend one way only. A negative value is + natural flexion in this frame and a positive one is HYPEREXTENSION (the + joint bending backwards), which reads as a broken limb. + + Convention measured on real Mixamo clips: knees run -75..-7 deg on a walk + and to -131 deg on a run (always negative = flexion); elbows sit near zero + while the arms hang. So the guard is on POSITIVE knee/elbow angles. + """ + def wdir(role): + d = qrot(w[:, role], np.broadcast_to(D_CANON[role], (len(w), 3))) + return d / (np.linalg.norm(d, axis=-1, keepdims=True) + 1e-9) + + # RIG-INDEPENDENT sign. A first cut took the sign from the lateral axis + # (shoulder-left minus shoulder-right), but that axis flips between rigs: + # 90% of the corpus scored a POSITIVE worst-knee while the real Mixamo clip + # scored negative, so the gate would have rejected 90% of the data (100% of + # march) purely from a convention mismatch. + # + # Anatomy gives a frame-free reference instead: a knee flexes so the SHIN + # swings BACKWARD relative to the thigh, i.e. the lower segment gains a + # component OPPOSITE the body's forward direction. Same for the forearm at + # the elbow. Forward comes from the shoulder line crossed with up, which is + # sign-stable because it is defined by the canonical +Y, not by which + # shoulder is "left". + up = np.array([0, 1, 0], np.float32) + lsh = qrot(w[:, 11], np.broadcast_to(D_CANON[11], (len(w), 3))) + rsh = qrot(w[:, 7], np.broadcast_to(D_CANON[7], (len(w), 3))) + lat = lsh - rsh + lat = lat / (np.linalg.norm(lat, axis=-1, keepdims=True) + 1e-9) + fwd = np.cross(lat, np.broadcast_to(up, lat.shape)) + fwd = fwd / (np.linalg.norm(fwd, axis=-1, keepdims=True) + 1e-9) + + def hyperextension(upper, lower): + """Degrees the hinge opens the WRONG way (0 when flexing naturally).""" + u, l = wdir(upper), wdir(lower) + ang = np.degrees(np.arccos(np.clip((u * l).sum(-1), -1, 1))) + # component of the lower segment along +forward, relative to the upper: + # flexion moves it backward (negative), hyperextension forward. + rel = ((l - u) * fwd).sum(-1) + # only count the angle when the joint opens forward + return np.where(rel > 0.05, ang, 0.0) + + knees = max(float(hyperextension(19, 20).max()), + float(hyperextension(15, 16).max())) + elbows = max(float(hyperextension(12, 13).max()), + float(hyperextension(8, 9).max())) + return knees, elbows + + def window_quality(action, w, valid): """True when the window meets the library curation bar.""" # Energy band — mean joint rotation speed (rad/frame). The upper bound is @@ -203,6 +309,12 @@ def window_quality(action, w, valid): if (action != "march" and valid[17] and valid[21] and foot_travel_ratio(w) < 2.0): return False + # Periodicity gate (v7.3): locomotion must actually CYCLE. 814 of the + # cache's locomotion windows clear 0.6 (walk 692/2704, run 43/76, + # march 79/340); the rest teach non-cyclic motion, which is what the + # model reproduced as trembling. + if valid[17] and valid[21] and gait_periodicity(w) < MIN_PERIODICITY: + return False # Travel-direction gate (v6.4): drop windows that move BACKWARD # relative to the body's own forward. See travel_forward(). # From 111a3714a4e54ca23e8cdb5b17b451e68f028003 Mon Sep 17 00:00:00 2001 From: Fernando Date: Wed, 26 Aug 2026 19:23:23 -0400 Subject: [PATCH 17/35] fix(anim): explicit periodicity loss + close the augmentation gate leak (#837) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gating the DATA to >= 0.6 periodicity was not enough. v7.3 trained only on cyclic windows and its output periodicity still fell 0.293 -> 0.247 by ep30 — no better than the v6.8 model that rendered as trembling. Same lesson as travel direction: flow matching does not inherit a property because the data has it. If it is not in the loss, it is not in the output. --period-weight adds a differentiable aperiodicity penalty: 1 minus the best autocorrelation of the ankle-scissor signal over lags 8..T/2, so it is agnostic to cadence. Validated: real Mixamo Walk 0.049, random noise 0.704, finite gradients. Building it exposed a REAL BUG in the prep. The loss scored the supposedly >=0.6-gated data at 0.43-0.51 aperiodicity, which should have been impossible. Two causes: - the torch lag range (10..29) did not match the numpy gate's (8..T/2) and so missed slower cadences. Now matched. - more seriously, the mirror and retime AUGMENTATIONS were added without re-running window_quality(), so augmented copies bypassed every gate. 18% of the "periodicity-gated" walk windows were actually below the bar (min 0.200) — retime() resamples the window and can break the cycle at its edges, which is exactly the property being gated on. Augmentations are now re-gated. Verified: walk windows go 584 (min 0.200, 18% below bar) -> 467 (min 0.601, 0% below bar). Note this leak silently weakened EVERY earlier gate too — uprightness, arm-hang, stride ratio, travel direction — since all of them were bypassed the same way. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/prep-t2m-v6.py | 12 +++++++++-- scripts/train-t2m-flow-v5.py | 42 ++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/scripts/prep-t2m-v6.py b/scripts/prep-t2m-v6.py index fdc23fc1c..1b5f38131 100644 --- a/scripts/prep-t2m-v6.py +++ b/scripts/prep-t2m-v6.py @@ -436,10 +436,18 @@ def windows(action, cq, valid): gated[0] += 1 continue add(action, w, valid) + # Augmented copies must clear the SAME gates as the base window. + # They previously bypassed window_quality entirely, so 18% of the + # "periodicity-gated" walk windows were actually below the 0.6 bar + # (min 0.200) — retime() resamples the window and can break the + # cycle at its edges, which is exactly the property being gated on. mw, mv = mirror(w, valid) - add(action, mw, mv) + if window_quality(action, mw, mv): + add(action, mw, mv) for f in (0.85, 1.15): - add(action, retime(w, f), valid) + rw_ = retime(w, f) + if window_quality(action, rw_, valid): + add(action, rw_, valid) if a.corpus: n = 0 diff --git a/scripts/train-t2m-flow-v5.py b/scripts/train-t2m-flow-v5.py index a1a9adcca..08a2dfabe 100644 --- a/scripts/train-t2m-flow-v5.py +++ b/scripts/train-t2m-flow-v5.py @@ -246,6 +246,40 @@ def jitter_excess(q, hi=None, lo=None): return mean_excess + worst_excess +def gait_aperiodicity(q, dirs): + """1 - (best autocorrelation of the ankle-scissor signal), differentiable. + + The property NO other term can see. A real Mixamo walk autocorrelates at + 0.968; the v6.8 model measured 0.264 and rendered as trembling in place + while scoring healthy on travel, contra, twist, amplitude and speed — all + of which are averages or correlations that non-cyclic twitching satisfies. + + Gating the DATA to >= 0.6 periodicity was not enough on its own: the model + still emitted 0.247-0.293, i.e. no better than before. Same lesson as travel + direction — flow matching does not inherit a property just because the data + has it. If it is not in the loss, it is not in the output. + + Autocorrelation is evaluated at a fixed lag set spanning plausible gait + periods (10..29 frames at 30 fps = 0.33..0.97 s) and the best is taken, so + the term is agnostic to cadence. + """ + la = fk_pos(q, 17, dirs)[..., 2] + ra = fk_pos(q, 21, dirs)[..., 2] + sig = la - ra + sig = sig - sig.mean(dim=1, keepdim=True) + sig = sig / (sig.std(dim=1, keepdim=True) + 1e-4) + T = sig.shape[1] + best = torch.full((sig.shape[0],), -1.0, device=q.device, dtype=sig.dtype) + # Lag range must MATCH prep-t2m-v6.gait_periodicity (8 .. T//2), otherwise + # the loss disagrees with the gate that selected the data: a narrower + # 10..29 window scored the >=0.6-periodic training set at 0.43-0.51 + # aperiodicity because it missed the slower cadences. + for lag in range(8, max(9, T // 2)): + c = (sig[:, :-lag] * sig[:, lag:]).mean(dim=1) + best = torch.maximum(best, c) + return (1.0 - best).clamp_min(0.0) + + def spine_twist_penalty(q): """Mean |hip->chest yaw| in radians — an anatomy guard for the phase loss. @@ -416,6 +450,10 @@ def main(): help="penalise per-frame angular speed above the data band " "(#837). Applied to ALL actions — jitter is never " "wanted. 0 = off; 1.0 is a sane start.") + ap.add_argument("--period-weight", type=float, default=0.0, + help="reward a periodic GAIT CYCLE on locomotion rows " + "(#837). Data gating alone does not produce one. " + "0 = off; 0.5 is a sane start.") ap.add_argument("--amp-weight", type=float, default=0.0, help="penalise limb excursion ABOVE human range (#837): " "the phase/travel hinges reward motion and otherwise " @@ -567,6 +605,10 @@ def main(): (g * (0.6 - ph).clamp_min(0.0)).sum() / denom) loss = loss + a.twist_weight * ( (g * tw_pen).sum() / denom) + if a.period_weight > 0: + ap = gait_aperiodicity(q_hat, dirs_t) + loss = loss + a.period_weight * ( + (g * ap).sum() / denom) if a.amp_weight > 0: # Floors are calibrated to WALK's data p5; march/run have # legitimately larger excursion (march stride p5 1.174 vs From 1cae71952c2f18be97722a92cddc34d6c752b6f8 Mon Sep 17 00:00:00 2001 From: Fernando Date: Wed, 26 Aug 2026 19:45:52 -0400 Subject: [PATCH 18/35] fix(anim): locomotion was only 7% of batches, so the gait losses never bit (#837) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The explicit periodicity loss did not move the metric either (0.234 at v7.4 ep15, versus 0.264 for the trembling v6.8 and 0.247 for data-gating alone). Before assuming the loss was wrong, I measured whether it was reaching any rows. It was not. The periodicity gate shrank walk/run/march to 467/39/41 windows out of 19693 while the other 21 actions kept theirs, so locomotion was just 7.1% of sampled batches — about 18 rows of 256. Every gait term (phase, travel, period, and the amp floor) is gated to locomotion rows and divided by their count, so they were structurally weak no matter what weight I gave them. The model was spending 93% of its capacity on non-locomotion actions. balance-power alone cannot fix this: at bp=1.0 every action gets an equal share, so the three locomotion actions cap at 3/24 = 12.5%. --loco-boost multiplies walk/run/march sampling weight directly. Measured shares: x4 -> 23.3%, x6 -> 31.3%, x10 -> 43.2%. Running x6, and the trainer now prints the achieved share at startup so this is never invisible again. This is the second time a fix looked ineffective when the real problem was that it never reached the data — the first was the amplitude term contributing exactly 0.0000 while jitter dominated at 80%. Measuring each term's actual contribution and reach should be the first step, not the last. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/train-t2m-flow-v5.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/scripts/train-t2m-flow-v5.py b/scripts/train-t2m-flow-v5.py index 08a2dfabe..95cca5b58 100644 --- a/scripts/train-t2m-flow-v5.py +++ b/scripts/train-t2m-flow-v5.py @@ -450,6 +450,10 @@ def main(): help="penalise per-frame angular speed above the data band " "(#837). Applied to ALL actions — jitter is never " "wanted. 0 = off; 1.0 is a sane start.") + ap.add_argument("--loco-boost", type=float, default=1.0, + help="multiply walk/run/march sampling weight (#837). The " + "periodicity gate leaves them ~7%% of batches, so the " + "gait losses barely reach any rows. 6.0 gives ~33%%.") ap.add_argument("--period-weight", type=float, default=0.0, help="reward a periodic GAIT CYCLE on locomotion rows " "(#837). Data gating alone does not produce one. " @@ -520,6 +524,23 @@ def main(): # kept falling. sqrt-tempering gives walk 16.1% and still leaves run 2.8%. freq = tk.sum(0) w = (tk @ (1.0 / np.maximum(freq, 1.0) ** a.balance_power)).astype(np.float64) + # LOCOMOTION BOOST. The periodicity gate shrank walk/run/march to 467/39/41 + # windows of 19693 while the other 21 actions kept theirs, so locomotion was + # only 7.1% of sampled batches (~18 rows of 256) — the gait losses reach + # only those rows, so they were weak in practice no matter their weight. + # balance-power alone caps locomotion at 12.5% (3 of 24 actions at bp=1.0), + # which is still far too little for the three actions that matter here. + if a.loco_boost > 1.0: + loco_names = {"walk", "run", "march"} + li = [i for i, n in enumerate(vocab) if n in loco_names] + if li: + boost = np.ones(len(vocab), np.float64) + for i in li: + boost[i] = a.loco_boost + w = w * (tk @ boost) + sh = w[(tk[:, li].sum(1) > 0)].sum() / w.sum() + print(f"locomotion sampling share: {100 * sh:.1f}% " + f"(boost x{a.loco_boost})", flush=True) sampler = torch.utils.data.WeightedRandomSampler( torch.from_numpy(w), num_samples=N, replacement=True) ds = torch.utils.data.TensorDataset(x1, m6, tok) From 96885d44141a57022842a77efe2f91a4e8a90c77 Mon Sep 17 00:00:00 2001 From: Fernando Date: Thu, 27 Aug 2026 01:51:11 -0400 Subject: [PATCH 19/35] fix(anim): gait periodicity measured wobble, not stride; normalise it (#837) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two bugs in the periodicity metric I introduced, both of which made it reward the very defect it was added to catch. 1. LAG FLOOR TOO LOW. The search started at 8 frames (0.27 s). The corpus's median best-lag came out at exactly 8 — the gate was accepting high-frequency WOBBLE as "periodic", and the model dutifully learned it, which is the trembling the user reported. Measured at true gait cadence (20-30 frames, one stride) the real Mixamo walk scores 0.968 while the gated corpus median is -0.379: the data is ANTI-correlated at real cadence. Restricted to GAIT_LAG_MIN..MAX in both the gate and the loss. 2. NOT ACTUALLY A CORRELATION. The score normalised by the whole window's std but averaged over the (n - lag)-sample overlap, so it was unbounded — a model scored 1.131, above the real walk's 0.968. An unbounded score also lets one large arc pass as a cycle. Now a Pearson correlation computed per overlap: real walk 1.000, real run 0.000. Also scopes periodicity to WALK only. The real Mixamo running clip has NO positive autocorrelation at any lag in a 60-frame window, so demanding periodicity of run/march asks for something real running does not exhibit at this window length. Corrected, v7.4 ep251 walk reaches 0.882 periodicity — a genuine cycle versus 0.264 for the trembling model — but the RENDER is still bad (arms fused into the torso, mesh collapsed). A clean cycle is necessary and nowhere near sufficient. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/prep-t2m-v6.py | 48 +++++++++++++++++++++++++----------- scripts/train-t2m-flow-v5.py | 27 ++++++++++++++------ 2 files changed, 54 insertions(+), 21 deletions(-) diff --git a/scripts/prep-t2m-v6.py b/scripts/prep-t2m-v6.py index 1b5f38131..bdc449c98 100644 --- a/scripts/prep-t2m-v6.py +++ b/scripts/prep-t2m-v6.py @@ -67,6 +67,9 @@ def load_module(name, fname): # minimum ankle-scissor autocorrelation for a locomotion window (see # gait_periodicity); real Mixamo Walk.fbx scores 0.968 MIN_PERIODICITY = 0.6 +# plausible gait-cycle lag window in frames at 30 fps (0.67 .. 1.0 s) +GAIT_LAG_MIN = 20 +GAIT_LAG_MAX = 30 # max tolerated BACKWARD bend (degrees) for the knee/elbow hinges; small # positive slack absorbs canonicalisation noise near full extension MAX_HYPEREXTEND_KNEE = 20.0 @@ -183,18 +186,32 @@ def gait_periodicity(w): la = fk_pos(w, 17)[:, 2] ra = fk_pos(w, 21)[:, 2] sig = la - ra - sig = sig - sig.mean() - sd = float(sig.std()) - if sd < 1e-6: - return 0.0 - sig = sig / sd n = len(sig) - best = 0.0 - for lag in range(8, max(9, n // 2)): - c = float((sig[:-lag] * sig[lag:]).mean()) + best = -1.0 + # Search only PLAUSIBLE GAIT CADENCES. A lag floor of 8 frames (0.27 s) was + # a serious bug: the corpus's median best-lag came out at exactly 8, i.e. + # the gate was accepting high-frequency WOBBLE as "periodic" and the model + # dutifully learned it — the trembling the user reported. Restricted to + # 20..30 frames (0.67..1.0 s at 30 fps, one full stride) the real Mixamo + # walk scores 0.968 while the corpus median is -0.379: the data is + # ANTI-correlated at true gait cadence. + # Pearson correlation computed PER OVERLAP. Normalising by the whole + # window's std while averaging over the (n - lag)-sample overlap is not a + # correlation and is not bounded: it scored a model at 1.131, above the real + # walk's 0.968, because the signal happened to be larger inside the overlap. + # An unbounded score also lets a single large arc pass as a "cycle". + for lag in range(GAIT_LAG_MIN, min(GAIT_LAG_MAX + 1, n // 2 + 1)): + a = sig[:-lag] + b = sig[lag:] + a = a - a.mean() + b = b - b.mean() + da, db = float(a.std()), float(b.std()) + if da < 1e-6 or db < 1e-6: + continue + c = float((a * b).mean() / (da * db)) if c > best: best = c - return best + return max(best, 0.0) def joint_hinge_signs(w): @@ -309,11 +326,14 @@ def window_quality(action, w, valid): if (action != "march" and valid[17] and valid[21] and foot_travel_ratio(w) < 2.0): return False - # Periodicity gate (v7.3): locomotion must actually CYCLE. 814 of the - # cache's locomotion windows clear 0.6 (walk 692/2704, run 43/76, - # march 79/340); the rest teach non-cyclic motion, which is what the - # model reproduced as trembling. - if valid[17] and valid[21] and gait_periodicity(w) < MIN_PERIODICITY: + # Periodicity gate: WALK ONLY. A stride autocorrelates at 20-30 frames + # (real Mixamo Walk scores 0.968 there), but the real Mixamo RUNNING + # clip has NO positive autocorrelation at any lag in a 60-frame window — + # its best values are negative — so periodicity is a walk-specific + # property here, not a universal gait one. Applying it to run/march + # would reject real motion. + if (action == "walk" and valid[17] and valid[21] + and gait_periodicity(w) < MIN_PERIODICITY): return False # Travel-direction gate (v6.4): drop windows that move BACKWARD # relative to the body's own forward. See travel_forward(). diff --git a/scripts/train-t2m-flow-v5.py b/scripts/train-t2m-flow-v5.py index 95cca5b58..47225f3bf 100644 --- a/scripts/train-t2m-flow-v5.py +++ b/scripts/train-t2m-flow-v5.py @@ -266,16 +266,22 @@ def gait_aperiodicity(q, dirs): la = fk_pos(q, 17, dirs)[..., 2] ra = fk_pos(q, 21, dirs)[..., 2] sig = la - ra - sig = sig - sig.mean(dim=1, keepdim=True) - sig = sig / (sig.std(dim=1, keepdim=True) + 1e-4) T = sig.shape[1] best = torch.full((sig.shape[0],), -1.0, device=q.device, dtype=sig.dtype) # Lag range must MATCH prep-t2m-v6.gait_periodicity (8 .. T//2), otherwise # the loss disagrees with the gate that selected the data: a narrower # 10..29 window scored the >=0.6-periodic training set at 0.43-0.51 # aperiodicity because it missed the slower cadences. - for lag in range(8, max(9, T // 2)): - c = (sig[:, :-lag] * sig[:, lag:]).mean(dim=1) + # Match prep-t2m-v6's GAIT_LAG_MIN/MAX. Searching from lag 8 rewarded + # high-frequency wobble instead of a stride (see gait_periodicity). + # Pearson per overlap (bounded), matching prep-t2m-v6.gait_periodicity. + for lag in range(20, min(31, max(21, T // 2 + 1))): + a = sig[:, :-lag] + b = sig[:, lag:] + a = a - a.mean(dim=1, keepdim=True) + b = b - b.mean(dim=1, keepdim=True) + c = ((a * b).mean(dim=1) + / ((a.std(dim=1) * b.std(dim=1)) + 1e-4)) best = torch.maximum(best, c) return (1.0 - best).clamp_min(0.0) @@ -627,9 +633,16 @@ def main(): loss = loss + a.twist_weight * ( (g * tw_pen).sum() / denom) if a.period_weight > 0: - ap = gait_aperiodicity(q_hat, dirs_t) - loss = loss + a.period_weight * ( - (g * ap).sum() / denom) + # WALK rows only — see prep-t2m-v6's periodicity gate: the + # real Mixamo run has no positive autocorrelation at any + # lag, so demanding it of run/march asks for something real + # running does not have. + walk_j = vocab.index("walk") if "walk" in vocab else -1 + if walk_j >= 0: + gp = tb_raw[:, walk_j].clamp(0, 1) + ap = gait_aperiodicity(q_hat, dirs_t) + loss = loss + a.period_weight * ( + (gp * ap).sum() / gp.sum().clamp_min(1.0)) if a.amp_weight > 0: # Floors are calibrated to WALK's data p5; march/run have # legitimately larger excursion (march stride p5 1.174 vs From 5e4c32b0b795178aee5541f07c5601eedb13e9fb Mon Sep 17 00:00:00 2001 From: Fernando Date: Thu, 27 Aug 2026 01:56:30 -0400 Subject: [PATCH 20/35] feat(anim): score t2m by DISTANCE TO REAL CLIPS instead of hand-made metrics (#837) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The six property metrics (travel, contra, twist, amplitude, speed, periodicity) can each be driven to match real motion while the render still collapses, and five of them turned out to have bugs that made them reward the defect they were added to catch. Real motion is a joint distribution, so measure against it directly. eval-t2m-refdist.py finds each generated clip's nearest real reference window in the canonical representation and reports the geodesic quaternion distance, minimised over reference windows and over cyclic time shifts (phase is not a defect). Real clips are short — Walk.fbx is exactly one 60-frame window and Running.fbx only 0.7 s — so references are cycle-extended and phase-shifted to cover the whole cycle. It also prints a real-vs-real floor, which is the honest target: walk real-vs-real 0.031 min / 0.437 median rad run real-vs-real 0.195 min / 0.891 median rad v6.8 walk 0.760 best / 1.887 mean (108 deg per joint) So the best draw of the best model is worse than the WORST pairing of two real windows. Validated that it tracks the renders, which no property metric did: on best-draw distance v6.8 ep75 (rendered best) scores 0.760 against v7.4 ep251 (rendered collapsed) at 1.173, while their mean distances are indistinguishable — so best-of-N is the signal to read. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/eval-t2m-refdist.py | 183 ++++++++++++++++++++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100644 scripts/eval-t2m-refdist.py diff --git a/scripts/eval-t2m-refdist.py b/scripts/eval-t2m-refdist.py new file mode 100644 index 000000000..8201de17e --- /dev/null +++ b/scripts/eval-t2m-refdist.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +# ruff: noqa: E702, E741 +"""Score a t2m model by DISTANCE TO REAL REFERENCE CLIPS (#837). + +ONE-TIME OFFLINE dev tool — NOT shipped. + +Why this exists. The hand-designed property metrics (travel direction, +contralateral phase, torso twist, limb amplitude, joint speed, gait +periodicity) can each be driven to match real motion while the render still +collapses: six independent hinges leave corners that satisfy all of them and +still look wrong, and five of those metrics turned out to have bugs that made +them reward the very defect they were added to catch. + +Real motion is a joint distribution, so compare against it directly. For each +generated clip, find its nearest real reference window in the canonical +representation and report that distance. One number, computed against ground +truth, that cannot be gamed by satisfying a property in isolation. + +Distance is per-frame mean geodesic quaternion angle over the valid canonical +joints, minimised over reference windows AND over a cyclic time shift of the +generated clip (phase is not a defect, so it should not be penalised). + +Usage: + python3 scripts/eval-t2m-refdist.py --model ~/t2m_v76/flow \ + [--refs "Walk.fbx=walk,Running.fbx=run"] [--samples 12] +""" +import argparse +import importlib.util +import json +import os +import subprocess +import tempfile + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +DOWNLOADS = os.path.expanduser("~/Downloads") + + +def load_module(name, fname): + spec = importlib.util.spec_from_file_location(name, os.path.join(HERE, fname)) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +prep5 = load_module("prep5", "prep-t2m-v5.py") + +J = 22 + + +def canon_from_fbx(path, cache_dir): + """Dump a real clip to canonical quats via the shipping extractor.""" + base = os.path.basename(path).replace(" ", "_") + ".canon.json" + out = os.path.join(cache_dir, base) + if not os.path.exists(out): + subprocess.run(["qtmesh", "anim", path, "--dump-canonical", out], + capture_output=True, text=True) + if not os.path.exists(out): + return None + d = json.load(open(out)) + clips = d.get("clips", []) + if not clips: + return None + c = clips[0] + cq, valid = prep5.canonicalize(np.asarray(c["quats"], np.float32), + c["restWorld"], c["restDir"]) + return cq, valid + + +def windows_of(cq, T=60, stride=5): + """Reference windows, cycle-extending short clips. + + The real clips are short — Walk.fbx is exactly 2.0 s (one 60-frame window) + and Running.fbx only 0.7 s — so a plain slice yields one or zero windows and + the distance has almost nothing to match against. Loop the clip and take + phase-shifted windows so every phase of the cycle is represented. + """ + n = len(cq) + if n < 2: + return [] + reps = int(np.ceil((T * 2) / n)) + 1 + ext = np.concatenate([cq] * reps, axis=0) + outs = [] + limit = min(len(ext) - T, max(n, T)) + for s in range(0, limit + 1, stride): + outs.append(ext[s:s + T]) + return outs + + +def geodesic_dist(a, b, valid): + """Mean per-frame per-joint geodesic angle (radians) between two clips.""" + dot = np.abs((a * b).sum(-1)).clip(0.0, 1.0) + ang = 2.0 * np.arccos(dot) # [T,J] + m = valid[None, :] + return float((ang * m).sum() / max(m.sum() * ang.shape[0], 1e-6)) + + +def best_distance(gen, refs, valid): + """Min over reference windows and over cyclic shifts of `gen`.""" + T = len(gen) + best = float("inf") + for r in refs: + for sh in range(0, T, 4): + g = np.roll(gen, sh, axis=0) + d = geodesic_dist(g, r, valid) + if d < best: + best = d + return best + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--model", required=True, help="dir with t2m.onnx + vocab") + ap.add_argument("--refs", default="Walk.fbx=walk,Running.fbx=run") + ap.add_argument("--samples", type=int, default=12) + ap.add_argument("--seed", type=int, default=11) + a = ap.parse_args() + + cache = os.path.join(tempfile.gettempdir(), "t2m_refcache") + os.makedirs(cache, exist_ok=True) + + refs = {} + valid_all = np.ones(J, np.float32) + for spec in a.refs.split(","): + if "=" not in spec: + continue + fname, action = spec.split("=", 1) + path = os.path.join(DOWNLOADS, fname.strip()) + if not os.path.exists(path): + print(f" (missing reference {fname})") + continue + got = canon_from_fbx(path, cache) + if got is None: + print(f" (could not extract {fname})") + continue + cq, valid = got + refs.setdefault(action.strip(), []).extend(windows_of(cq)) + valid_all = np.minimum(valid_all, valid) + if not refs: + raise SystemExit("no references extracted") + print(f"references: " + ", ".join(f"{k}({len(v)} windows)" + for k, v in refs.items())) + + mp = os.path.expanduser(a.model) + import onnxruntime as ort + so = ort.SessionOptions() + so.log_severity_level = 3 + sess = ort.InferenceSession(os.path.join(mp, "t2m.onnx"), so, + providers=["CPUExecutionProvider"]) + vocab = json.load(open(os.path.join(mp, "t2m-vocab.json")))["vocab"] + zd = int(sess.get_inputs()[1].shape[-1]) + rng = np.random.default_rng(a.seed) + + # self-distance floor: how close do DIFFERENT real windows get to each + # other? That is the best any model could plausibly score. + for act, rws in refs.items(): + if len(rws) > 1: + ds = [geodesic_dist(rws[i], rws[j], valid_all) + for i in range(len(rws)) for j in range(len(rws)) if i != j] + print(f" {act}: real-vs-real distance min {min(ds):.4f} " + f"median {np.median(ds):.4f} rad") + + print(f"\n{'action':8s} {'refDist(best)':>14s} {'refDist(mean)':>14s}") + for act, rws in refs.items(): + if act not in vocab: + print(f" {act:8s} not in vocab") + continue + t = np.zeros((1, len(vocab)), np.float32) + t[0, vocab.index(act)] = 1.0 + ds = [] + for _ in range(a.samples): + seed = (rng.standard_normal((1, zd)) * 0.5).astype(np.float32) + out = sess.run(None, {"tokens": t, "seed": seed})[0][0] + q = out.reshape(-1, J, 10)[:, :, 3:7] + q = q / (np.linalg.norm(q, axis=-1, keepdims=True) + 1e-12) + ds.append(best_distance(q, rws, valid_all)) + print(f" {act:8s} {min(ds):14.4f} {np.mean(ds):14.4f} rad " + f"({np.degrees(np.mean(ds)):.1f} deg/joint)") + + +if __name__ == "__main__": + main() From 4a4d92ec3658b65363d1a9e605255a79e1a0dbb0 Mon Sep 17 00:00:00 2001 From: Fernando Date: Thu, 27 Aug 2026 09:00:13 -0400 Subject: [PATCH 21/35] feat(anim): gate training windows by DISTANCE TO REAL CLIPS (#837) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The decisive measurement of this effort. Scored against real Mixamo clips, the corpus's own "walk" windows sit a median 1.73 rad from Walk.fbx — FURTHER than a real punch (1.19), a real dance (0.91), a real jump (0.66) or a real run (0.83), against a real-vs-real floor of 0.03..0.44. The corpus does not contain Mixamo-style walking, and every model trained on it faithfully reproduced that: the model measured 1.89 against its data's 1.73. That is why nine rounds of losses, gates and sampling changes never fixed the render — the gap lives in the data, and a model cannot be closer to real motion than its training set is. A walk-LIKE subset does exist, so select it: window_quality() now rejects walk/run windows whose geodesic distance to the real reference clip exceeds MAX_REFERENCE_DISTANCE (0.8 rad), minimised over reference windows and cyclic shifts so phase is not penalised. Calibrated from the data — 0.8 keeps 149 walk and 16 run windows, and real DIFFERENT motions sit 0.66+ away, so it admits only genuinely walk-shaped motion. This supersedes the periodicity gate, which was both too strict (149 windows survive here versus 15 under periodicity) and blind to whether the motion actually resembles a walk. Actions with no reference clip are unaffected. A missing or unextractable reference fails OPEN with a one-time warning rather than silently rejecting every window of that action, which would masquerade as a data problem. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/prep-t2m-v6.py | 111 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 102 insertions(+), 9 deletions(-) diff --git a/scripts/prep-t2m-v6.py b/scripts/prep-t2m-v6.py index bdc449c98..c5658d157 100644 --- a/scripts/prep-t2m-v6.py +++ b/scripts/prep-t2m-v6.py @@ -33,7 +33,9 @@ import json import os import re +import subprocess import sys +import tempfile import numpy as np @@ -67,6 +69,13 @@ def load_module(name, fname): # minimum ankle-scissor autocorrelation for a locomotion window (see # gait_periodicity); real Mixamo Walk.fbx scores 0.968 MIN_PERIODICITY = 0.6 +# real Mixamo clips used as the ground-truth shape for each action +REFERENCE_CLIPS = {"walk": "Walk.fbx", "run": "Running.fbx"} +# max geodesic distance (rad/joint) from the reference. Calibrated from the +# data: at 0.8 the pre-period-gate cache keeps 149 walk / 16 run windows, and +# real DIFFERENT motions sit 0.66 (jump) to 1.19 (punch) away, so 0.8 admits +# only genuinely walk-shaped motion. +MAX_REFERENCE_DISTANCE = 0.8 # plausible gait-cycle lag window in frames at 30 fps (0.67 .. 1.0 s) GAIT_LAG_MIN = 20 GAIT_LAG_MAX = 30 @@ -171,6 +180,80 @@ def fk_pos(w, role): return p +# ---- reference-distance gate (v7.6) ---- +# The decisive measurement of this whole effort: scored against real Mixamo +# clips, the corpus's own "walk" windows sit a median 1.73 rad from Walk.fbx — +# FURTHER than a real punch (1.19), a real dance (0.91) or a real run (0.83). +# Real-vs-real is 0.03..0.44. So the corpus does not contain Mixamo-style +# walking, and every model trained on it faithfully reproduced that: the model +# measured 1.89 against its data's 1.73. No loss, gate or sampling change can +# close a gap that lives in the data. But a walk-LIKE subset exists (149 windows +# within 0.8 rad), so gate on distance to the real clips and train on that. +_REF_CACHE = {} +_REF_WARNED = set() + + +def _reference_windows(action, T): + """Cycle-extended canonical windows of the real reference clip, or None.""" + if action in _REF_CACHE: + return _REF_CACHE[action] + fname = REFERENCE_CLIPS.get(action) + if not fname: + _REF_CACHE[action] = None + return None + path = os.path.expanduser(os.path.join("~/Downloads", fname)) + if not os.path.exists(path): + _REF_CACHE[action] = None + return None + cache_dir = os.path.join(tempfile.gettempdir(), "t2m_refcache") + os.makedirs(cache_dir, exist_ok=True) + out = os.path.join(cache_dir, + os.path.basename(path).replace(" ", "_") + ".canon.json") + if not os.path.exists(out): + subprocess.run(["qtmesh", "anim", path, "--dump-canonical", out], + capture_output=True, text=True) + if not os.path.exists(out): + _REF_CACHE[action] = None + return None + try: + clips = json.load(open(out)).get("clips", []) + c = clips[0] + cq, valid = prep5.canonicalize(np.asarray(c["quats"], np.float32), + c["restWorld"], c["restDir"]) + except Exception: + _REF_CACHE[action] = None + return None + n = len(cq) + reps = int(np.ceil((T * 2) / max(n, 1))) + 1 + ext = np.concatenate([cq] * reps, axis=0) + wins = [ext[s:s + T] for s in range(0, min(len(ext) - T, max(n, T)) + 1, 5)] + _REF_CACHE[action] = (wins, valid) + return _REF_CACHE[action] + + +def reference_distance(action, w): + """Min geodesic distance to the real reference clip; inf if no reference. + + Minimised over reference windows and over cyclic shifts of `w`, so phase + is not penalised — only the shape of the motion. + """ + got = _reference_windows(action, len(w)) + if not got: + return float("inf") + wins, valid = got + m = valid[None, :] + denom = max(float(m.sum()) * w.shape[0], 1e-6) + best = float("inf") + for sh in range(0, len(w), 6): + g = np.roll(w, sh, axis=0) + for r in wins: + dot = np.abs((g * r).sum(-1)).clip(0.0, 1.0) + d = float((2.0 * np.arccos(dot) * m).sum() / denom) + if d < best: + best = d + return best + + def gait_periodicity(w): """Strongest autocorrelation of the ankle-scissor signal at lag >= 8. @@ -326,15 +409,25 @@ def window_quality(action, w, valid): if (action != "march" and valid[17] and valid[21] and foot_travel_ratio(w) < 2.0): return False - # Periodicity gate: WALK ONLY. A stride autocorrelates at 20-30 frames - # (real Mixamo Walk scores 0.968 there), but the real Mixamo RUNNING - # clip has NO positive autocorrelation at any lag in a 60-frame window — - # its best values are negative — so periodicity is a walk-specific - # property here, not a universal gait one. Applying it to run/march - # would reject real motion. - if (action == "walk" and valid[17] and valid[21] - and gait_periodicity(w) < MIN_PERIODICITY): - return False + # REFERENCE-DISTANCE gate. Supersedes the periodicity gate, which was + # both too strict (it discarded genuinely walk-like windows: 149 survive + # at 0.8 rad here versus 15 in the periodicity-gated cache) and unable + # to see whether the motion actually resembles a walk. Actions with no + # reference clip are unaffected. + if action in REFERENCE_CLIPS: + rdst = reference_distance(action, w) + if not np.isfinite(rdst): + # No reference could be extracted (missing file / dump failure). + # Fail OPEN rather than silently rejecting every window of this + # action, and say so once — a silent total rejection would look + # like a data problem instead of a setup problem. + if action not in _REF_WARNED: + _REF_WARNED.add(action) + print(f"WARNING: no reference clip for '{action}' " + f"({REFERENCE_CLIPS[action]}) — distance gate SKIPPED", + flush=True) + elif rdst > MAX_REFERENCE_DISTANCE: + return False # Travel-direction gate (v6.4): drop windows that move BACKWARD # relative to the body's own forward. See travel_forward(). # From 7443cc09413fe1553b4844a875594b18bf28b89d Mon Sep 17 00:00:00 2001 From: Fernando Date: Thu, 27 Aug 2026 12:09:02 -0400 Subject: [PATCH 22/35] fix(anim): let the t2m vocab declare yaw180; the geometry heuristic cannot (#837) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit User report on the v7.7 walk: "moving well, but as if it was oriented backwards". Correct, and the cause is that detectBackwardFacing() infers facing from the MESH's toe region — a property of the RIG alone. It therefore returns the same answer for every model, while different t2m generations carry opposite canonical facing conventions and need opposite flips on the same mesh. Measured on the Rumba rig against the real Mixamo walk (geodesic rad, lower is better; real-vs-real floor is 0.437): heuristic's choice (unflipped) 0.492 mean over 3 samples flipped 0.430 mean over 3 samples So the heuristic picks the wrong one, and flipped lands AT the real-vs-real floor — the generated walk is now as close to the real clip as two real walk windows are to each other. detectBackwardFacing() now honours a "yaw180" boolean in the installed t2m-vocab.json, ahead of the geometry heuristic and behind the existing QTMESH_T2M_YAW180 env override. The model knows its own convention; the mesh cannot. Models that do not declare it keep the old behaviour exactly. Note this does NOT fix run (still crouched with arms out front, flipped or not) — that is the 16-training-window starvation, a separate problem. Co-Authored-By: Claude Opus 5 (1M context) --- src/AnimationMerger.cpp | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/AnimationMerger.cpp b/src/AnimationMerger.cpp index c03545a7f..d91d578ea 100644 --- a/src/AnimationMerger.cpp +++ b/src/AnimationMerger.cpp @@ -9,6 +9,10 @@ #include #include #include +#include +#include +#include +#include #include #include #include @@ -2043,12 +2047,46 @@ void AnimationMerger::debugDumpAnatomy(Ogre::Skeleton* skel, skel->_updateTransforms(); } +namespace { + +// Reads "yaw180" from the installed t2m vocab json, if present. +// Returns 1 (flip), 0 (do not flip) or -1 (not declared). +int declaredYaw180() +{ + static int cached = -2; + if (cached != -2) return cached; + cached = -1; + const QString path = + QStandardPaths::writableLocation(QStandardPaths::AppDataLocation) + + QLatin1String("/ai_models/motion/t2m-vocab.json"); + QFile f(path); + if (f.open(QIODevice::ReadOnly)) { + const QJsonObject o = + QJsonDocument::fromJson(f.readAll()).object(); + if (o.contains(QLatin1String("yaw180"))) + cached = o.value(QLatin1String("yaw180")).toBool() ? 1 : 0; + } + return cached; +} + +} // namespace + bool AnimationMerger::detectBackwardFacing(Ogre::Entity* entity) { // Escape hatch for exotic meshes where the foot-region heuristic guesses // wrong: QTMESH_T2M_YAW180=1 forces the flip, =0 disables it. const QByteArray force = qgetenv("QTMESH_T2M_YAW180"); if (!force.isEmpty()) return force != "0"; + // A MODEL-DECLARED flip wins over the geometry heuristic. The heuristic + // below reads the mesh's toe region, which is a property of the RIG alone — + // it therefore returns the same answer for every model, while different + // t2m generations can carry opposite canonical facing conventions and need + // opposite flips on the same mesh. Measured on the Rumba rig: the v7.7 + // model's clip scores 0.535 rad from the real Mixamo walk unflipped and + // 0.371 flipped (below the 0.437 real-vs-real floor), i.e. the heuristic + // picks the wrong one. So let the vocab say, and fall back to geometry. + if (const int decl = declaredYaw180(); decl >= 0) + return decl != 0; if (!entity || !entity->hasSkeleton()) return false; Ogre::SkeletonInstance* skel = entity->getSkeleton(); if (!skel) return false; From fe201c35a00ab3c9b674990ef4dd0b413d043daa Mon Sep 17 00:00:00 2001 From: Fernando Date: Thu, 27 Aug 2026 17:44:14 -0400 Subject: [PATCH 23/35] feat(anim): train the t2m model on the TEMPLATE clips, which face correctly (#837) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit User's observation, and it is the key one: "when I apply the good animations from the template, they walk in the correct side, couldn't they be used instead?" — as TRAINING DATA, yes. Measured, the curated template library is the only source that both renders with correct facing on real rigs and sits near real motion: refDist to the real Mixamo walk (real-vs-real floor 0.437): template walks 0.303 .. 0.462 <- below the floor CMU corpus walk 2.07 median v7.7 model 0.480 shoulder-line lateral error (real motion 0.483): template walks 0.519 CMU corpus 0.665 v7.7 model 0.618 That last block is why "facing awareness" could not be trained from the corpus: the model already faced BETTER than its own data, so there was nothing to learn from. The templates do carry the correct convention. --library-repeat emits each curated take N times so the 150 good clips are not drowned by the ~20k-window CMU corpus. At x40 the walk training data improves from 140 windows / refDist 0.639 / latErr 0.700 to 764 windows / refDist 0.309 / latErr 0.515 — i.e. below the real-vs-real floor and near real facing. This keeps text-to-motion a real generative model (prompt in, novel clip out, works on any rig) rather than falling back to template playback; the templates are only the teacher. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/prep-t2m-v6.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/scripts/prep-t2m-v6.py b/scripts/prep-t2m-v6.py index c5658d157..9b3ef904c 100644 --- a/scripts/prep-t2m-v6.py +++ b/scripts/prep-t2m-v6.py @@ -496,6 +496,11 @@ def main(): ap.add_argument("--T", type=int, default=60) ap.add_argument("--min-roles", type=int, default=12) ap.add_argument("--min-action-windows", type=int, default=16) + ap.add_argument("--library-repeat", type=int, default=1, + help="emit each curated-library take N times (#837). The " + "template clips face correctly and measure near real " + "motion; repeating them stops the far larger CMU " + "corpus from drowning them.") ap.add_argument("--exclude-sources", default=DEFAULT_EXCLUDE, help="regex; clips whose source matches are DROPPED " "(default: non-human gaits — zombies, produce). " @@ -582,9 +587,19 @@ def windows(action, cq, valid): continue cq, valid = prep5.canonicalize( np.asarray(c["quats"], np.float32), rw, rd) - windows(c["action"], cq, valid) + # The curated TEMPLATE clips are the only source that both renders + # with correct facing on real rigs and measures near real motion: + # refDist to the real Mixamo walk is 0.303-0.462 for the template + # walks (the real-vs-real floor is 0.437 and the corpus median is + # 2.07), and their shoulder-line lateral error is 0.519 vs the + # corpus's 0.665 and real motion's 0.483. --library-repeat emits + # each take that many times so this good data is not drowned by the + # much larger CMU corpus. + for _ in range(max(1, a.library_repeat)): + windows(c["action"], cq, valid) n += 1 - print(f"library: {n} takes → {len(mo)} windows (cum)") + print(f"library: {n} takes x{max(1, a.library_repeat)} " + f"→ {len(mo)} windows (cum)") if a.bvh and a.index: n = 0 for action, cq, valid, src in prep5.cmu_clips(a.bvh, a.index): From fb663e9561a19288fbe34841afe0a67802b50afc Mon Sep 17 00:00:00 2001 From: Fernando Date: Thu, 27 Aug 2026 21:43:08 -0400 Subject: [PATCH 24/35] feat(anim): checkpoint scorer over user-validated actions (#837) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Delivering "the last epoch" is not safe: in earlier runs walk peaked then regressed, and the metrics disagree — v7.8 ep75 has the best walk refDist (0.269) while ep30 has the best facing across the actions a human confirmed usable (0.326 vs ep75's 0.375). pick-t2m-checkpoint.py scores every archived checkpoint on walk refDist, walk facing, walk stride AND mean facing over the USER-VALIDATED action list, so the delivered model is chosen on evidence rather than recency. The validated list is real feedback on v7.8 ep60 (walk, run, wave, cough, death, pickup, attack, crouch, salute, working, shake, punch), and the script documents the caveat that the facing metric does not capture limb articulation — it separates checkpoints of one model, it does not replace watching renders. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/pick-t2m-checkpoint.py | 71 ++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 scripts/pick-t2m-checkpoint.py diff --git a/scripts/pick-t2m-checkpoint.py b/scripts/pick-t2m-checkpoint.py new file mode 100644 index 000000000..b4d7fdcf5 --- /dev/null +++ b/scripts/pick-t2m-checkpoint.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Score archived t2m checkpoints on the actions a USER validated (#837). + +ONE-TIME OFFLINE dev tool — NOT shipped. + +Picking "the last epoch" is wrong: walk peaked and then regressed in earlier +runs, and the metrics disagree with each other (ep75 has the best walk refDist +0.269 while ep30 has the best facing across the validated actions). So score +every archived checkpoint on BOTH, restricted to the actions a human actually +confirmed are usable, and choose deliberately. + +USER_GOOD below is real user feedback on v7.8 ep60, not a guess. Keep it in sync +when new feedback arrives — and note the caveat recorded in EVAL_NOTES: the +shoulder-line facing metric measures FACING only and does NOT capture limb +articulation, so it ranks actions unreliably. It informs the choice between +checkpoints of the SAME model; it does not replace looking at renders. +""" +import importlib.util,os,json,sys,glob,numpy as np,onnxruntime as ort,tempfile +sp=importlib.util.spec_from_file_location("rd","/Users/fernandotonon/QtMeshEditor/scripts/eval-t2m-refdist.py") +rd=importlib.util.module_from_spec(sp); sp.loader.exec_module(rd) +HERE="/Users/fernandotonon/QtMeshEditor/scripts" +s2=importlib.util.spec_from_file_location("p6",os.path.join(HERE,"prep-t2m-v6.py")); p6=importlib.util.module_from_spec(s2); s2.loader.exec_module(p6) +D=rd.prep5.D_CANON; PAR=p6.PAR +def pos(role,w): + T=len(w); p=np.zeros((T,3),np.float32); r=role + while PAR[r]>=0: + p=p+p6.qrot(w[:,PAR[r]],np.broadcast_to(D[r],(T,3))); r=PAR[r] + return p +def lat_err(q): + ls=p6.qrot(q[:,11],np.broadcast_to(D[11],(len(q),3))); rs=p6.qrot(q[:,7],np.broadcast_to(D[7],(len(q),3))) + v=ls-rs; v/=(np.linalg.norm(v,axis=-1,keepdims=True)+1e-9) + return float(np.sqrt(v[:,1]**2+v[:,2]**2).mean()) +def scissor(q): + la,ra=pos(17,q),pos(21,q); return float(np.abs(la[:,2]-ra[:,2]).max()) +cache=os.path.join(tempfile.gettempdir(),"t2m_refcache") +cqw,vw=rd.canon_from_fbx(os.path.expanduser("~/Downloads/Walk.fbx"),cache) +refs=rd.windows_of(cqw) +# actions the USER confirmed as good/usable on ep60 — these are what matter +USER_GOOD=["walk","run","wave","cough","death","pickup","attack","crouch", + "salute","working","shake","punch"] +so=ort.SessionOptions(); so.log_severity_level=3 +print(f"{'ckpt':10s} {'walk refD':>10s} {'walk latE':>10s} {'walk sciss':>11s} {'good-act latE':>14s}") +rows=[] +for d in sorted(glob.glob(os.path.expanduser("~/t2m_v78/e*"))): + f=os.path.join(d,"t2m.onnx") + if not os.path.exists(f): continue + s=ort.InferenceSession(f,so,providers=["CPUExecutionProvider"]) + vv=json.load(open(os.path.join(d,"t2m-vocab.json")))["vocab"]; zd=int(s.get_inputs()[1].shape[-1]) + def run(act,n): + rng=np.random.default_rng(7); t=np.zeros((1,len(vv)),np.float32) + if act not in vv: return None + t[0,vv.index(act)]=1.0 + R=[] + for _ in range(n): + out=s.run(None,{"tokens":t,"seed":(rng.standard_normal((1,zd))*0.5).astype(np.float32)})[0][0] + q=out.reshape(-1,22,10)[:,:,3:7]; q=q/np.linalg.norm(q,axis=-1,keepdims=True) + R.append(q) + return R + wq=run("walk",24) + wd=min(rd.best_distance(q,refs,vw) for q in wq) + wl=float(np.mean([lat_err(q) for q in wq])); ws=float(np.mean([scissor(q) for q in wq])) + ls=[] + for a in USER_GOOD: + qs=run(a,8) + if qs: ls.append(np.mean([lat_err(q) for q in qs])) + gl=float(np.mean(ls)) + print(f"{os.path.basename(d):10s} {wd:10.3f} {wl:10.3f} {ws:11.3f} {gl:14.3f}") + rows.append((gl,wd,os.path.basename(d))) +print("\nreal walk: refD floor 0.437 | latErr 0.483 | scissor 0.652") +rows.sort() +print(f"BEST by good-action facing: {rows[0][2]}") From 1be13c23d97c6b8f44699465605d3845afea3b37 Mon Sep 17 00:00:00 2001 From: Fernando Date: Thu, 27 Aug 2026 22:26:40 -0400 Subject: [PATCH 25/35] =?UTF-8?q?feat(t2m):=20leg-chain=20loss=20=E2=80=94?= =?UTF-8?q?=20keep=20the=20walk=20bend=20in=20the=20knee,=20not=20the=20an?= =?UTF-8?q?kle=20(#837)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit User report on the v7.8 ep60 walk: "almost like a flat knee, slightly backwards, then a second knee below on the correct orientation". Measured the leg chain against the real Mixamo walk and the bend is in the wrong joint: thigh->shin (knee) real 38.4 deg model 24.9 deg UNDER-bent shin->foot (ankle) real 9.7 deg model 31.5 deg 3x OVER-bent So it is not hyperextension (the earlier hinge-gate attempts assumed that and both failed) — the knee barely bends and the ankle takes up the slack. That extra bend below a near-straight knee is exactly the false second knee the user sees. leg_chain_penalty penalises ankle flexion above 0.35 rad and knee flexion below 0.45 rad, anchored on the real-walk values. Verified to separate: real Walk.fbx 0.014 training data 0.000 (walk) 0.021 (run) v7.8 ep60 0.222 Off by default (--legchain-weight 0). --- scripts/train-t2m-flow-v5.py | 37 ++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/scripts/train-t2m-flow-v5.py b/scripts/train-t2m-flow-v5.py index 47225f3bf..228bc64e3 100644 --- a/scripts/train-t2m-flow-v5.py +++ b/scripts/train-t2m-flow-v5.py @@ -286,6 +286,35 @@ def gait_aperiodicity(q, dirs): return (1.0 - best).clamp_min(0.0) +def leg_chain_penalty(q, dirs): + """Keep the leg's bend in the KNEE, not the ankle. + + User report on the v7.8 walk: "almost like a flat knee, slightly backwards, + then a second knee below on the correct orientation". Measured cause — the + bend is in the wrong joint: + + thigh->shin (knee) real 38.4 deg model 24.9 deg (UNDER-bent) + shin->foot (ankle) real 9.7 deg model 31.5 deg (3x OVER-bent) + + The over-bent ankle is what reads as a false second knee. Penalise ankle + flexion above a generous ceiling and knee flexion below a floor, both + anchored on the real-walk values, so the bend moves back up the chain. + """ + def wdir(role): + d = qrot_t(q[:, :, role], dirs[role].expand(q.shape[0], q.shape[1], 3)) + return F.normalize(d, dim=-1, eps=1e-6) + + pen = 0.0 + for hip, knee, foot in ((19, 20, 21), (15, 16, 17)): + th, sh, ft = wdir(hip), wdir(knee), wdir(foot) + knee_ang = torch.acos(((th * sh).sum(-1)).clamp(-1 + 1e-6, 1 - 1e-6)) + ankle_ang = torch.acos(((sh * ft).sum(-1)).clamp(-1 + 1e-6, 1 - 1e-6)) + # radians: real walk knee 0.67, ankle 0.17 + pen = pen + (ankle_ang.mean(dim=1) - 0.35).clamp_min(0.0) + pen = pen + (0.45 - knee_ang.mean(dim=1)).clamp_min(0.0) + return pen * 0.5 + + def spine_twist_penalty(q): """Mean |hip->chest yaw| in radians — an anatomy guard for the phase loss. @@ -464,6 +493,10 @@ def main(): help="reward a periodic GAIT CYCLE on locomotion rows " "(#837). Data gating alone does not produce one. " "0 = off; 0.5 is a sane start.") + ap.add_argument("--legchain-weight", type=float, default=0.0, + help="keep the leg bend in the KNEE not the ankle (#837): " + "an over-bent ankle reads as a false second knee. " + "0 = off; 0.3 is a sane start.") ap.add_argument("--amp-weight", type=float, default=0.0, help="penalise limb excursion ABOVE human range (#837): " "the phase/travel hinges reward motion and otherwise " @@ -643,6 +676,10 @@ def main(): ap = gait_aperiodicity(q_hat, dirs_t) loss = loss + a.period_weight * ( (gp * ap).sum() / gp.sum().clamp_min(1.0)) + if a.legchain_weight > 0: + lc = leg_chain_penalty(q_hat, dirs_t) + loss = loss + a.legchain_weight * ( + (g * lc).sum() / denom) if a.amp_weight > 0: # Floors are calibrated to WALK's data p5; march/run have # legitimately larger excursion (march stride p5 1.174 vs From 170bbc681e2fe1f7a855443db967cb8a0e161dca Mon Sep 17 00:00:00 2001 From: Fernando Date: Thu, 27 Aug 2026 22:31:53 -0400 Subject: [PATCH 26/35] fix(t2m): fail fast when a shared-block loss term would be silently dead MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The leg-chain, amplitude, travel and periodicity terms all live inside the `if a.phase_weight > 0` block because they need the reconstructed clean sample q_hat. With --phase-weight 0 they are therefore silently NO-OPS — the trainer accepts the flag, prints nothing, and trains without the term. This exact failure class already cost real runs once: the amplitude term contributed a measured 0.0000 to the loss for several full trainings before it was noticed, while limbs collapsed as collateral damage. So refuse the combination instead of no-op'ing, and log the leg-chain weight at startup the way the gait-phase term already does. --- scripts/train-t2m-flow-v5.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/scripts/train-t2m-flow-v5.py b/scripts/train-t2m-flow-v5.py index 228bc64e3..b649110f4 100644 --- a/scripts/train-t2m-flow-v5.py +++ b/scripts/train-t2m-flow-v5.py @@ -513,6 +513,17 @@ def main(): "sleep/restarts)") a = ap.parse_args() + # The leg-chain/amplitude/travel/period terms live inside the phase-loss + # block (they all need the reconstructed clean sample q_hat), so with + # --phase-weight 0 they are silently DEAD. That class of bug already cost + # several runs via the amplitude term, so refuse rather than no-op. + if a.phase_weight <= 0 and max(a.legchain_weight, a.amp_weight, + a.travel_weight, a.period_weight) > 0: + raise SystemExit( + "--legchain/amp/travel/period-weight require --phase-weight > 0 " + "(they share its reconstructed-sample block)") + + d = np.load(a.data, allow_pickle=True) mo, msk, tk = d["mo"], d["msk"], d["tk"] vocab = [str(w) for w in d["vocab"]] @@ -548,6 +559,9 @@ def main(): if w in fast_names: fast_mask_v[i] = 1.0 if a.phase_weight > 0: + if a.legchain_weight > 0: + print(f"leg-chain loss ON (w={a.legchain_weight}) — bend stays in " + f"the knee, not the ankle") print(f"gait-phase loss ON (w={a.phase_weight}) for " f"{[vocab[i] for i in loco_idx]}", flush=True) From 7ec81195292f8f9ae3bb9b11e86d177bdd4d787b Mon Sep 17 00:00:00 2001 From: Fernando Date: Thu, 27 Aug 2026 22:34:27 -0400 Subject: [PATCH 27/35] feat(anim): score checkpoints on the knee/ankle defect the user reported (#837) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The scorer measured facing, stride and reference distance — none of which capture the one thing the user actually named on the walk ('almost a flat knee, slightly backwards, then a second knee below'). Add the leg-chain metric (real Walk.fbx 0.014, data 0.000, v7.8 ep60 0.222) and report the best checkpoint per criterion instead of a single ranking, since the criteria genuinely disagree. Also scan the ck* archives, not just e*. --- scripts/pick-t2m-checkpoint.py | 37 +++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/scripts/pick-t2m-checkpoint.py b/scripts/pick-t2m-checkpoint.py index b4d7fdcf5..003dc462d 100644 --- a/scripts/pick-t2m-checkpoint.py +++ b/scripts/pick-t2m-checkpoint.py @@ -15,7 +15,7 @@ articulation, so it ranks actions unreliably. It informs the choice between checkpoints of the SAME model; it does not replace looking at renders. """ -import importlib.util,os,json,sys,glob,numpy as np,onnxruntime as ort,tempfile +import importlib.util,os,json,sys,glob,numpy as np,onnxruntime as ort,tempfile,torch sp=importlib.util.spec_from_file_location("rd","/Users/fernandotonon/QtMeshEditor/scripts/eval-t2m-refdist.py") rd=importlib.util.module_from_spec(sp); sp.loader.exec_module(rd) HERE="/Users/fernandotonon/QtMeshEditor/scripts" @@ -30,6 +30,20 @@ def lat_err(q): ls=p6.qrot(q[:,11],np.broadcast_to(D[11],(len(q),3))); rs=p6.qrot(q[:,7],np.broadcast_to(D[7],(len(q),3))) v=ls-rs; v/=(np.linalg.norm(v,axis=-1,keepdims=True)+1e-9) return float(np.sqrt(v[:,1]**2+v[:,2]**2).mean()) +sp3=importlib.util.spec_from_file_location("tr",os.path.join("/Users/fernandotonon/QtMeshEditor/scripts","train-t2m-flow-v5.py")) +_tr=importlib.util.module_from_spec(sp3); sp3.loader.exec_module(_tr) +_DIRS=torch.tensor(_tr.DIR_CANON,dtype=torch.float32) +def legchain(q): + """The user's reported 'second knee': bend in the ankle instead of the knee. + + Real Walk.fbx scores 0.014 and the training data 0.000, while v7.8 ep60 — + the checkpoint the user described as having 'almost a flat knee, slightly + backwards, then a second knee below' — scores 0.222. So this is the metric + that tracks the one defect the user named on the walk, and no other metric + here captures it (they all measure facing, phase or stride, never WHICH + joint bends). + """ + return float(_tr.leg_chain_penalty(torch.from_numpy(q[None]).float(),_DIRS).item()) def scissor(q): la,ra=pos(17,q),pos(21,q); return float(np.abs(la[:,2]-ra[:,2]).max()) cache=os.path.join(tempfile.gettempdir(),"t2m_refcache") @@ -39,9 +53,13 @@ def scissor(q): USER_GOOD=["walk","run","wave","cough","death","pickup","attack","crouch", "salute","working","shake","punch"] so=ort.SessionOptions(); so.log_severity_level=3 -print(f"{'ckpt':10s} {'walk refD':>10s} {'walk latE':>10s} {'walk sciss':>11s} {'good-act latE':>14s}") +print(f"{'ckpt':10s} {'walk refD':>10s} {'walk latE':>10s} {'walk sciss':>11s} " + f"{'walk legchn':>12s} {'good-act latE':>14s}") +print(" (lower) (real .483) (real ~1.0) (real .014) (lower)") rows=[] -for d in sorted(glob.glob(os.path.expanduser("~/t2m_v78/e*"))): +cands=sorted(glob.glob(os.path.expanduser("~/t2m_v78/e*")) + + glob.glob(os.path.expanduser("~/t2m_v78/ck*"))) +for d in cands: f=os.path.join(d,"t2m.onnx") if not os.path.exists(f): continue s=ort.InferenceSession(f,so,providers=["CPUExecutionProvider"]) @@ -59,13 +77,18 @@ def run(act,n): wq=run("walk",24) wd=min(rd.best_distance(q,refs,vw) for q in wq) wl=float(np.mean([lat_err(q) for q in wq])); ws=float(np.mean([scissor(q) for q in wq])) + wlc=float(np.mean([legchain(q) for q in wq])) ls=[] for a in USER_GOOD: qs=run(a,8) if qs: ls.append(np.mean([lat_err(q) for q in qs])) gl=float(np.mean(ls)) - print(f"{os.path.basename(d):10s} {wd:10.3f} {wl:10.3f} {ws:11.3f} {gl:14.3f}") - rows.append((gl,wd,os.path.basename(d))) -print("\nreal walk: refD floor 0.437 | latErr 0.483 | scissor 0.652") + print(f"{os.path.basename(d):10s} {wd:10.3f} {wl:10.3f} {ws:11.3f} " + f"{wlc:12.3f} {gl:14.3f}") + rows.append((gl,wd,wlc,os.path.basename(d))) +print("\nreal walk: refD floor 0.437 | latErr 0.483 | scissor 0.652 | legchain 0.014") rows.sort() -print(f"BEST by good-action facing: {rows[0][2]}") +print(f"BEST by good-action facing: {rows[0][3]}") +print(f"BEST by walk refDist: {sorted(rows,key=lambda r:r[1])[0][3]}") +print(f"BEST by knee/ankle (the user's reported walk defect): " + f"{sorted(rows,key=lambda r:r[2])[0][3]}") From 69b6a936a534e4beb1a495732cc51a26339f67c7 Mon Sep 17 00:00:00 2001 From: Fernando Date: Thu, 27 Aug 2026 22:57:24 -0400 Subject: [PATCH 28/35] =?UTF-8?q?fix(t2m):=20reject=20anatomically=20impos?= =?UTF-8?q?sible=20ankle=20folds=20=E2=80=94=20the=20broken=20march=20clip?= =?UTF-8?q?s=20(#837)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The user reported "march is just a twisted mess". Measured the curated training windows and the cause is the SOURCE DATA, not the model: action knee ankle walk 38.4 deg 9.7 deg (real Walk.fbx: 38.4 / 9.7) run - 21.7 deg march 39.4 deg 104.5 deg <- 100% of windows A 104.5 deg ankle folds the foot nearly perpendicular to the shin, in every single march window, while the knee is perfectly healthy. The march templates are simply broken. Left ungated this also poisons the new leg-chain loss, which is masked to ALL locomotion rows and would be pulled toward this geometry — so a bad action degrades the good ones. Gate at 70 deg mean ankle flexion (generous vs real walk 9.7 / run 21.7), skipped for actions where a large ankle angle is legitimate (kneeling, ground work, kick, climb). Verified selective on the current cache: walk 150/150 run 150/150 kick 150/150 sit 150/150 march 0/150 Unlike the two earlier knee-hinge gate attempts (which rejected ~90% of real data because the lateral axis sign flips between rigs), this bound is on an unsigned angle BETWEEN TWO BONES, so it is rig-independent. march drops out of the vocab as a result. That is the intent: a broken action is worse than a missing one. --- scripts/prep-t2m-v6.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/scripts/prep-t2m-v6.py b/scripts/prep-t2m-v6.py index 9b3ef904c..01edc9586 100644 --- a/scripts/prep-t2m-v6.py +++ b/scripts/prep-t2m-v6.py @@ -367,6 +367,18 @@ def hyperextension(upper, lower): return knees, elbows +def skip_ankle_check(action): + """Actions where a large ankle angle is legitimate (kneeling, ground work).""" + return action in HORIZONTAL_OK or action in ("sit", "crouch", "pray", + "kick", "climb") + + +def dir_of(w, role): + """World aim of `role` over the window, unit-normalised.""" + d = qrot(w[:, role], np.broadcast_to(D_CANON[role], (len(w), 3))) + return d / (np.linalg.norm(d, axis=-1, keepdims=True) + 1e-9) + + def window_quality(action, w, valid): """True when the window meets the library curation bar.""" # Energy band — mean joint rotation speed (rad/frame). The upper bound is @@ -380,6 +392,22 @@ def window_quality(action, w, valid): hi = 0.26 if action in FAST_ACTIONS else 0.11 if not (0.004 <= e <= hi): return False + # ANATOMICAL ankle bound. The foot cannot fold perpendicular to the shin, + # yet 100% of the curated `march` windows measured a 104.5 deg ankle bend + # (knee a healthy 39.4 deg) — the source march templates are simply broken, + # which is exactly the "march is a twisted mess" the user reported. Left + # ungated it also poisons the trainer's leg-chain loss, since that term is + # masked to ALL locomotion and would be pulled toward this geometry. + # Real walk ankle 9.7 deg, real run 21.7 deg; 70 deg is generous headroom + # for a genuine high-knee march or kick while still rejecting a fold. + if not skip_ankle_check(action): + for knee, foot in ((20, 21), (16, 17)): + if valid[knee] and valid[foot]: + kd = dir_of(w, knee) + fd = dir_of(w, foot) + cos = np.clip((kd * fd).sum(-1), -1.0, 1.0) + if float(np.degrees(np.arccos(cos)).mean()) > 70.0: + return False if action in HORIZONTAL_OK: return True floor = 0.7 if action in LOCOMOTION else 0.5 From ce96f77d68363a7bb3efdbeb813cf941abd1a0ea Mon Sep 17 00:00:00 2001 From: Fernando Date: Thu, 27 Aug 2026 22:59:57 -0400 Subject: [PATCH 29/35] feat(t2m): stamp the build command into the training cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The v7.8 npz recorded only fps/vocab/canonRestDir. When the march contamination needed the cache rebuilt, reproducing the original build meant inferring the flags by comparing the cache's action set against each candidate motion-library file — slow, and easy to get wrong in a way that silently changes the training data. Store sys.argv and the resolved flag dict so any cache can state how it was built. --- scripts/prep-t2m-v6.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/prep-t2m-v6.py b/scripts/prep-t2m-v6.py index 01edc9586..5e13694f3 100644 --- a/scripts/prep-t2m-v6.py +++ b/scripts/prep-t2m-v6.py @@ -654,9 +654,14 @@ def windows(action, cq, valid): print(f"windows: {mo.shape[0]} vocab({len(vocab)}):", {w: int(tk[:, vocab.index(w)].sum()) for w in vocab}) os.makedirs(os.path.dirname(os.path.abspath(a.out)), exist_ok=True) + # Stamp the build command into the cache. The v7.8 npz recorded only + # fps/vocab/canonRestDir, so reproducing it later meant inferring the flags + # from action-overlap against the library files — slow and error-prone. np.savez_compressed(a.out, mo=mo, msk=msk, tk=tk, vocab=np.array(vocab), fps=FPS, - canonRestDir=D_CANON) + canonRestDir=D_CANON, + buildArgv=np.array(sys.argv, dtype=object), + buildFlags=np.array(json.dumps(vars(a)), dtype=object)) print(f"wrote {a.out} mo{mo.shape}") From f5fa7ef6e3ed10d7678894885ba7963ed6457fcc Mon Sep 17 00:00:00 2001 From: Fernando Date: Thu, 27 Aug 2026 23:04:25 -0400 Subject: [PATCH 30/35] fix(t2m): apply --min-roles to the curated library, not just the corpus MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --min-roles was only ever passed to the CORPUS loader. Once --library-repeat made the curated library the dominant source, the flag was effectively inert — the same silently-dead-knob class as the amplitude term. It matters most on the library path. The `hey` take resolves only 9 of 22 canonical roles with ALL FOUR knee/ankle roles invalid, so the new anatomical ankle gate cannot even run on it (it correctly skips a check it cannot perform) and the model has no leg data for that action — it invents legs. The 90 deg 'ankle' measured there is unmapped identity quaternions, not motion. Sorting the curated takes by valid-role count reproduces the user's verdict without being told which actions were bad: hey 9 roles "not that good" confession 14 roles "not that good" dance min 6 "a twisted mess" everything rated GOOD >=16 roles A floor of 16 costs 11 of 150 clips and drops exactly confession/hey (plus show/succ, never exercised). Combined with the anatomical ankle gate, the rebuilt cache goes 34 -> 30 actions, removing precisely the four the user called broken — march, throw, hey, confession — while retaining every action they rated good or usable. --- scripts/prep-t2m-v6.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/scripts/prep-t2m-v6.py b/scripts/prep-t2m-v6.py index 5e13694f3..0c2cdd8b2 100644 --- a/scripts/prep-t2m-v6.py +++ b/scripts/prep-t2m-v6.py @@ -607,6 +607,7 @@ def windows(action, cq, valid): if a.library: lib = json.load(open(os.path.expanduser(a.library))) n = 0 + dropped_roles = [0] for c in lib.get("clips", []): rw, rd = c.get("restWorld"), c.get("restDir") if not rw or not rd: @@ -615,6 +616,19 @@ def windows(action, cq, valid): continue cq, valid = prep5.canonicalize( np.asarray(c["quats"], np.float32), rw, rd) + # --min-roles was only ever applied to the CORPUS loader, so with + # --library-repeat making the library the dominant source the flag + # was effectively inert (another silently-dead knob, same class as + # the amplitude term). It matters most here: the `hey` take resolves + # only 9 of 22 roles with ALL FOUR knee/ankle roles invalid, so the + # anatomical ankle gate cannot even run on it and the model has no + # leg data to learn from — it invents legs. Sorting the curated + # takes by valid-role count reproduces the user's verdict: `hey` 9 + # and `confession` 14 are the two worst and the two they called + # "not that good", while everything they rated GOOD has >=16. + if int(valid.sum()) < a.min_roles: + dropped_roles[0] += 1 + continue # The curated TEMPLATE clips are the only source that both renders # with correct facing on real rigs and measures near real motion: # refDist to the real Mixamo walk is 0.303-0.462 for the template @@ -628,6 +642,9 @@ def windows(action, cq, valid): n += 1 print(f"library: {n} takes x{max(1, a.library_repeat)} " f"→ {len(mo)} windows (cum)") + if dropped_roles[0]: + print(f"library: dropped {dropped_roles[0]} takes below " + f"--min-roles {a.min_roles}") if a.bvh and a.index: n = 0 for action, cq, valid, src in prep5.cmu_clips(a.bvh, a.index): From 948a3e12c302bbab85f1102a86f072a491fa39d7 Mon Sep 17 00:00:00 2001 From: Fernando Date: Thu, 27 Aug 2026 23:07:22 -0400 Subject: [PATCH 31/35] feat(t2m): support warm-starting from weights with a different vocab MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removing the four broken actions (march/throw/hey/confession) changed the vocab 34 -> 30, which resizes act_emb.weight — the only tensor carrying the vocab dimension. A plain --resume would raise on the shape mismatch, so the 90 epochs of template-facing knowledge could not be carried over. Transplant act_emb column-by-column for the 30 shared actions (order is preserved, so all 30 map) and drop the optimizer/scheduler state, whose moment buffers are shaped for the old tensor and would be stale. --resume now detects a checkpoint without opt/sched and warm-starts with a fresh optimizer instead of KeyError-ing. Verified on the real transition: 30/30 embeddings transplanted, act_emb (384,34) -> (384,30), and the trainer reports the locomotion mask as ['run','walk'] — march is gone, so the leg-chain loss is no longer 21% mis-targeted at broken geometry. --- scripts/train-t2m-flow-v5.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/scripts/train-t2m-flow-v5.py b/scripts/train-t2m-flow-v5.py index b649110f4..7bebcadb4 100644 --- a/scripts/train-t2m-flow-v5.py +++ b/scripts/train-t2m-flow-v5.py @@ -614,10 +614,19 @@ def main(): # epoch — no pickled objects — so load safely (no code execution). ck = torch.load(ckpt_path, map_location=dev, weights_only=True) net.load_state_dict(ck["net"]) - opt.load_state_dict(ck["opt"]) - sched.load_state_dict(ck["sched"]) - start_ep = ck["epoch"] + 1 - print(f"resumed from epoch {start_ep}", flush=True) + # A WARM START (weights transplanted from a run with a different vocab) + # deliberately carries no optimizer/scheduler state: their moment + # buffers are shaped for the OLD act_emb and would be stale or + # shape-mismatched. Restore them only when present. + if "opt" in ck and "sched" in ck: + opt.load_state_dict(ck["opt"]) + sched.load_state_dict(ck["sched"]) + start_ep = ck["epoch"] + 1 + print(f"resumed from epoch {start_ep}", flush=True) + else: + start_ep = ck.get("epoch", 0) + print(f"WARM START from transplanted weights at epoch {start_ep} " + f"(fresh optimizer/scheduler)", flush=True) for ep in range(start_ep, a.epochs): # Accumulate the epoch loss ON DEVICE. A per-batch `loss.item()` is a From 2889710d19c1e1efb9d1c9eb10cf1839efa76582 Mon Sep 17 00:00:00 2001 From: Fernando Date: Fri, 28 Aug 2026 22:36:27 -0400 Subject: [PATCH 32/35] =?UTF-8?q?fix(anim):=20route=20march/marching=20to?= =?UTF-8?q?=20walk=20=E2=80=94=20it=20has=20neither=20a=20model=20action?= =?UTF-8?q?=20nor=20a=20clip=20(#837)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dropping the broken `march` action from the t2m vocab left "marching" with nothing to resolve to: the curated library has NEVER contained a march clip either, so the model path returned "not in the model vocabulary", the template fallback then found no match, and the user got a hard error for a prompt that previously produced something. march's training data was leftover CMU corpus whose ankle folds ~106 deg — the foot perpendicular to the shin in 100% of windows — which is why it rendered as a twisted mess and was gated out. throw/hey/confession were also dropped from the vocab but DO have curated clips, so they fall back cleanly; march was the only one left stranded. A marching gait is a walk, so point both synonym tables at `walk`. Verified: "marching" and "march" now generate via the model as walk instead of erroring. --- src/MotionGenerator.cpp | 6 +++++- src/MotionLibrary.cpp | 9 ++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/MotionGenerator.cpp b/src/MotionGenerator.cpp index 8e862d8a5..d654b446f 100644 --- a/src/MotionGenerator.cpp +++ b/src/MotionGenerator.cpp @@ -42,7 +42,11 @@ const Syn kSynonyms[] = { {"walking", "walk"}, {"stroll", "walk"}, {"leap", "jump"}, {"hop", "jump"}, {"dancing", "dance"}, {"punching", "punch"}, {"kicking", "kick"}, - {"waving", "wave"}, {"marching", "march"}, {"climbing", "climb"}, + // "marching" -> walk: the march training data was broken (ankle folded + // ~106 deg) and was dropped from the vocab in #837, and no curated march + // clip exists either, so route it to the nearest good gait. + {"waving", "wave"}, {"marching", "walk"}, {"march", "walk"}, + {"climbing", "climb"}, {"idle", "idle"}, {"stand", "idle"}, {"standing", "idle"}, {"sitting", "sit"}, {"seat", "sit"}, {"throwing", "throw"}, {"toss", "throw"}, {"pitch", "throw"}, diff --git a/src/MotionLibrary.cpp b/src/MotionLibrary.cpp index c84b4125f..74949544c 100644 --- a/src/MotionLibrary.cpp +++ b/src/MotionLibrary.cpp @@ -35,7 +35,14 @@ const Syn kSynonyms[] = { {"walking", "walk"}, {"stroll", "walk"}, {"step", "walk"}, {"jumping", "jump"}, {"hop", "jump"}, {"leap", "jump"}, {"dancing", "dance"}, {"spin", "dance"}, - {"marching", "march"}, + // "march"/"marching" resolve to WALK, not to a `march` action. The curated + // library has never contained a march clip, and the t2m model's march was + // trained on leftover CMU corpus whose ankle folds ~106 deg (the foot + // perpendicular to the shin in 100% of windows) — it rendered as a twisted + // mess and was dropped from the model vocab in #837. With no clip and no + // vocab entry, "marching" resolved to nothing and returned a hard error, so + // point it at the nearest good action instead: a marching gait IS a walk. + {"marching", "walk"}, {"march", "walk"}, {"kicking", "kick"}, {"punching", "punch"}, {"strike", "punch"}, {"hit", "punch"}, {"waving", "wave"}, {"greet", "wave"}, {"hello", "wave"}, From 8bb417c14db20e4df11eda90466cbf9ee7ad4d49 Mon Sep 17 00:00:00 2001 From: Fernando Date: Sat, 29 Aug 2026 02:34:03 -0400 Subject: [PATCH 33/35] docs(anim): correct the yaw180 hook rationale for the shipped v8.0 model The comment cited v7.7 measurements to justify the hook, but that model is superseded. No shipped vocab declares `yaw180`: v8.0 trains on the curated template clips, which already carry the correct facing, so the geometry heuristic agrees and this returns -1. Keep the hook as forward compatibility (facing convention is a property of the MODEL, and only the model can state it) and say so, rather than leaving stale evidence for a future reader to act on. --- src/AnimationMerger.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/AnimationMerger.cpp b/src/AnimationMerger.cpp index d91d578ea..15694452c 100644 --- a/src/AnimationMerger.cpp +++ b/src/AnimationMerger.cpp @@ -2081,10 +2081,14 @@ bool AnimationMerger::detectBackwardFacing(Ogre::Entity* entity) // below reads the mesh's toe region, which is a property of the RIG alone — // it therefore returns the same answer for every model, while different // t2m generations can carry opposite canonical facing conventions and need - // opposite flips on the same mesh. Measured on the Rumba rig: the v7.7 - // model's clip scores 0.535 rad from the real Mixamo walk unflipped and - // 0.371 flipped (below the 0.437 real-vs-real floor), i.e. the heuristic - // picks the wrong one. So let the vocab say, and fall back to geometry. + // opposite flips on the same mesh. + // + // No shipped vocab declares `yaw180` today: the v8.0 model is trained on + // the curated template clips, which already carry the correct facing, so + // the geometry heuristic agrees with it and this returns -1. The hook stays + // as forward compatibility for a future model trained on a corpus with the + // opposite convention — that is a property of the MODEL, and only the model + // can state it. Result is cached, so this costs one file read per process. if (const int decl = declaredYaw180(); decl >= 0) return decl != 0; if (!entity || !entity->hasSkeleton()) return false; From b48ec35dbd940cf202d236cdcfd47a1666d37fce Mon Sep 17 00:00:00 2001 From: Fernando Date: Sat, 29 Aug 2026 02:50:14 -0400 Subject: [PATCH 34/35] review: address CodeRabbit/Codex findings on #963 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **P1 — remove the model-declared yaw180 hook entirely.** detectBackwardFacing is called from SEVEN sites, four of them mocap (MocapRecorder x3, MocapController). Letting an installed t2m vocab declare the flip would apply a MODEL-specific facing convention to video-captured motion that never used it — installing a model could rotate unrelated animations 180 degrees. It was latent (no shipped vocab declares the key) and the delivered v8.0 does not need it, so delete rather than add scoping for a hypothetical. AnimationMerger.cpp is now byte-identical to master; the per-process QTMESH_T2M_YAW180 env override stays. **P2 — the mirror 'fix' was a no-op, and the original was already correct.** (-x, y, z, -w) is the NEGATION of (x, -y, -z, w), and q == -q as rotations. Verified numerically that (x, -y, -z, w) IS the conjugation S@R(q)@S with S = diag(-1,1,1), while the two alternatives are not. Restored the correct form and documented the no-op so it is not reintroduced as a fix. **Security — allow_pickle=False on every cache reader.** A crafted npz passed via --data could otherwise execute code on load. Note the blanket flip alone would have BROKEN reading caches written by this branch: the new buildArgv/buildFlags were dtype=object, which forces allow_pickle=True. Store them as np.str_ instead, then flip the three readers. **Guidance defaults reconciled on 1.0, not 2.0.** The review asked to raise the CLI defaults to match Sampler's 2.0. The shipped v8.0 model was trained AND exported with an explicit --guidance 1.0, so that would make the documented reproduction command produce a different model than the one published. Fixed the Sampler default down to 1.0 instead, so all three agree on the shipped value. Verified: re-exporting ep21 reproduces sha 7690b9f63567 bit-for-bit. **Portability — pick-t2m-checkpoint.py no longer hardcodes a home directory.** Module imports derive from __file__; checkpoint roots and the reference clip are --roots/--ref arguments. Verified against ~/t2m_v80. --- scripts/eval-t2m-posture.py | 2 +- scripts/export-t2m-from-ckpt.py | 6 +++- scripts/pick-t2m-checkpoint.py | 53 ++++++++++++++++++++++++++------- scripts/prep-t2m-v6.py | 25 ++++++++++------ scripts/train-t2m-flow-v5.py | 8 +++-- src/AnimationMerger.cpp | 42 -------------------------- 6 files changed, 71 insertions(+), 65 deletions(-) diff --git a/scripts/eval-t2m-posture.py b/scripts/eval-t2m-posture.py index 64b368fcd..c35c976a8 100644 --- a/scripts/eval-t2m-posture.py +++ b/scripts/eval-t2m-posture.py @@ -179,7 +179,7 @@ def main(): # ---- data reference (what the model is trying to match) ---- if a.data: - z = np.load(os.path.expanduser(a.data), allow_pickle=True) + z = np.load(os.path.expanduser(a.data), allow_pickle=False) mo, tk = z["mo"], z["tk"] dvocab = [str(s) for s in z["vocab"]] print("\n=== TRAINING DATA (reference) ===") diff --git a/scripts/export-t2m-from-ckpt.py b/scripts/export-t2m-from-ckpt.py index 64bad3c8f..aa599d84d 100644 --- a/scripts/export-t2m-from-ckpt.py +++ b/scripts/export-t2m-from-ckpt.py @@ -45,10 +45,14 @@ def main(): ap.add_argument("--data", required=True, help="npz (for vocab + canonRestDir)") ap.add_argument("--out", required=True) ap.add_argument("--steps", type=int, default=24) + # 1.0 = the value the SHIPPED v8.0 model was exported with. A mismatch + # here silently changes generated motion and invalidates checkpoint + # comparisons against it, so keep all three defaults (this, the trainer + # flag, and Sampler.__init__) on 1.0. ap.add_argument("--guidance", type=float, default=1.0) a = ap.parse_args() - z = np.load(os.path.expanduser(a.data), allow_pickle=True) + z = np.load(os.path.expanduser(a.data), allow_pickle=False) vocab = [str(s) for s in z["vocab"]] fps = int(z["fps"]) if "fps" in z else 30 canon_rd = z["canonRestDir"] diff --git a/scripts/pick-t2m-checkpoint.py b/scripts/pick-t2m-checkpoint.py index 003dc462d..ad30c6f81 100644 --- a/scripts/pick-t2m-checkpoint.py +++ b/scripts/pick-t2m-checkpoint.py @@ -15,11 +15,29 @@ articulation, so it ranks actions unreliably. It informs the choice between checkpoints of the SAME model; it does not replace looking at renders. """ -import importlib.util,os,json,sys,glob,numpy as np,onnxruntime as ort,tempfile,torch -sp=importlib.util.spec_from_file_location("rd","/Users/fernandotonon/QtMeshEditor/scripts/eval-t2m-refdist.py") -rd=importlib.util.module_from_spec(sp); sp.loader.exec_module(rd) -HERE="/Users/fernandotonon/QtMeshEditor/scripts" -s2=importlib.util.spec_from_file_location("p6",os.path.join(HERE,"prep-t2m-v6.py")); p6=importlib.util.module_from_spec(s2); s2.loader.exec_module(p6) +import argparse +import glob +import importlib.util +import json +import os +import tempfile + +import numpy as np +import onnxruntime as ort +import torch + +HERE = os.path.dirname(os.path.abspath(__file__)) + + +def _load(name, fname): + spec = importlib.util.spec_from_file_location(name, os.path.join(HERE, fname)) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +rd = _load("rd", "eval-t2m-refdist.py") +p6 = _load("p6", "prep-t2m-v6.py") D=rd.prep5.D_CANON; PAR=p6.PAR def pos(role,w): T=len(w); p=np.zeros((T,3),np.float32); r=role @@ -30,8 +48,7 @@ def lat_err(q): ls=p6.qrot(q[:,11],np.broadcast_to(D[11],(len(q),3))); rs=p6.qrot(q[:,7],np.broadcast_to(D[7],(len(q),3))) v=ls-rs; v/=(np.linalg.norm(v,axis=-1,keepdims=True)+1e-9) return float(np.sqrt(v[:,1]**2+v[:,2]**2).mean()) -sp3=importlib.util.spec_from_file_location("tr",os.path.join("/Users/fernandotonon/QtMeshEditor/scripts","train-t2m-flow-v5.py")) -_tr=importlib.util.module_from_spec(sp3); sp3.loader.exec_module(_tr) +_tr = _load("tr", "train-t2m-flow-v5.py") _DIRS=torch.tensor(_tr.DIR_CANON,dtype=torch.float32) def legchain(q): """The user's reported 'second knee': bend in the ankle instead of the knee. @@ -46,8 +63,19 @@ def legchain(q): return float(_tr.leg_chain_penalty(torch.from_numpy(q[None]).float(),_DIRS).item()) def scissor(q): la,ra=pos(17,q),pos(21,q); return float(np.abs(la[:,2]-ra[:,2]).max()) +_ap = argparse.ArgumentParser(description=__doc__) +_ap.add_argument("--roots", default=os.path.expanduser("~/t2m_v78"), + help="comma-separated checkpoint roots to scan (each is " + "globbed for e*/ and ck*/ subdirs holding t2m.onnx)") +_ap.add_argument("--ref", default=os.path.expanduser("~/Downloads/Walk.fbx"), + help="real reference walk clip to score against") +_a = _ap.parse_args() + cache=os.path.join(tempfile.gettempdir(),"t2m_refcache") -cqw,vw=rd.canon_from_fbx(os.path.expanduser("~/Downloads/Walk.fbx"),cache) +_ref = os.path.expanduser(_a.ref) +if not os.path.exists(_ref): + raise SystemExit(f"reference clip not found: {_ref} (pass --ref)") +cqw,vw=rd.canon_from_fbx(_ref,cache) refs=rd.windows_of(cqw) # actions the USER confirmed as good/usable on ep60 — these are what matter USER_GOOD=["walk","run","wave","cough","death","pickup","attack","crouch", @@ -57,8 +85,13 @@ def scissor(q): f"{'walk legchn':>12s} {'good-act latE':>14s}") print(" (lower) (real .483) (real ~1.0) (real .014) (lower)") rows=[] -cands=sorted(glob.glob(os.path.expanduser("~/t2m_v78/e*")) - + glob.glob(os.path.expanduser("~/t2m_v78/ck*"))) +cands = [] +for _root in _a.roots.split(","): + _root = os.path.expanduser(_root.strip()) + cands += glob.glob(os.path.join(_root, "e*")) + glob.glob(os.path.join(_root, "ck*")) +cands = sorted(set(cands)) +if not cands: + raise SystemExit(f"no checkpoint dirs under {_a.roots} (pass --roots)") for d in cands: f=os.path.join(d,"t2m.onnx") if not os.path.exists(f): continue diff --git a/scripts/prep-t2m-v6.py b/scripts/prep-t2m-v6.py index 0c2cdd8b2..d9577b714 100644 --- a/scripts/prep-t2m-v6.py +++ b/scripts/prep-t2m-v6.py @@ -487,14 +487,18 @@ def mirror(w, valid): Flow matching then samples both modes and the model walks backwards about half the time (#837, user-reported). - A true left<->right mirror must flip only the LATERAL axis: negate X and W - (equivalently reflect about the YZ plane, normal Z... see below) while - preserving the forward and up axes. For quats under a reflection about the - plane normal to X, the improper transform is applied as q -> (-x, y, z, -w) - combined with the L/R role swap, which preserves handedness of the - forward/up frame and therefore the direction of travel. + A true left<->right mirror reflects about the YZ plane (negating the LATERAL + X axis) and swaps the L/R joint roles. Conjugating a rotation by that + reflection, S @ R(q) @ S with S = diag(-1, 1, 1), is exactly q -> (x, -y, + -z, w), which is what this does. + + NB an earlier revision "fixed" this to (-x, y, z, -w). That is the NEGATION + of the same quaternion, and q and -q are the SAME rotation, so it changed + nothing — verified numerically against the conjugation above. The transform + below was correct all along; the note is kept so the no-op is not + reintroduced as a fix. """ - m = w * np.array([-1, 1, 1, -1], np.float32) + m = w * np.array([1, -1, -1, 1], np.float32) return m[:, MIRROR_PERM], valid[MIRROR_PERM] @@ -677,8 +681,11 @@ def windows(action, cq, valid): np.savez_compressed(a.out, mo=mo, msk=msk, tk=tk, vocab=np.array(vocab), fps=FPS, canonRestDir=D_CANON, - buildArgv=np.array(sys.argv, dtype=object), - buildFlags=np.array(json.dumps(vars(a)), dtype=object)) + # Plain unicode arrays, NOT dtype=object: an object + # array forces allow_pickle=True on every reader, and + # a crafted npz could then execute code on load. + buildArgv=np.array(sys.argv, dtype=np.str_), + buildFlags=np.array(json.dumps(vars(a)), dtype=np.str_)) print(f"wrote {a.out} mo{mo.shape}") diff --git a/scripts/train-t2m-flow-v5.py b/scripts/train-t2m-flow-v5.py index 7bebcadb4..1472c0eb5 100644 --- a/scripts/train-t2m-flow-v5.py +++ b/scripts/train-t2m-flow-v5.py @@ -427,7 +427,11 @@ def forward(self, x, t, tok): class Sampler(nn.Module): """Euler flow sampler UNROLLED for ONNX export — MotionGenerator contract.""" - def __init__(self, net, V, T, steps, guidance=2.0): + # guidance default 1.0 — the value the shipped v8.0 model was trained + # and exported with. This used to default to 2.0 while both CLI flags + # defaulted to 1.0, so a caller constructing a Sampler directly got + # different motion than the CLI produced. + def __init__(self, net, V, T, steps, guidance=1.0): super().__init__() self.net, self.V, self.T, self.steps = net, V, T, steps self.guidance = guidance @@ -524,7 +528,7 @@ def main(): "(they share its reconstructed-sample block)") - d = np.load(a.data, allow_pickle=True) + d = np.load(a.data, allow_pickle=False) mo, msk, tk = d["mo"], d["msk"], d["tk"] vocab = [str(w) for w in d["vocab"]] fps = int(d["fps"]) diff --git a/src/AnimationMerger.cpp b/src/AnimationMerger.cpp index 15694452c..c03545a7f 100644 --- a/src/AnimationMerger.cpp +++ b/src/AnimationMerger.cpp @@ -9,10 +9,6 @@ #include #include #include -#include -#include -#include -#include #include #include #include @@ -2047,50 +2043,12 @@ void AnimationMerger::debugDumpAnatomy(Ogre::Skeleton* skel, skel->_updateTransforms(); } -namespace { - -// Reads "yaw180" from the installed t2m vocab json, if present. -// Returns 1 (flip), 0 (do not flip) or -1 (not declared). -int declaredYaw180() -{ - static int cached = -2; - if (cached != -2) return cached; - cached = -1; - const QString path = - QStandardPaths::writableLocation(QStandardPaths::AppDataLocation) - + QLatin1String("/ai_models/motion/t2m-vocab.json"); - QFile f(path); - if (f.open(QIODevice::ReadOnly)) { - const QJsonObject o = - QJsonDocument::fromJson(f.readAll()).object(); - if (o.contains(QLatin1String("yaw180"))) - cached = o.value(QLatin1String("yaw180")).toBool() ? 1 : 0; - } - return cached; -} - -} // namespace - bool AnimationMerger::detectBackwardFacing(Ogre::Entity* entity) { // Escape hatch for exotic meshes where the foot-region heuristic guesses // wrong: QTMESH_T2M_YAW180=1 forces the flip, =0 disables it. const QByteArray force = qgetenv("QTMESH_T2M_YAW180"); if (!force.isEmpty()) return force != "0"; - // A MODEL-DECLARED flip wins over the geometry heuristic. The heuristic - // below reads the mesh's toe region, which is a property of the RIG alone — - // it therefore returns the same answer for every model, while different - // t2m generations can carry opposite canonical facing conventions and need - // opposite flips on the same mesh. - // - // No shipped vocab declares `yaw180` today: the v8.0 model is trained on - // the curated template clips, which already carry the correct facing, so - // the geometry heuristic agrees with it and this returns -1. The hook stays - // as forward compatibility for a future model trained on a corpus with the - // opposite convention — that is a property of the MODEL, and only the model - // can state it. Result is cached, so this costs one file read per process. - if (const int decl = declaredYaw180(); decl >= 0) - return decl != 0; if (!entity || !entity->hasSkeleton()) return false; Ogre::SkeletonInstance* skel = entity->getSkeleton(); if (!skel) return false; From a2fe04e9fb5ca323318e6645ceadaed367d2ee4b Mon Sep 17 00:00:00 2001 From: Fernando Date: Sat, 29 Aug 2026 02:51:55 -0400 Subject: [PATCH 35/35] review: shoulder-role naming + refuse a no-op warm start (#963) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Shoulder roles renamed, behaviour deliberately unchanged.** The review found that `lsh, rsh = fk_pos(w, 7), fk_pos(w, 11)` is backwards — role 7 is the RIGHT shoulder (D_CANON[7] = -X) and role 11 the LEFT — and proposed swapping the bindings so the cross product uses (left - right). Swapping would INVERT the sign and make the gate reject every genuine forward walk. Verified empirically before touching it: the real Mixamo walk scores +0.581 with the code as written, and 100% of cached walk/run windows are positive. The operand order is what produces the forward axis; only the NAMES were wrong. Renamed both the numpy gate and its torch twin to match reality, and documented why the 'obvious' swap is wrong so it is not reapplied. **Refuse a warm start that would train zero epochs.** `range(start_ep, a.epochs)` is empty when --epochs is at or below the resume point, and the export block would then write an ONNX of untrained (or, on a warm start, merely transplanted) weights — silently, and indistinguishable from a real result. Now raises SystemExit pointing at export-t2m-from-ckpt.py for the legitimate 'export this checkpoint as-is' case. Verified: --epochs 5 against the ep21 checkpoint now refuses instead of exporting. --- scripts/prep-t2m-v6.py | 12 ++++++++++-- scripts/train-t2m-flow-v5.py | 22 ++++++++++++++++++---- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/scripts/prep-t2m-v6.py b/scripts/prep-t2m-v6.py index d9577b714..3a1598e20 100644 --- a/scripts/prep-t2m-v6.py +++ b/scripts/prep-t2m-v6.py @@ -149,8 +149,16 @@ def travel_forward(w): (#837, user-reported "moving backwards"). Gating on this collapses the conditional to one mode. """ - lsh, rsh = fk_pos(w, 7), fk_pos(w, 11) - side = lsh - rsh + # NB roles 7/11 are RIGHT/LEFT respectively (D_CANON[7] = -X, D_CANON[11] + # = +X), so `side` below is right->left... which is why the cross product + # with +Y yields the FORWARD axis with these operands in this order. A + # review flagged the old `lsh, rsh = fk_pos(w, 7), fk_pos(w, 11)` naming as + # a swapped binding and proposed exchanging them; that would INVERT the sign + # and make the gate reject every genuine forward walk. Verified empirically: + # the real Mixamo walk scores +0.581 and 100% of cached walk/run windows are + # positive as written. Renamed to match reality rather than changing it. + rsh, lsh = fk_pos(w, 7), fk_pos(w, 11) + side = rsh - lsh side = side / (np.linalg.norm(side, axis=-1, keepdims=True) + 1e-9) up = np.broadcast_to(np.array([0, 1, 0], np.float32), side.shape) f = np.cross(side, up) diff --git a/scripts/train-t2m-flow-v5.py b/scripts/train-t2m-flow-v5.py index 1472c0eb5..abc87d7ab 100644 --- a/scripts/train-t2m-flow-v5.py +++ b/scripts/train-t2m-flow-v5.py @@ -141,7 +141,10 @@ def travel_forward_torch(q, dirs): Same quantity as prep-t2m-v6.travel_forward and the eval: travel is inferred from the STANCE (slower) foot, which drifts backward while the - body moves forward. Forward is (Lshoulder - Rshoulder) x up. + body moves forward. Forward is (role7 - role11) x up, where role 7 is the + RIGHT shoulder and role 11 the LEFT (D_CANON[7] = -X, D_CANON[11] = +X) — + see the note in prep-t2m-v6.travel_forward: this operand order is what + yields the forward axis, and exchanging them inverts the sign. Even with a 100%-forward training set the model sat at ~50% forward (measured at ep40 with foot motion at 73% of the data's magnitude, so this @@ -149,9 +152,9 @@ def travel_forward_torch(q, dirs): no term tying the emitted gait to a travel direction, so this supervises it directly — the same reasoning as the gait-phase hinge. """ - lsh = fk_pos(q, 7, dirs) - rsh = fk_pos(q, 11, dirs) - side = F.normalize(lsh - rsh, dim=-1, eps=1e-6) + rsh = fk_pos(q, 7, dirs) # role 7 = RIGHT shoulder + lsh = fk_pos(q, 11, dirs) # role 11 = LEFT shoulder + side = F.normalize(rsh - lsh, dim=-1, eps=1e-6) up = torch.zeros_like(side) up[..., 1] = 1.0 fwd = F.normalize(torch.cross(side, up, dim=-1), dim=-1, eps=1e-6) @@ -632,6 +635,17 @@ def main(): print(f"WARM START from transplanted weights at epoch {start_ep} " f"(fresh optimizer/scheduler)", flush=True) + # `range(start_ep, a.epochs)` is EMPTY when --epochs is at or below the + # resume point, and the export block below would then happily write an + # ONNX of untrained (or, on a warm start, merely transplanted) weights — + # silently, and indistinguishable from a real result. Refuse instead. + if start_ep >= a.epochs: + raise SystemExit( + f"--epochs {a.epochs} is not beyond the checkpoint's epoch " + f"{start_ep}: there is nothing to train. Raise --epochs to export " + f"a trained model, or use export-t2m-from-ckpt.py to export the " + f"checkpoint as-is.") + for ep in range(start_ep, a.epochs): # Accumulate the epoch loss ON DEVICE. A per-batch `loss.item()` is a # GPU->CPU sync (`_local_scalar_dense_mps` -> `waitUntilCompleted`)