From 7b8fe83ec5f75bd877dd3348793aea8aa4197869 Mon Sep 17 00:00:00 2001 From: James Date: Mon, 2 Nov 2026 08:07:47 -0700 Subject: [PATCH] The shader split reads each SARC member's name up to its own NUL Fire Emblem Fortune's Weave's shader split took 12 minutes, 6 of them in one 9.8 MB SARC (Layout/Common.arc.zst). sarc_members took each member's name by splitting the whole rest of the archive at every NUL and keeping the first piece, so the work grew with the archive's size times its member count. It now finds the name's own NUL and decodes only that. The names are the same; the split of that game took 1:51. Checked: test_split_nesting.py's new test reads an 800-member archive and compares the names and data; the reading takes about 1.5 ms, against 47 s with the old code. The archive is built before the timing starts, so a slower runner does not count its construction. Donkey Kong Bananza's split gives the same 23,638 shaders. Co-Authored-By: Claude Opus 5.5 --- dm/gfx/tools/shader_split.py | 3 ++- dm/tools/tests/test_split_nesting.py | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/dm/gfx/tools/shader_split.py b/dm/gfx/tools/shader_split.py index b7fdb43334..d39f540e59 100644 --- a/dm/gfx/tools/shader_split.py +++ b/dm/gfx/tools/shader_split.py @@ -2117,7 +2117,8 @@ def sarc_members(d, label='SARC'): for i in range(n): h, attr, s, e = struct.unpack_from('> 24 else f'#{i}' + o = names + (attr & 0xffffff) * 4; z = d.find(b'\0', o) # up to the name's own NUL: splitting the rest of the archive per member was quadratic (Fire Emblem Fortune's Weave's Layout/Common.arc.zst: 6 minutes) + nm = d[o:z if z >= 0 else len(d)].decode('utf-8', 'replace') if attr >> 24 else f'#{i}' yield nm, d[doff + s:doff + e] NV_STAGES = ['vs', 'ps', 'gs', 'hs', 'ds', 'cs'] # NVN stage order diff --git a/dm/tools/tests/test_split_nesting.py b/dm/tools/tests/test_split_nesting.py index 1257157ffb..f2ddf420ed 100644 --- a/dm/tools/tests/test_split_nesting.py +++ b/dm/tools/tests/test_split_nesting.py @@ -74,5 +74,15 @@ def test_yaz0_expansion_is_charged_before_it_is_decoded(self): self.assertEqual(dec.call_count, 0) +class MemberNames(unittest.TestCase): + def test_names_and_time(self): # each name ends at its own NUL; splitting the rest of the archive per member made Fire Emblem Fortune's Weave's 9.8 MB layout SARC take 6 minutes + members = [(b'tex%04d.bntx' % i, bytes(0x4000)) for i in range(800)] + import time + archive = fz.sarc(members) # built outside the timing: only the member reading is measured (about 1 ms here, 47 s with the per-member split) + t = time.monotonic(); got = list(ss.sarc_members(archive)); took = time.monotonic() - t + self.assertEqual(got, [(n.decode(), m) for n, m in members]) + self.assertLess(took, 1.0) + + if __name__ == '__main__': unittest.main()