From 7ad5fe1d667da80c6051f533a9db2e20c3dc45bd Mon Sep 17 00:00:00 2001 From: Randy Olson Date: Sat, 10 Oct 2026 15:32:08 -0700 Subject: [PATCH] Read long careers with national teams and flagged name pools; release 0.5.9 A 26.3.2 career save failed 19 of 27 readers. Four general causes: - The name-pool header stores nonzero values in a 4-byte field most saves leave at zero. Those bytes are no longer compared; every other header byte, each pool entry and the uniqueness check still are. - The stage table sat 34.2 MB from the end of game_db, past the 32 MiB search cap that had already been raised as careers grew. The search now doubles to the start of the section, and the stage gates judge the rows. - The manager also runs national teams, whose tactic blocks the header counts. Tactics validation compares the header with the blocks found by structure, and each club block ends at the next stored block. - Generated players with object kind 3 and generated staff with ability codes outside the observed set are read instead of dropped. Refs #20. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 11 +++++++ pyproject.toml | 2 +- src/fmsave/_checks.py | 12 ++++++-- src/fmsave/_layouts/__init__.py | 21 +++++++------ src/fmsave/_layouts/fm26_26_3_2.py | 29 ++++++++++-------- src/fmsave/_reader_stats.py | 7 +++-- src/fmsave/_save.py | 1 + src/fmsave/readers/names.py | 14 ++++++--- src/fmsave/readers/player_scan.py | 4 +-- src/fmsave/readers/staff.py | 6 ++-- src/fmsave/readers/stages.py | 23 ++++++++------ src/fmsave/readers/tactics.py | 44 ++++++++++++++++++++++----- tests/__snapshots__/test_layouts.ambr | 15 +++++++-- tests/test_checks.py | 1 + tests/test_names.py | 33 ++++++++++++++++++++ tests/test_players.py | 10 ++++-- tests/test_staff.py | 15 +++++++-- tests/test_stages.py | 36 +++++++++++++++++----- tests/test_tactics.py | 42 +++++++++++++++++++++++++ uv.lock | 2 +- 20 files changed, 260 insertions(+), 68 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d1bfe39..f1a1d2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,17 @@ This project follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) an ## [Unreleased] +## [0.5.9] - 2026-10-10 + +### Fixed + +- Name pools are found when their header stores nonzero values in a field that most saves leave at zero. Every other header byte, each pool entry and the uniqueness check are still verified. +- The stage table is found however far it sits from the end of `game_db`. Longer careers move it further back, which made stages, competitions, fixtures, league tables, competition rules, match stats, stadiums and job vacancies fail. +- Tactics validation accepts managers who also run national teams, whose tactic blocks the section stores alongside the club's. +- Newly generated players stored with object kind 3, and newly generated staff whose ability codes are outside the observed code set, are read instead of being dropped. + +Refs #20. + ## [0.5.8] - 2026-10-07 ### Fixed diff --git a/pyproject.toml b/pyproject.toml index 760b084..3c405e9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "fmsave" -version = "0.5.8" +version = "0.5.9" description = "Read Football Manager 26 save files into Python records, DataFrames, CSV and JSON." readme = "README.md" requires-python = ">=3.12" diff --git a/src/fmsave/_checks.py b/src/fmsave/_checks.py index 291bb48..724c84b 100644 --- a/src/fmsave/_checks.py +++ b/src/fmsave/_checks.py @@ -2092,8 +2092,10 @@ def evaluate_tactics( `tactics_team_blocks_match_club` carries the header's own claim as well as the blocks found, because the two fail differently: a team-id locator that has moved leaves a team without its single block, while a header read from the wrong offset claims a number of - blocks that is not the managed club's team count at all. A header that disagrees scores - zero here whatever the locator found. + blocks that is not the number the section stores at all. The header counts every team the + manager runs, national teams included, so it is compared with the blocks found by + structure and must cover at least the club's teams. A header that disagrees scores zero + here whatever the locator found. The record-count share checks the team's declared count independently. The two walk shares are judged on all stored tactic records. They are excused when the @@ -2104,7 +2106,10 @@ def evaluate_tactics( """ applied = _applies(bounds, game_db_bytes) and stats.managed_club_exists slots_applied = applied and (stats.tactic_blocks > 0 or stats.stored_tactics > 0) - blocks_found = stats.blocks_found if stats.header_blocks == stats.club_team_count else 0 + header_agrees = ( + stats.header_blocks == stats.stored_blocks and stats.header_blocks >= stats.club_team_count + ) + blocks_found = stats.blocks_found if header_agrees else 0 return ( _gate( "tactics_manager_selector_matches", @@ -2189,6 +2194,7 @@ def check_tactics(stats: TacticStats, bounds: GateBounds, game_db_bytes: int) -> FrozenMapping( { "preset_tactics": stats.preset_tactics, + "non_club_blocks": stats.stored_blocks - stats.blocks_found, "tactic_blocks_count_mismatched": ( stats.tactic_blocks - stats.tactic_blocks_count_matching ), diff --git a/src/fmsave/_layouts/__init__.py b/src/fmsave/_layouts/__init__.py index d076ab2..1b35058 100644 --- a/src/fmsave/_layouts/__init__.py +++ b/src/fmsave/_layouts/__init__.py @@ -92,15 +92,17 @@ class NamePoolLayout: """How to find and check the three name pools in `game_db`. `signature` sits immediately before the first pool's entry count. Its three capacity - words may scale together up to `max_signature_scale`. Pools have no minimum size, - since databases differ in size; every entry's id and length are checked instead. - `max_name_bytes` bounds each name's byte length independently of section bounds; + words may scale together up to `max_signature_scale`. The bytes at + `unchecked_signature_bytes` may hold any value; they are read as zero. Pools have no + minimum size, since databases differ in size; every entry's id and length are checked + instead. `max_name_bytes` bounds each name's byte length independently of section bounds; it is a parser safety ceiling, not a verified limit on names stored by the game. """ signature: bytes max_name_bytes: int max_signature_scale: int = 1 + unchecked_signature_bytes: tuple[int, ...] = () @dataclass(frozen=True, slots=True) @@ -375,7 +377,7 @@ class PlayerRecordLayout: rejected one. Ordinary identity words match. In the allocated namespace starting at - `allocated_uid_minimum`, a distinct second word also requires `player_kind` at + `allocated_uid_minimum`, a distinct second word also requires one of `player_kinds` at `object_kind_offset`, together with every ability, reputation and attribute check. The distinct word's meaning is not inferred. @@ -391,7 +393,7 @@ class PlayerRecordLayout: uid_copy_offset: int allocated_uid_minimum: int object_kind_offset: int - player_kind: int + player_kinds: tuple[int, ...] home_reputation_offset: int current_reputation_offset: int world_reputation_offset: int @@ -1124,9 +1126,10 @@ class StageTableLayout: The table is found in the last `search_bytes` of `game_db`, at the first offset where `chain_rows` rows in a row all decode `row_bytes` apart. When none does, the window doubles - until it reaches `max_search_bytes`, searching only the stretch each doubling adds, so a - save whose table sits near the end pays for the first window alone. The head of that chain - is then reached by stepping back a row at a time. A row decodes when the byte at + until it reaches `max_search_bytes`, or the start of `game_db` when that is None, + searching only the stretch each doubling adds, so a save whose table sits near the end + pays for the first window alone. The head of that chain is then reached by stepping back + a row at a time. A row decodes when the byte at `zero_byte_offset` is zero, the stage id at `stage_id_offset` is repeated at `stage_id_copy_offset` and lies strictly between the bounds of `stage_id_exclusive_range`, and the word at `previous_stage_id_offset` is either that id @@ -1151,7 +1154,7 @@ class StageTableLayout: stage_id_exclusive_range: tuple[int, int] competition_id_limit: int search_bytes: int - max_search_bytes: int + max_search_bytes: int | None chain_rows: int resynchronisation_bytes: int diff --git a/src/fmsave/_layouts/fm26_26_3_2.py b/src/fmsave/_layouts/fm26_26_3_2.py index 342f47c..8454fb1 100644 --- a/src/fmsave/_layouts/fm26_26_3_2.py +++ b/src/fmsave/_layouts/fm26_26_3_2.py @@ -264,6 +264,8 @@ # original sample-based ceiling of 64 bytes (Refs #14). max_name_bytes=1024, max_signature_scale=1024, + # Zero in most saves; one 26.3.2 career save stores 03 03 03 03 here (Refs #20). + unchecked_signature_bytes=(5, 6, 7, 8), ) CLUB_RECORDS = ClubRecordLayout( @@ -328,7 +330,9 @@ # and the complete rating/attribute structure must still match a player. allocated_uid_minimum=2_000_000_000, object_kind_offset=-7, - player_kind=2, + # Ordinary player records carry kind 2 or 3 (96,392 of 5.2 million across the corpus are + # 3), so an allocated player can carry either (Refs #20). + player_kinds=(2, 3), home_reputation_offset=-6, current_reputation_offset=-4, world_reputation_offset=-2, @@ -1011,10 +1015,10 @@ ) # A stage row is 33 bytes. The table starts 0.65 to 0.94 MB from the end of `game_db` on the -# corpus saves, so the last 2 MB is searched first; one community career started on an older -# build has 4.1 MB after its table. Another continued career has 20.7 MB after the table, so -# the window may double to 32 MiB before the table counts as missing. 200 rows is far longer -# than any run of look-alike bytes seen before it. +# corpus saves, so the last 2 MB is searched first. What follows the table grows as a career +# goes on (4.1, 20.7 and 34.2 MB on longer careers, Refs #5 and #20), so no fixed window +# holds every save: the window keeps doubling to the start of `game_db`. 200 rows is far +# longer than any run of look-alike bytes seen before it, and the stage gates check the rows. STAGE_TABLE = StageTableLayout( row_bytes=33, previous_stage_id_offset=0, @@ -1031,7 +1035,7 @@ # competitions a save holds; this limit rejects it without touching a real competition. competition_id_limit=1_000_000, search_bytes=2_000_000, - max_search_bytes=32 * 1024 * 1024, + max_search_bytes=None, chain_rows=200, resynchronisation_bytes=4_096, ) @@ -1276,9 +1280,9 @@ # The last word is the missing value on about 98.4% of rows and carries a small number on # the rest, so this is a shape check rather than a sentinel. stage_trailing_sentinel=(0.95, None), - # As far back as the stage table search reaches (see STAGE_TABLE): 0.63 to 0.92 MiB on the - # corpus saves, 3.94 MiB on one older-build career, and 20.7 MB on another. - stage_table_tail_bytes=(None, 32 * 1024 * 1024), + # Reported, not bounded: the bytes after the table grow with the career (see STAGE_TABLE), + # and the other stage gates judge the rows themselves. + stage_table_tail_bytes=(None, None), competitions_minimum=(50, None), # The id-pair records name 91.47% to 91.52% of the stage table's competitions across two # careers and a live save, and no save leaves a single competition in conflict. The share @@ -1573,9 +1577,10 @@ # byte or four bytes late it names nobody at all, so only equality passes. tactics_manager_selector_matches=(1, 1), # Every team of the managed club has exactly one block on every save measured, and the - # header claims that same number. Reading the count four bytes late gives a - # team id instead, and one byte late gives 1,610,612,736, so the header agreement fails on - # its own; a team-id locator read one byte late matches more places than there are teams + # header claims as many blocks as the section stores: the club's teams, plus any national + # teams the manager also runs (7 on one career, Refs #20). Reading the count four bytes + # late gives a team id instead, and one byte late gives 1,610,612,736, so the header + # agreement fails on its own; a team-id locator read one byte late matches more places than there are teams # over the same team ids, which leaves a team without its single block. tactics_team_blocks_match_club=(1.0, None), # Every user tactic record walks its 22 slot blocks, and the eleven diff --git a/src/fmsave/_reader_stats.py b/src/fmsave/_reader_stats.py index f41f88b..9805abc 100644 --- a/src/fmsave/_reader_stats.py +++ b/src/fmsave/_reader_stats.py @@ -748,8 +748,10 @@ class TacticStats: read and both tables are empty. `club_team_count` is how many teams the managed club fields, `header_blocks` the number of - blocks the section header claims and `blocks_found` how many teams the section stores - exactly one block for. `selector_matches` is whether the section header names the same + blocks the section header claims, `stored_blocks` how many blocks the section holds when + found by structure (national teams the manager also runs included; 0 when the first is + not where the header ends) and `blocks_found` how many teams the section stores exactly + one block for. `selector_matches` is whether the section header names the same human manager as the `humans` section. `selection_selectors` counts every player selector the blocks store, over the selection @@ -775,6 +777,7 @@ class TacticStats: managed_club_exists: bool club_team_count: int header_blocks: int + stored_blocks: int blocks_found: int selector_matches: bool selection_selectors: int diff --git a/src/fmsave/_save.py b/src/fmsave/_save.py index 54eb85e..21e7cd4 100644 --- a/src/fmsave/_save.py +++ b/src/fmsave/_save.py @@ -200,6 +200,7 @@ class _FinanceTables(NamedTuple): managed_club_exists=False, club_team_count=0, header_blocks=0, + stored_blocks=0, blocks_found=0, selector_matches=False, selection_selectors=0, diff --git a/src/fmsave/readers/names.py b/src/fmsave/readers/names.py index 8df220a..937246a 100644 --- a/src/fmsave/readers/names.py +++ b/src/fmsave/readers/names.py @@ -114,26 +114,32 @@ def locate_name_pools(game_db: bytes, layout: NamePoolLayout, file_name: str) -> def _signature_candidates(game_db: bytes, layout: NamePoolLayout) -> Iterator[int]: - """Capacity words can grow together; every candidate still needs three checked pools.""" + """Capacity words can grow together and unchecked bytes may vary; every candidate still + needs three checked pools.""" signature = layout.signature - if layout.max_signature_scale == 1: + unchecked = layout.unchecked_signature_bytes + if layout.max_signature_scale == 1 and not unchecked: start = 0 while (at := game_db.find(signature, start)) >= 0: yield at start = at + 1 return words = struct.unpack("<6I", signature) - prefix = signature[:8] + prefix = signature[: min((8, *unchecked))] start = 0 while (at := game_db.find(prefix, start)) >= 0: start = at + 1 if at + len(signature) > len(game_db): continue - stored = struct.unpack_from("<6I", game_db, at) + stored_bytes = bytearray(game_db[at : at + len(signature)]) + for byte_at in unchecked: + stored_bytes[byte_at] = 0 + stored = struct.unpack("<6I", stored_bytes) scale, remainder = divmod(stored[3], words[3]) if ( remainder == 0 and 1 <= scale <= layout.max_signature_scale + and stored[:2] == words[:2] and stored[2:5] == tuple(value * scale for value in words[2:5]) and stored[5] == words[5] ): diff --git a/src/fmsave/readers/player_scan.py b/src/fmsave/readers/player_scan.py index 47f35ab..e543de2 100644 --- a/src/fmsave/readers/player_scan.py +++ b/src/fmsave/readers/player_scan.py @@ -361,7 +361,7 @@ def _accept_candidate( return None if uid != uid_copy and ( uid < layout.allocated_uid_minimum - or game_db[record_offset + layout.object_kind_offset] != layout.player_kind + or game_db[record_offset + layout.object_kind_offset] not in layout.player_kinds ): return None header_values = header_layout.struct_object.unpack_from(game_db, header_absolute_start) @@ -384,7 +384,7 @@ def _accept_candidate( layout.edited_ability_maximum is None or layout.edited_attribute_maximum is None or uid != uid_copy - or game_db[record_offset + layout.object_kind_offset] != layout.player_kind + or game_db[record_offset + layout.object_kind_offset] not in layout.player_kinds ): return None highest_ability = layout.edited_ability_maximum diff --git a/src/fmsave/readers/staff.py b/src/fmsave/readers/staff.py index d3895b3..1e252ec 100644 --- a/src/fmsave/readers/staff.py +++ b/src/fmsave/readers/staff.py @@ -184,8 +184,10 @@ def _header_values( if uid != other: if layout is None or uid < layout.allocated_uid_minimum or kind != layout.staff_kind: return None - block, slots_ok, codes_ok, further_ok = _read_ability_block(game_db, header, layout) - if block is None or not (slots_ok and codes_ok and further_ok): + # The code set is the codes observed, not a closed set (`staff_codes_in_set` is a + # share for that reason), so a code outside it cannot rule out a header (Refs #20). + block, slots_ok, _codes_ok, further_ok = _read_ability_block(game_db, header, layout) + if block is None or not (slots_ok and further_ok): return None return uid, kind diff --git a/src/fmsave/readers/stages.py b/src/fmsave/readers/stages.py index 3e5c9ab..8a49f70 100644 --- a/src/fmsave/readers/stages.py +++ b/src/fmsave/readers/stages.py @@ -90,7 +90,7 @@ class _StageTableScan: row_bytes: int chain_rows: int search_bytes: int - max_search_bytes: int + max_search_bytes: int | None resynchronisation_bytes: int lowest_stage_id: int highest_stage_id: int @@ -140,7 +140,7 @@ def _stage_table_scan(layout: StageTableLayout) -> _StageTableScan: raise ValueError(f"chain_rows {layout.chain_rows} must be at least 1") if layout.competition_id_limit < 1: raise ValueError(f"competition_id_limit {layout.competition_id_limit} must be at least 1") - if layout.max_search_bytes < layout.search_bytes: + if layout.max_search_bytes is not None and layout.max_search_bytes < layout.search_bytes: raise ValueError( f"max_search_bytes {layout.max_search_bytes} is narrower than search_bytes " f"{layout.search_bytes}" @@ -219,8 +219,9 @@ def find_table_start(game_db: bytes, scan: _StageTableScan) -> int | None: The search covers the last `search_bytes` of `game_db`, since the table sits at the end of the section. When no chain starts there, the window doubles until it reaches - `max_search_bytes`, and each doubling searches only the stretch it adds, so a save whose - table sits near the end pays for the first window alone and no offset is tried twice. + `max_search_bytes`, or the start of `game_db` when that is None, and each doubling + searches only the stretch it adds, so a save whose table sits near the end pays for the + first window alone and no offset is tried twice. The first offset that starts a full chain is found one byte at a time, and the head is then reached by stepping back while the row before still decodes, so rows before the chain are @@ -239,10 +240,12 @@ def find_table_start(game_db: bytes, scan: _StageTableScan) -> int | None: table_start = _find_table_start_between(game_db, scan, window_start, searched_from) if table_start is not None: return table_start - if window_bytes >= scan.max_search_bytes: + if scan.max_search_bytes is not None and window_bytes >= scan.max_search_bytes: return None searched_from = window_start - window_bytes = min(window_bytes * 2, scan.max_search_bytes) + window_bytes *= 2 + if scan.max_search_bytes is not None: + window_bytes = min(window_bytes, scan.max_search_bytes) return None @@ -330,10 +333,12 @@ def read_stage_index(game_db: bytes, layout: StageTableLayout, file_name: str) - scan = _stage_table_scan(layout) table_start = find_table_start(game_db, scan) if table_start is None: - raise layout_mismatch( - file_name, - f"no stage table was found in the last {scan.max_search_bytes:,} bytes of game_db", + searched = ( + "game_db" + if scan.max_search_bytes is None + else f"the last {scan.max_search_bytes:,} bytes of game_db" ) + raise layout_mismatch(file_name, f"no stage table was found in {searched}") walked = _walk_rows(game_db, table_start, scan) stages: list[Stage] = [] diff --git a/src/fmsave/readers/tactics.py b/src/fmsave/readers/tactics.py index 439c423..29975f0 100644 --- a/src/fmsave/readers/tactics.py +++ b/src/fmsave/readers/tactics.py @@ -21,6 +21,7 @@ from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass, field +from fmsave._errors import ReaderCheckError from fmsave._frozen import FrozenMapping from fmsave._layouts import TacticsLayout, find_layout from fmsave._reader_stats import TacticStats @@ -89,15 +90,18 @@ class TacticWalkCounts: `club_team_count` is how many teams the managed club fields, `blocks_found` how many of them the section stores exactly one block for, and `header_blocks` the number of blocks - the section header claims. `tactic_blocks` counts blocks whose stored count claims a - tactic record, and `tactic_blocks_count_matching` those where every record the count claims - was found. `preset_tactics` counts the records carrying the preset signature, which are + the section header claims. `stored_blocks` is how many blocks the section holds, found by + structure rather than by team id, which includes the blocks of any national teams the + manager also runs; it is 0 when the first one is not where the header ends. + `tactic_blocks` counts blocks whose stored count claims a tactic record, and + `tactic_blocks_count_matching` those where every record the count claims was found. `preset_tactics` counts the records carrying the preset signature, which are decoded along with the other records in the team's declared list. `routine_blocks_complete` counts blocks whose default runs and counted routine groups were all read completely. """ club_team_count: int header_blocks: int + stored_blocks: int blocks_found: int tactic_blocks: int tactic_blocks_count_matching: int @@ -157,6 +161,27 @@ def _block_starts( return starts +def _stored_block_starts(section: bytes, layout: TacticsLayout, file_name: str) -> list[int]: + """The start of every team block the section stores, in order, found by structure. + + A block is a team id, the block marker and a whole selection part, so every marker hit is + tried as the end of a team id and kept when the selection part after it parses. This finds + blocks for teams the club index does not list, such as the manager's national teams. + """ + starts: list[int] = [] + marker_at = section.find(layout.block_marker, layout.first_block_offset + _UINT32.size) + while marker_at >= 0: + block_start = marker_at - _UINT32.size + try: + _read_selection_part(section, block_start, len(section), layout, file_name) + except ReaderCheckError: + pass + else: + starts.append(block_start) + marker_at = section.find(layout.block_marker, marker_at + 1) + return starts + + def _read_text( section: bytes, cursor: int, limit: int, layout: TacticsLayout, file_name: str, what: str ) -> tuple[str, int]: @@ -660,15 +685,16 @@ def walk_tactic_blocks( _selector, header_blocks = read_tactics_header(section, layout, file_name) starts = _block_starts(section, club_team_ids, layout) ordered_starts = sorted(starts.items(), key=lambda pair: pair[1]) + stored_starts = _stored_block_starts(section, layout, file_name) blocks: list[RawTacticBlock] = [] routine_blocks_complete = 0 tactic_blocks = 0 tactic_blocks_count_matching = 0 preset_tactics = 0 - for position, (team_id, block_start) in enumerate(ordered_starts): - block_end = ( - ordered_starts[position + 1][1] if position + 1 < len(ordered_starts) else len(section) - ) + for team_id, block_start in ordered_starts: + # A block ends where the next stored block begins, whichever team that block is for: + # the manager's national teams have blocks too, and the club index lists none of them. + block_end = next((start for start in stored_starts if start > block_start), len(section)) selection = _read_selection_part(section, block_start, block_end, layout, file_name) routine_names, routines_complete, records_end = _read_routines( section, selection.end, block_end, layout @@ -710,6 +736,9 @@ def walk_tactic_blocks( counts = TacticWalkCounts( len(club_team_ids), header_blocks, + # Counted only when the first block sits where the header ends, so a scan that has + # drifted off the real blocks cannot agree with the header by accident. + len(stored_starts) if stored_starts[:1] == [layout.first_block_offset] else 0, len(blocks), tactic_blocks, tactic_blocks_count_matching, @@ -810,6 +839,7 @@ def build_tactic_tables( managed_club_exists=True, club_team_count=counts.club_team_count, header_blocks=counts.header_blocks, + stored_blocks=counts.stored_blocks, blocks_found=counts.blocks_found, selector_matches=human_selector is not None and header_selector == human_selector, selection_selectors=selectors, diff --git a/tests/__snapshots__/test_layouts.ambr b/tests/__snapshots__/test_layouts.ambr index 9a0bcdb..295c68e 100644 --- a/tests/__snapshots__/test_layouts.ambr +++ b/tests/__snapshots__/test_layouts.ambr @@ -55,6 +55,12 @@ 'max_name_bytes': 1024, 'max_signature_scale': 1024, 'signature': b'U\xb5\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x01\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00', + 'unchecked_signature_bytes': tuple( + 5, + 6, + 7, + 8, + ), }), 'region': 'game_db', 'schema': 4000, @@ -217,7 +223,10 @@ 'match_sharpness_offset': 106, 'object_kind_offset': -7, 'pindex_offset': -19, - 'player_kind': 2, + 'player_kinds': tuple( + 2, + 3, + ), 'position_codes': tuple( 'GK', 'SW', @@ -857,7 +866,7 @@ 'competition_id_limit': 1000000, 'competition_id_offset': 13, 'group_id_offset': 17, - 'max_search_bytes': 33554432, + 'max_search_bytes': None, 'previous_stage_id_offset': 0, 'resynchronisation_bytes': 4096, 'round_offset': 21, @@ -2390,7 +2399,7 @@ ), 'stage_table_tail_bytes': tuple( None, - 33554432, + None, ), 'stage_trailing_sentinel': tuple( 0.95, diff --git a/tests/test_checks.py b/tests/test_checks.py index 41d2340..c73bed7 100644 --- a/tests/test_checks.py +++ b/tests/test_checks.py @@ -1304,6 +1304,7 @@ def test_reader_passes_collect_the_counts_their_gates_check(counted_fragment_pat "mentoring": {"members_without_a_player": 2, "members_elsewhere": 0}, "tactics": { "preset_tactics": 0, + "non_club_blocks": 0, "tactic_blocks_count_mismatched": 0, "unresolved_selectors": 5, }, diff --git a/tests/test_names.py b/tests/test_names.py index ca43260..07bcc4b 100644 --- a/tests/test_names.py +++ b/tests/test_names.py @@ -127,6 +127,39 @@ def test_two_valid_pool_groups_with_different_scales_are_ambiguous() -> None: locate(wrapped(pools + scaled), layout) +def with_flag_bytes(signature: bytes, value: int) -> bytes: + return signature[:5] + bytes([value] * 4) + signature[9:] + + +@pytest.mark.parametrize("scale", [1, 7]) +def test_unchecked_signature_bytes_may_hold_any_value(scale: int) -> None: + # A 26.3.2 career save stores 03 03 03 03 in signature bytes 5-8 (Refs #20). + signature = with_flag_bytes(scaled_signature(scale), 3) + pools = name_pools_bytes(["Alex"], ["Example"], ["Exo"], signature=signature) + game_db = wrapped(pools) + assert locate(game_db, REGISTERED_LAYOUT).first_names.name_at(game_db, 0) == "Alex" + + +def test_signature_bytes_outside_the_unchecked_ones_are_still_checked() -> None: + signature = with_flag_bytes(NAME_POOL_SIGNATURE, 3) + pools = name_pools_bytes(["Alex"], ["Example"], ["Exo"], signature=signature) + with pytest.raises(ReaderCheckError, match="name pools not found"): + locate(wrapped(pools)) + altered = signature[:9] + b"\x05" + signature[10:] + pools = name_pools_bytes(["Alex"], ["Example"], ["Exo"], signature=altered) + with pytest.raises(ReaderCheckError, match="name pools not found"): + locate(wrapped(pools), REGISTERED_LAYOUT) + + +def test_two_valid_pool_groups_differing_in_unchecked_bytes_are_ambiguous() -> None: + pools = example_pools() + flagged = name_pools_bytes( + ["Alex"], ["Example"], ["Exo"], signature=with_flag_bytes(NAME_POOL_SIGNATURE, 3) + ) + with pytest.raises(ReaderCheckError, match="not unique"): + locate(wrapped(pools + flagged), REGISTERED_LAYOUT) + + @pytest.mark.parametrize("words", [(7168, 1792, 2048), (7168, 1793, 14336), (0, 0, 0)]) def test_capacity_words_must_scale_together(words: tuple[int, int, int]) -> None: signature = struct.pack("<6I", 46421, 0, *words, 0) diff --git a/tests/test_players.py b/tests/test_players.py index df9363a..e62ad84 100644 --- a/tests/test_players.py +++ b/tests/test_players.py @@ -361,15 +361,19 @@ def test_doubled_uid_mismatch_is_rejected() -> None: assert 900006 not in player_records.uids +# Ordinary player records carry kind 2 or 3, so an allocated one can carry either (Refs #20). +@pytest.mark.parametrize("object_kind", [2, 3]) @pytest.mark.parametrize("marker", [bytes.fromhex("01006c07"), bytes(4)]) -def test_allocated_player_header_with_distinct_identity_word_is_decoded(marker: bytes) -> None: +def test_allocated_player_header_with_distinct_identity_word_is_decoded( + marker: bytes, object_kind: int +) -> None: record = player_record_bytes( **{ **PLAYER_A, "pindex": 30, "uid": 2_000_000_123, "identity_word": 810_123, - "object_kind": 2, + "object_kind": object_kind, "marker": marker, } ) @@ -385,7 +389,7 @@ def test_allocated_player_header_with_distinct_identity_word_is_decoded(marker: "override", [ {"object_kind": 1}, - {"object_kind": 3}, + {"object_kind": 4}, {"identity_word": 0}, {"identity_word": 0xFFFFFFFF}, {"current_ability": 201}, diff --git a/tests/test_staff.py b/tests/test_staff.py index 9e19b1e..dce66b0 100644 --- a/tests/test_staff.py +++ b/tests/test_staff.py @@ -63,7 +63,11 @@ person_block_bytes, player_record_bytes, ) -from tests.fixtures.staff import CAREER_STAFF_PREFERENCES, staff_object_bytes +from tests.fixtures.staff import ( + CAREER_STAFF_CODES, + CAREER_STAFF_PREFERENCES, + staff_object_bytes, +) BOUNDS = find_layout(GateBounds, GAME_DB_SECTION, GAME_DB_SCHEMA, BUILD_STRING).layout PLAYER_LAYOUT = find_layout( @@ -425,13 +429,19 @@ def test_a_decoy_of_another_object_kind_is_ignored(tmp_path: Path) -> None: assert staff_check.anomalies["ambiguous_headers"] == 0 -def test_allocated_staff_header_recovers_its_contract_name_and_ability(tmp_path: Path) -> None: +# The code set is the codes observed, not a closed set: ordinary staff carry codes outside it, +# so an allocated header with one is still a person's (Refs #20). +@pytest.mark.parametrize("codes", [CAREER_STAFF_CODES, (4, 29, 69, 4, 22, 27, 22, 27), (0,) * 8]) +def test_allocated_staff_header_recovers_its_contract_name_and_ability( + tmp_path: Path, codes: Sequence[int] +) -> None: record, _tag = _far_contract_record() person = staff_object_bytes( person_id=FAR_CONTRACT_PERSON_ID, uid=2_000_000_123, identity_word=810_123, contract=record, + codes=codes, person_block=person_block_bytes( first_name_id=0, surname_id=1, @@ -499,7 +509,6 @@ def test_high_person_selector_contract_is_discovered(tmp_path: Path) -> None: {"identity_word": 0xFFFFFFFF}, {"sentinel": 0}, {"current_ability": 0}, - {"codes": (0,) * 8}, {"preferences": (0,) * 26}, {"block_40": (0,) * 26}, ], diff --git a/tests/test_stages.py b/tests/test_stages.py index 81b91a3..eb95b9a 100644 --- a/tests/test_stages.py +++ b/tests/test_stages.py @@ -413,9 +413,7 @@ def test_healthy_stage_stats_pass_every_gate_and_a_small_section_applies_none() pytest.param({"with_competition": 6_400}, ["stage_rows_with_competition"], id="few-joins"), pytest.param({"bytes_after_table": 4_132_399}, [], id="older-build-career-tail"), pytest.param({"bytes_after_table": 20_693_727}, [], id="continued-career-tail"), - pytest.param( - {"bytes_after_table": 34_000_000}, ["stage_table_tail_bytes"], id="table-too-early" - ), + pytest.param({"bytes_after_table": 34_173_312}, [], id="long-career-tail"), pytest.param({"rows": 999}, ["stage_rows_minimum"], id="too-few-rows"), pytest.param({"ascending_steps": 7_000}, ["stage_ids_ascending"], id="ids-not-ascending"), pytest.param( @@ -576,13 +574,37 @@ def test_rows_before_the_search_window_are_kept_by_walking_back_to_the_head() -> assert stage_index.stages[0].id == 1 -def small_search_layout() -> StageTableLayout: +SMALL_MAX_SEARCH_BYTES = 40_000 + + +def small_search_layout(max_search_bytes: int | None = SMALL_MAX_SEARCH_BYTES) -> StageTableLayout: """The registered layout with its search windows scaled down, so a test pads kilobytes.""" return dataclasses.replace( - find_stage_layout(GAME_DB_SCHEMA, ""), search_bytes=10_000, max_search_bytes=40_000 + find_stage_layout(GAME_DB_SCHEMA, ""), + search_bytes=10_000, + max_search_bytes=max_search_bytes, ) +def test_an_uncapped_search_reaches_the_start_of_game_db() -> None: + """The bytes after the table grow with a career: 34.2 MB on one (Refs #20).""" + layout = small_search_layout(max_search_bytes=None) + trailing_bytes = layout.search_bytes * 37 + game_db = stage_table_bytes(career_stage_rows(), trailing_bytes=trailing_bytes) + + stage_index = read_stage_index(game_db, layout, FILE_NAME) + + assert len(stage_index.stages) == STAGE_ROW_COUNT + assert stage_index.stats.bytes_after_table == trailing_bytes + + +def test_an_uncapped_search_without_a_table_names_the_whole_section() -> None: + layout = small_search_layout(max_search_bytes=None) + + with pytest.raises(fmsave.ReaderCheckError, match="no stage table was found in game_db"): + read_stage_index(b"\x01" * 50_000, layout, FILE_NAME) + + def test_a_table_further_back_than_the_first_window_is_found_by_widening_it() -> None: """One community career has 4.1 MB after its table, twice the first window's 2 MB.""" layout = small_search_layout() @@ -607,12 +629,12 @@ def test_a_table_with_twenty_megabytes_after_it_is_found() -> None: def test_a_table_past_the_widest_window_is_not_found() -> None: layout = small_search_layout() - game_db = stage_table_bytes(career_stage_rows(), trailing_bytes=layout.max_search_bytes) + game_db = stage_table_bytes(career_stage_rows(), trailing_bytes=SMALL_MAX_SEARCH_BYTES) with pytest.raises(fmsave.ReaderCheckError) as error_info: read_stage_index(game_db, layout, FILE_NAME) - assert f"no stage table was found in the last {layout.max_search_bytes:,} bytes" in str( + assert f"no stage table was found in the last {SMALL_MAX_SEARCH_BYTES:,} bytes" in str( error_info.value ) diff --git a/tests/test_tactics.py b/tests/test_tactics.py index 0db21a6..4f45db3 100644 --- a/tests/test_tactics.py +++ b/tests/test_tactics.py @@ -2,6 +2,7 @@ import copy import pickle +import struct from collections.abc import Sequence from pathlib import Path @@ -107,6 +108,7 @@ def healthy_stats(**overrides: object) -> TacticStats: "managed_club_exists": True, "club_team_count": 5, "header_blocks": 5, + "stored_blocks": 5, "blocks_found": 5, "selector_matches": True, "selection_selectors": 1069, @@ -259,6 +261,7 @@ def test_the_walk_counts_the_blocks_the_records_and_the_selectors(career_path: P _tactics, _routines, stats = decoded_tactics(career_path) assert stats.header_blocks == 2 + assert stats.stored_blocks == 2 assert stats.blocks_found == 2 assert stats.club_team_count == 2 assert stats.selector_matches is True @@ -510,6 +513,36 @@ def test_a_team_id_the_section_stores_twice_gets_no_block(tmp_path: Path) -> Non assert {row.team_id for row in routines} == {NORTHBRIDGE_TEAM_A} +def test_a_national_team_block_counts_for_the_header_and_ends_the_club_block( + tmp_path: Path, +) -> None: + """A manager who also runs a national team has a block the club index lists no team for.""" + national_block = selection_part_bytes( + team_id=999_001, + label="", + slots=(), + list_a=(), + list_b=(), + single=NO_SELECTOR, + tactics_value=HAS_TACTICS_VALUE, + tactic_count=0, + ) + body = bytearray(career_tactics_body()) + count_at = body.index(struct.pack(" None: save_path = tactics_path(tmp_path, career_tactics_body(selector=MANAGER_SELECTOR + 3)) @@ -662,6 +695,15 @@ def test_the_tables_are_read_once_and_raise_after_the_save_is_closed(career_path pytest.param({"selector_matches": False}, [SELECTOR_GATE_NAME], id="a foreign selector"), pytest.param({"blocks_found": 4}, [BLOCKS_GATE_NAME], id="a team with no block"), pytest.param({"header_blocks": 6}, [BLOCKS_GATE_NAME], id="a header count of its own"), + pytest.param( + {"header_blocks": 1_610_612_736}, [BLOCKS_GATE_NAME], id="a header read a byte late" + ), + pytest.param({"stored_blocks": 0}, [BLOCKS_GATE_NAME], id="no block where header ends"), + pytest.param( + {"header_blocks": 4, "stored_blocks": 4}, [BLOCKS_GATE_NAME], id="fewer than the club" + ), + # A manager who also runs national teams has their blocks too (Refs #20). + pytest.param({"header_blocks": 12, "stored_blocks": 12}, [], id="national teams too"), pytest.param({"slot_walks_complete": 3}, [SLOT_WALK_GATE_NAME], id="half the walks"), pytest.param({"oop_index_permutations": 3}, [INDEX_GATE_NAME], id="half the indexes"), pytest.param( diff --git a/uv.lock b/uv.lock index 3aabb28..787e0f0 100644 --- a/uv.lock +++ b/uv.lock @@ -359,7 +359,7 @@ wheels = [ [[package]] name = "fmsave" -version = "0.5.8" +version = "0.5.9" source = { editable = "." } dependencies = [ { name = "backports-zstd", marker = "python_full_version < '3.14'" },