diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4ec7a1fc..2ca9eb7e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,74 +44,27 @@ jobs: unzip -q /tmp/godot.zip -d ~/godot mv ~/godot/Godot_v${GODOT_VERSION}_linux.x86_64 ~/godot/godot - - name: Import project - run: ~/godot/godot --headless --path "$PROJECT" --import + - name: Cache imported resources + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ env.PROJECT }}/.godot + key: godot-import-${{ env.GODOT_VERSION }}-${{ hashFiles('hoppys-adventure-arcade/project.godot', 'hoppys-adventure-arcade/**/*.import') }} + # A stale cache is safe: Godot reimports any source whose hash changed. + restore-keys: godot-import-${{ env.GODOT_VERSION }}- - # Headless suites only. Not listed here: - # - capture_runner_menu_previews, perf_probe, runner_perf_probe, runner_pickup_hitch - # need a real window/GPU and are run locally (see CONTRIBUTING.md). - # - menu_smoke and menu_hoppy_smoke rely on pointer hover, which doesn't happen headless - # (2 failures each, also on main). - - name: Run headless suites - run: | - suites=( - code_city_assets - code_city_celebration - code_city_logic - code_city_mini_boss - code_city_motion - code_city_smoke - controller_service - stage_flow - beatemup_knockout - beatemup_story - beatemup_minimap - heroes_villains_assets - heroes_villains_logic - heroes_villains_motion - heroes_villains_smoke - hoppy_armed_assets - enchanted_assets - enchanted_boundaries - enchanted_repo_smoke - menu_return_layout - orbital_assets - orbital_ops_smoke - platformer_boss - platformer_knockout - platformer_smoke - platformer_traversal - runner_boss_validator - runner_chunk_validator - runner_logic - runner_run_intro - runner_run_intro_ui - runner_scaffold_check - runner_smoke - runner_synth_threads - runner_visual_smoke - runner_warmup_check - ) - failed=() - for suite in "${suites[@]}"; do - echo "::group::$suite" - log="/tmp/$suite.log" - set +e - timeout 600 ~/godot/godot --headless --path "$PROJECT" \ - --script "res://tests/$suite.gd" 2>&1 | tee "$log" - rc=${PIPESTATUS[0]} - set -e - echo "::endgroup::" - # A script that fails to parse can still exit 0, so treat SCRIPT ERROR as a failure too. - if [ "$rc" -ne 0 ] || grep -q "SCRIPT ERROR" "$log"; then - echo "::error title=$suite::failed (exit $rc)" - failed+=("$suite") - fi - done - if [ ${#failed[@]} -gt 0 ]; then - echo "Failed suites: ${failed[*]}" - exit 1 - fi + # The runner imports first (failing on any engine error), then runs every tests/**/*.gd; + # tools/tests/suites.toml lists the few skipped or serial suites and why. + - name: Import and run headless suites + run: python3 tools/tests/run_tests.py --godot ~/godot/godot --jobs 4 + + - name: Upload test results + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: test-results + path: build/test-results/ + retention-days: 7 + if-no-files-found: ignore # Quits from the hub, fights, the Code City celebration, the runner intro and mid-transition # with their audio playing. Godot prints its exit-leak lines only after the script is gone, @@ -133,6 +86,8 @@ jobs: - run: pip install ruff gdtoolkit==4.* - name: Ruff (tools) run: ruff check tools hoppys-adventure-arcade/tools + - name: Test runner unit tests + run: python -m unittest discover -s tools/tests -v # Advisory until the existing code is formatted; make it blocking afterwards. - name: gdlint (advisory) continue-on-error: true diff --git a/.gitignore b/.gitignore index 184628f8..ee942587 100644 --- a/.gitignore +++ b/.gitignore @@ -24,5 +24,5 @@ __pycache__/ .vscode/ .DS_Store -# Exported game builds (tools/build/export_game.py) +# Build outputs: exported games (tools/build) and test results (tools/tests) /build/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 41f3a684..101b8f3d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -29,14 +29,26 @@ Thanks for helping out! This guide covers setup, workflow, testing and the asset ## Tests -The tests are standalone `SceneTree` scripts. Each one exits nonzero on failure: +The tests are standalone `SceneTree` scripts under `hoppys-adventure-arcade/tests/`. Run them all the way CI does: + +```sh +python3 tools/tests/run_tests.py --godot /path/to/Godot # or set $GODOT +python3 tools/tests/run_tests.py -k runner # suites whose name contains "runner" +python3 tools/tests/run_tests.py --list # what runs and what is skipped +``` + +- **Every new `.gd` file under `tests/` that `extends SceneTree` or `MainLoop` runs automatically.** Shared helpers that extend anything else, such as `tests/support/`, are never run as suites. There's no list to update. To keep a suite out of CI (it needs a GPU, a window or pointer hover), add it to `[skip]` in [`tools/tests/suites.toml`](tools/tests/suites.toml) with the reason. +- **A suite fails if it exits nonzero, times out, or prints an engine or script error or an exit-time leak.** Suites run with `--verbose`, so a leak names the objects involved. The one tolerated shape is the engine's audio-teardown race: only `AudioStream*` playbacks and audio files, reported as a note. Anything else that leaks fails. That includes `ERROR:`, `SCRIPT ERROR`, parse errors and "leaked at exit". If a test provokes an engine error on purpose, list a pattern for it under `[expected_errors]`; the runner fails if that pattern stops matching. +- **Finish a suite with `await root.get_node("AudioService").settle()` before `quit()`.** It stops the shared music and waits until the audio thread has released it. A fixed delay is not enough under parallel load. +- Each suite gets a private user-data root (`HOME`, the XDG directories, and `APPDATA` on Windows), so `user://` files (settings, leaderboards) never touch your real ones. The runner checks this with Godot before starting and refuses to run if `user://` would escape. Suites listed under `[serial]` (wall-clock budgets, or suites that occupy every core) run alone after the parallel batch. Results are written to `build/test-results/summary.json`. + +To run a single suite directly: ```sh godot --headless --path hoppys-adventure-arcade --script res://tests/runner_logic.gd -godot --headless --path hoppys-adventure-arcade --script res://tests/platformer_smoke.gd ``` -CI runs every headless suite listed in `.github/workflows/ci.yml`. It also runs `python3 tools/checks/exit_leak.py --godot `, which quits from screens with audio playing and fails on Godot's "leaked at exit" lines. Add a focused regression test when you change behavior. Some suites capture screenshots when run without `--headless`; include those in the PR for visual changes. On macOS, pass `--always-on-top` for graphical runs so the capture doesn't stall when the window is covered. +CI also runs `python3 tools/checks/exit_leak.py --godot `, which quits from screens with audio playing and fails on Godot's "leaked at exit" lines. Add a focused regression test when you change behavior. Some suites capture screenshots when run without `--headless`; include those in the PR for visual changes. On macOS, pass `--always-on-top` for graphical runs so the capture doesn't stall when the window is covered. Per-mode notes and test details live in [`docs/`](docs/). diff --git a/README.md b/README.md index 43997e78..bf0bff7f 100644 --- a/README.md +++ b/README.md @@ -43,11 +43,10 @@ Versioned macOS, Windows, Linux and Web builds come from the [Build workflow](.g ## Tests ```sh -godot --headless --path hoppys-adventure-arcade --import -godot --headless --path hoppys-adventure-arcade --script res://tests/runner_logic.gd +python3 tools/tests/run_tests.py --godot /path/to/Godot ``` -Each file in `hoppys-adventure-arcade/tests/` is a standalone script that exits nonzero on failure. CI runs the headless suites on every PR. See [CONTRIBUTING.md](CONTRIBUTING.md) for the full workflow. +CI discovers the suites in `hoppys-adventure-arcade/tests/`: every script that extends `SceneTree` or `MainLoop`. Helpers such as `tests/support/` are not suites. It runs all of them on every PR, except the few listed with a reason in [`tools/tests/suites.toml`](tools/tests/suites.toml). A suite fails on a nonzero exit, an engine or script error, or a leak. See [CONTRIBUTING.md](CONTRIBUTING.md#tests) for details. ## Contributing diff --git a/hoppys-adventure-arcade/Assets/runner/hoppy/hoppy_accessory_albedo.png.import b/hoppys-adventure-arcade/Assets/runner/hoppy/hoppy_accessory_albedo.png.import index 4556aa30..e5135bce 100644 --- a/hoppys-adventure-arcade/Assets/runner/hoppy/hoppy_accessory_albedo.png.import +++ b/hoppys-adventure-arcade/Assets/runner/hoppy/hoppy_accessory_albedo.png.import @@ -3,10 +3,9 @@ importer="texture" type="CompressedTexture2D" uid="uid://0smaw6gfvyj5" -path.s3tc="res://.godot/imported/hoppy_accessory_albedo.png-c122b57ea96ce5ad67ded19ac527ca90.s3tc.ctex" +path="res://.godot/imported/hoppy_accessory_albedo.png-c122b57ea96ce5ad67ded19ac527ca90.ctex" metadata={ -"imported_formats": ["s3tc_bptc"], -"vram_texture": true +"vram_texture": false } generator_parameters={ "md5": "12ea3843bd9e25b4241e0d5e33b33f0e" @@ -15,11 +14,11 @@ generator_parameters={ [deps] source_file="res://Assets/runner/hoppy/hoppy_accessory_albedo.png" -dest_files=["res://.godot/imported/hoppy_accessory_albedo.png-c122b57ea96ce5ad67ded19ac527ca90.s3tc.ctex"] +dest_files=["res://.godot/imported/hoppy_accessory_albedo.png-c122b57ea96ce5ad67ded19ac527ca90.ctex"] [params] -compress/mode=2 +compress/mode=0 compress/high_quality=false compress/lossy_quality=0.7 compress/uastc_level=0 diff --git a/hoppys-adventure-arcade/Assets/runner/hoppy/hoppy_accessory_emissive.png.import b/hoppys-adventure-arcade/Assets/runner/hoppy/hoppy_accessory_emissive.png.import index 99c24327..11527a38 100644 --- a/hoppys-adventure-arcade/Assets/runner/hoppy/hoppy_accessory_emissive.png.import +++ b/hoppys-adventure-arcade/Assets/runner/hoppy/hoppy_accessory_emissive.png.import @@ -3,10 +3,9 @@ importer="texture" type="CompressedTexture2D" uid="uid://dk6fxo7twe0us" -path.s3tc="res://.godot/imported/hoppy_accessory_emissive.png-12050a52c862b1593e1bc121f8f122fa.s3tc.ctex" +path="res://.godot/imported/hoppy_accessory_emissive.png-12050a52c862b1593e1bc121f8f122fa.ctex" metadata={ -"imported_formats": ["s3tc_bptc"], -"vram_texture": true +"vram_texture": false } generator_parameters={ "md5": "ee72f210ceb66159b7d3d224a51ebf7a" @@ -15,11 +14,11 @@ generator_parameters={ [deps] source_file="res://Assets/runner/hoppy/hoppy_accessory_emissive.png" -dest_files=["res://.godot/imported/hoppy_accessory_emissive.png-12050a52c862b1593e1bc121f8f122fa.s3tc.ctex"] +dest_files=["res://.godot/imported/hoppy_accessory_emissive.png-12050a52c862b1593e1bc121f8f122fa.ctex"] [params] -compress/mode=2 +compress/mode=0 compress/high_quality=false compress/lossy_quality=0.7 compress/uastc_level=0 diff --git a/hoppys-adventure-arcade/Assets/runner/hoppy/hoppy_accessory_normal.png.import b/hoppys-adventure-arcade/Assets/runner/hoppy/hoppy_accessory_normal.png.import index 743e2ae2..ed693784 100644 --- a/hoppys-adventure-arcade/Assets/runner/hoppy/hoppy_accessory_normal.png.import +++ b/hoppys-adventure-arcade/Assets/runner/hoppy/hoppy_accessory_normal.png.import @@ -3,10 +3,9 @@ importer="texture" type="CompressedTexture2D" uid="uid://b27dm5dn5qhlj" -path.s3tc="res://.godot/imported/hoppy_accessory_normal.png-8498ac34154d1c36039fcb9428c418ee.s3tc.ctex" +path="res://.godot/imported/hoppy_accessory_normal.png-8498ac34154d1c36039fcb9428c418ee.ctex" metadata={ -"imported_formats": ["s3tc_bptc"], -"vram_texture": true +"vram_texture": false } generator_parameters={ "md5": "ca837196a548101c4ce1d30a9644f167" @@ -15,11 +14,11 @@ generator_parameters={ [deps] source_file="res://Assets/runner/hoppy/hoppy_accessory_normal.png" -dest_files=["res://.godot/imported/hoppy_accessory_normal.png-8498ac34154d1c36039fcb9428c418ee.s3tc.ctex"] +dest_files=["res://.godot/imported/hoppy_accessory_normal.png-8498ac34154d1c36039fcb9428c418ee.ctex"] [params] -compress/mode=2 +compress/mode=0 compress/high_quality=false compress/lossy_quality=0.7 compress/uastc_level=0 diff --git a/hoppys-adventure-arcade/Assets/runner/hoppy/hoppy_accessory_orm.png.import b/hoppys-adventure-arcade/Assets/runner/hoppy/hoppy_accessory_orm.png.import index 7838bf8b..9186cc7a 100644 --- a/hoppys-adventure-arcade/Assets/runner/hoppy/hoppy_accessory_orm.png.import +++ b/hoppys-adventure-arcade/Assets/runner/hoppy/hoppy_accessory_orm.png.import @@ -3,10 +3,9 @@ importer="texture" type="CompressedTexture2D" uid="uid://bq52lv7petqa7" -path.s3tc="res://.godot/imported/hoppy_accessory_orm.png-a24d2bbfa7539917534e8ce2748bee64.s3tc.ctex" +path="res://.godot/imported/hoppy_accessory_orm.png-a24d2bbfa7539917534e8ce2748bee64.ctex" metadata={ -"imported_formats": ["s3tc_bptc"], -"vram_texture": true +"vram_texture": false } generator_parameters={ "md5": "16227a4c91bb686a5186ac40be1c526f" @@ -15,11 +14,11 @@ generator_parameters={ [deps] source_file="res://Assets/runner/hoppy/hoppy_accessory_orm.png" -dest_files=["res://.godot/imported/hoppy_accessory_orm.png-a24d2bbfa7539917534e8ce2748bee64.s3tc.ctex"] +dest_files=["res://.godot/imported/hoppy_accessory_orm.png-a24d2bbfa7539917534e8ce2748bee64.ctex"] [params] -compress/mode=2 +compress/mode=0 compress/high_quality=false compress/lossy_quality=0.7 compress/uastc_level=0 diff --git a/hoppys-adventure-arcade/Assets/runner/hoppy/hoppy_body_albedo.png.import b/hoppys-adventure-arcade/Assets/runner/hoppy/hoppy_body_albedo.png.import index e3441b66..ab0a8960 100644 --- a/hoppys-adventure-arcade/Assets/runner/hoppy/hoppy_body_albedo.png.import +++ b/hoppys-adventure-arcade/Assets/runner/hoppy/hoppy_body_albedo.png.import @@ -3,10 +3,9 @@ importer="texture" type="CompressedTexture2D" uid="uid://dk1u3h8nede88" -path.s3tc="res://.godot/imported/hoppy_body_albedo.png-a4c37408013e37c8de18072982be504d.s3tc.ctex" +path="res://.godot/imported/hoppy_body_albedo.png-a4c37408013e37c8de18072982be504d.ctex" metadata={ -"imported_formats": ["s3tc_bptc"], -"vram_texture": true +"vram_texture": false } generator_parameters={ "md5": "8d1ce8c41029007e13f41ee644bd020e" @@ -15,11 +14,11 @@ generator_parameters={ [deps] source_file="res://Assets/runner/hoppy/hoppy_body_albedo.png" -dest_files=["res://.godot/imported/hoppy_body_albedo.png-a4c37408013e37c8de18072982be504d.s3tc.ctex"] +dest_files=["res://.godot/imported/hoppy_body_albedo.png-a4c37408013e37c8de18072982be504d.ctex"] [params] -compress/mode=2 +compress/mode=0 compress/high_quality=false compress/lossy_quality=0.7 compress/uastc_level=0 diff --git a/hoppys-adventure-arcade/Assets/runner/hoppy/hoppy_body_emissive.png.import b/hoppys-adventure-arcade/Assets/runner/hoppy/hoppy_body_emissive.png.import index af6f5bcc..681166a6 100644 --- a/hoppys-adventure-arcade/Assets/runner/hoppy/hoppy_body_emissive.png.import +++ b/hoppys-adventure-arcade/Assets/runner/hoppy/hoppy_body_emissive.png.import @@ -3,10 +3,9 @@ importer="texture" type="CompressedTexture2D" uid="uid://cnwbddowongvs" -path.s3tc="res://.godot/imported/hoppy_body_emissive.png-15b9193f78dd32e62b3bc8d4f2eb5591.s3tc.ctex" +path="res://.godot/imported/hoppy_body_emissive.png-15b9193f78dd32e62b3bc8d4f2eb5591.ctex" metadata={ -"imported_formats": ["s3tc_bptc"], -"vram_texture": true +"vram_texture": false } generator_parameters={ "md5": "efeebdda98ec1d7fb2ad83d23f0713bf" @@ -15,11 +14,11 @@ generator_parameters={ [deps] source_file="res://Assets/runner/hoppy/hoppy_body_emissive.png" -dest_files=["res://.godot/imported/hoppy_body_emissive.png-15b9193f78dd32e62b3bc8d4f2eb5591.s3tc.ctex"] +dest_files=["res://.godot/imported/hoppy_body_emissive.png-15b9193f78dd32e62b3bc8d4f2eb5591.ctex"] [params] -compress/mode=2 +compress/mode=0 compress/high_quality=false compress/lossy_quality=0.7 compress/uastc_level=0 diff --git a/hoppys-adventure-arcade/Assets/runner/hoppy/hoppy_body_normal.png.import b/hoppys-adventure-arcade/Assets/runner/hoppy/hoppy_body_normal.png.import index e609bffd..2bf62b1c 100644 --- a/hoppys-adventure-arcade/Assets/runner/hoppy/hoppy_body_normal.png.import +++ b/hoppys-adventure-arcade/Assets/runner/hoppy/hoppy_body_normal.png.import @@ -3,10 +3,9 @@ importer="texture" type="CompressedTexture2D" uid="uid://d2rcbm37d6nu4" -path.s3tc="res://.godot/imported/hoppy_body_normal.png-904e19f0aa8db2d232ac2f9e1ac495c8.s3tc.ctex" +path="res://.godot/imported/hoppy_body_normal.png-904e19f0aa8db2d232ac2f9e1ac495c8.ctex" metadata={ -"imported_formats": ["s3tc_bptc"], -"vram_texture": true +"vram_texture": false } generator_parameters={ "md5": "d338760e3eb0d148b22941e704792983" @@ -15,11 +14,11 @@ generator_parameters={ [deps] source_file="res://Assets/runner/hoppy/hoppy_body_normal.png" -dest_files=["res://.godot/imported/hoppy_body_normal.png-904e19f0aa8db2d232ac2f9e1ac495c8.s3tc.ctex"] +dest_files=["res://.godot/imported/hoppy_body_normal.png-904e19f0aa8db2d232ac2f9e1ac495c8.ctex"] [params] -compress/mode=2 +compress/mode=0 compress/high_quality=false compress/lossy_quality=0.7 compress/uastc_level=0 diff --git a/hoppys-adventure-arcade/Assets/runner/hoppy/hoppy_body_orm.png.import b/hoppys-adventure-arcade/Assets/runner/hoppy/hoppy_body_orm.png.import index 8c320401..9bb1c5d0 100644 --- a/hoppys-adventure-arcade/Assets/runner/hoppy/hoppy_body_orm.png.import +++ b/hoppys-adventure-arcade/Assets/runner/hoppy/hoppy_body_orm.png.import @@ -3,10 +3,9 @@ importer="texture" type="CompressedTexture2D" uid="uid://bjkoyu6nxsyci" -path.s3tc="res://.godot/imported/hoppy_body_orm.png-a36dae46ff5d3b78351e386b721c189f.s3tc.ctex" +path="res://.godot/imported/hoppy_body_orm.png-a36dae46ff5d3b78351e386b721c189f.ctex" metadata={ -"imported_formats": ["s3tc_bptc"], -"vram_texture": true +"vram_texture": false } generator_parameters={ "md5": "f684053ef742dca3be2d514aacf3a5a3" @@ -15,11 +14,11 @@ generator_parameters={ [deps] source_file="res://Assets/runner/hoppy/hoppy_body_orm.png" -dest_files=["res://.godot/imported/hoppy_body_orm.png-a36dae46ff5d3b78351e386b721c189f.s3tc.ctex"] +dest_files=["res://.godot/imported/hoppy_body_orm.png-a36dae46ff5d3b78351e386b721c189f.ctex"] [params] -compress/mode=2 +compress/mode=0 compress/high_quality=false compress/lossy_quality=0.7 compress/uastc_level=0 diff --git a/hoppys-adventure-arcade/app/audio_service.gd b/hoppys-adventure-arcade/app/audio_service.gd index 563afd31..07db13d9 100644 --- a/hoppys-adventure-arcade/app/audio_service.gd +++ b/hoppys-adventure-arcade/app/audio_service.gd @@ -96,6 +96,19 @@ func shutdown() -> void: music.stream = null sfx.stream = null +## Stops all audio, then waits until the audio thread has let go of every playback, so the +## caller can quit without "leaked at exit" reports. A fixed delay races the mixer under load. +func settle(max_seconds := 2.0) -> void: + var held: Array[AudioStreamPlayback] = [] + for player: AudioStreamPlayer in [music, sfx]: + if player.has_stream_playback(): + held.append(player.get_stream_playback()) + shutdown() + var deadline := Time.get_ticks_msec() + int(max_seconds * 1000.0) + while held.any(func(playback: AudioStreamPlayback) -> bool: return playback.get_reference_count() > 1) \ + and Time.get_ticks_msec() < deadline: + await get_tree().create_timer(0.02, true, false, true).timeout + ## The autoload only leaves the tree when the app exits, after the current scene has left it. ## A voice the mixer still holds when the engine shuts down is reported as leaked ("ObjectDB ## instances were leaked", "resources still in use"), and a stopped voice is let go only by the diff --git a/hoppys-adventure-arcade/tests/assets/a3_check.gd b/hoppys-adventure-arcade/tests/assets/a3_check.gd index 2fbfb73f..8bcc8b10 100644 --- a/hoppys-adventure-arcade/tests/assets/a3_check.gd +++ b/hoppys-adventure-arcade/tests/assets/a3_check.gd @@ -247,6 +247,7 @@ func _run() -> void: check(runtime_errors.count == 0,"no runtime errors during asset checks") preload("res://scenes/runner/hazards/visuals/hazard_art.gd").release_caches() print("ASSET A3: %s (%d checks)" % ["PASS" if failures == 0 else "FAIL",checks]) + await root.get_node("AudioService").settle() quit(0 if failures == 0 else 1) func _round_two(visual: Base, title: String) -> void: diff --git a/hoppys-adventure-arcade/tests/assets/a6_check.gd b/hoppys-adventure-arcade/tests/assets/a6_check.gd index bc5e93c0..bfb5565c 100644 --- a/hoppys-adventure-arcade/tests/assets/a6_check.gd +++ b/hoppys-adventure-arcade/tests/assets/a6_check.gd @@ -87,7 +87,6 @@ func run() -> void: print("A6 HUNTER CENSUS: %d draws, %d triangles, no textures" % [draws,triangles]) print("A6 HUNTER ENVELOPE: ",envelope) visual.free() - root.get_node("AudioService").shutdown() - await create_timer(0.3).timeout + await root.get_node("AudioService").settle() print("ASSET A6: %s (%d checks)" % ["PASS" if failures == 0 else "FAIL",checks]) quit(0 if failures == 0 else 1) diff --git a/hoppys-adventure-arcade/tests/assets/a9_check.gd b/hoppys-adventure-arcade/tests/assets/a9_check.gd index 114f94ed..46397121 100644 --- a/hoppys-adventure-arcade/tests/assets/a9_check.gd +++ b/hoppys-adventure-arcade/tests/assets/a9_check.gd @@ -222,8 +222,7 @@ func run() -> void: check(not fixture.post.settings.pursuit and value(fixture.post, "grain") > 0.0, "preview restores safe integrated defaults") fixture.free() if root.has_node("AudioService"): - root.get_node("AudioService").shutdown() - await create_timer(0.3).timeout + await root.get_node("AudioService").settle() print("ASSET A9: %s (%d checks)" % ["PASS" if failures == 0 else "FAIL", checks]) quit(0 if failures == 0 else 1) diff --git a/hoppys-adventure-arcade/tests/assets/starship_check.gd b/hoppys-adventure-arcade/tests/assets/starship_check.gd index c1910258..6a734b1a 100644 --- a/hoppys-adventure-arcade/tests/assets/starship_check.gd +++ b/hoppys-adventure-arcade/tests/assets/starship_check.gd @@ -105,4 +105,5 @@ func run() -> void: print("STARSHIP TOTAL: %d triangles including deck, far bulkhead, entry bulkhead and reactor dock" % (triangles + extra_triangles)) print("STARSHIP GEOMETRY: repeated modules=%d triangles; 6 batches; 1 shared 256x256 texture" % triangles) print("STARSHIP CHECK: %s (%d checks, %d failures)" % ["PASS" if failures == 0 else "FAIL",checks,failures]) + await root.get_node("AudioService").settle() quit(0 if failures == 0 else 1) diff --git a/hoppys-adventure-arcade/tests/beatemup_knockout.gd b/hoppys-adventure-arcade/tests/beatemup_knockout.gd index 281f61ab..833c04f3 100644 --- a/hoppys-adventure-arcade/tests/beatemup_knockout.gd +++ b/hoppys-adventure-arcade/tests/beatemup_knockout.gd @@ -192,8 +192,7 @@ func run() -> void: await run_mode("res://scenes/beatem_up/reinvent/heroes_vs_villains.tscn", true) print("Knockout passes: ", passes, " failures: ", failures) if is_instance_valid(current_scene): current_scene.queue_free() - root.get_node("AudioService").shutdown() - await create_timer(.4).timeout + await root.get_node("AudioService").settle() quit(1 if failures else 0) ## Takes the scene just changed to as `game` once it has built. Without it the suite cannot go ## on: records a failure, quits 1 and returns false. diff --git a/hoppys-adventure-arcade/tests/beatemup_minimap.gd b/hoppys-adventure-arcade/tests/beatemup_minimap.gd index b91cb0fa..9d63a1de 100644 --- a/hoppys-adventure-arcade/tests/beatemup_minimap.gd +++ b/hoppys-adventure-arcade/tests/beatemup_minimap.gd @@ -56,8 +56,7 @@ func run() -> void: print("Minimap checks: ", checks, " failures: ", failures) paused = false if is_instance_valid(current_scene): current_scene.queue_free() - root.get_node("AudioService").shutdown() - await create_timer(0.4).timeout + await root.get_node("AudioService").settle() quit(1 if failures else 0) ## The pure helpers, on a standalone minimap. diff --git a/hoppys-adventure-arcade/tests/beatemup_story.gd b/hoppys-adventure-arcade/tests/beatemup_story.gd index 6213f198..c257dd9c 100644 --- a/hoppys-adventure-arcade/tests/beatemup_story.gd +++ b/hoppys-adventure-arcade/tests/beatemup_story.gd @@ -84,8 +84,7 @@ func run() -> void: print("Story checks: ", checks, " failures: ", failures) paused = false if is_instance_valid(current_scene): current_scene.queue_free() - root.get_node("AudioService").shutdown() - await create_timer(0.4).timeout + await root.get_node("AudioService").settle() quit(1 if failures else 0) func check_textures() -> void: diff --git a/hoppys-adventure-arcade/tests/booth_mode.gd b/hoppys-adventure-arcade/tests/booth_mode.gd index 74c0ea34..1f2b1318 100644 --- a/hoppys-adventure-arcade/tests/booth_mode.gd +++ b/hoppys-adventure-arcade/tests/booth_mode.gd @@ -86,7 +86,6 @@ func run() -> void: router.set_booth(false) router.idle_limit = router.IDLE_SECONDS - root.get_node("AudioService").shutdown() - await wait(0.4) + await root.get_node("AudioService").settle() print("Booth mode failures: ", failures) quit(1 if failures else 0) diff --git a/hoppys-adventure-arcade/tests/code_city_assets.gd b/hoppys-adventure-arcade/tests/code_city_assets.gd index 999cc3f3..3c4f8f07 100644 --- a/hoppys-adventure-arcade/tests/code_city_assets.gd +++ b/hoppys-adventure-arcade/tests/code_city_assets.gd @@ -253,6 +253,5 @@ func piece(key: String, clip: String, pos: Vector2, size: Vector2) -> void: stage.add_child(sprite) func finish(code: int = 0) -> void: - root.get_node("AudioService").shutdown() - await create_timer(0.3).timeout + await root.get_node("AudioService").settle() quit(code) diff --git a/hoppys-adventure-arcade/tests/code_city_logic.gd b/hoppys-adventure-arcade/tests/code_city_logic.gd index fa370e85..a453f974 100644 --- a/hoppys-adventure-arcade/tests/code_city_logic.gd +++ b/hoppys-adventure-arcade/tests/code_city_logic.gd @@ -222,6 +222,5 @@ func run() -> void: check(InputMap.event_is_action(pad,binding[0]),"%s answers pad %d" % [binding[0],device]) print("Code City logic failures: ",failures) game.queue_free() - root.get_node("AudioService").shutdown() - await create_timer(.4).timeout + await root.get_node("AudioService").settle() quit(1 if failures else 0) diff --git a/hoppys-adventure-arcade/tests/code_city_motion.gd b/hoppys-adventure-arcade/tests/code_city_motion.gd index 5f7984b0..0d417149 100644 --- a/hoppys-adventure-arcade/tests/code_city_motion.gd +++ b/hoppys-adventure-arcade/tests/code_city_motion.gd @@ -1007,6 +1007,5 @@ func run() -> void: check(boss.position==transition_pos and not boss.leaping and not boss.core_cue.visible,"Defeat clears movement and the exposed-core cue") print("Code City motion failures: ",failures) game.queue_free() - root.get_node("AudioService").shutdown() - await create_timer(.4).timeout + await root.get_node("AudioService").settle() quit(1 if failures else 0) diff --git a/hoppys-adventure-arcade/tests/code_city_smoke.gd b/hoppys-adventure-arcade/tests/code_city_smoke.gd index 693fadd7..5d4a9f43 100644 --- a/hoppys-adventure-arcade/tests/code_city_smoke.gd +++ b/hoppys-adventure-arcade/tests/code_city_smoke.gd @@ -180,6 +180,5 @@ func run() -> void: await process_frame;await process_frame check(not paused and current_scene.name=="BeatemMenu","Completion returns cleanly to the mode menu") current_scene.queue_free() - root.get_node("AudioService").shutdown() - await create_timer(.4).timeout + await root.get_node("AudioService").settle() quit(1 if failures else 0) diff --git a/hoppys-adventure-arcade/tests/enchanted_assets.gd b/hoppys-adventure-arcade/tests/enchanted_assets.gd index cbcccd84..c5035cd7 100644 --- a/hoppys-adventure-arcade/tests/enchanted_assets.gd +++ b/hoppys-adventure-arcade/tests/enchanted_assets.gd @@ -102,8 +102,7 @@ func run() -> void: await finish() func finish(code: int = 0) -> void: - root.get_node("AudioService").shutdown() - await create_timer(0.3).timeout + await root.get_node("AudioService").settle() quit(code) func plate(key: String, pos: Vector2, size: Vector2, opacity: float = 1.0) -> void: diff --git a/hoppys-adventure-arcade/tests/enchanted_boundaries.gd b/hoppys-adventure-arcade/tests/enchanted_boundaries.gd index 6ef6e73c..30badf1b 100644 --- a/hoppys-adventure-arcade/tests/enchanted_boundaries.gd +++ b/hoppys-adventure-arcade/tests/enchanted_boundaries.gd @@ -80,6 +80,5 @@ func run() -> void: check(current_scene.boss.hp==12 and current_scene.boss.armor_stage==0,"Actor state never mutates shared SpriteFrames") change_scene_to_file("res://scenes/platformer/platformer_menu.tscn") await frames(3) - root.get_node("AudioService").shutdown() - await create_timer(.3).timeout + await root.get_node("AudioService").settle() quit(failures) diff --git a/hoppys-adventure-arcade/tests/enchanted_repo_smoke.gd b/hoppys-adventure-arcade/tests/enchanted_repo_smoke.gd index f3d7a85e..d07c26ae 100644 --- a/hoppys-adventure-arcade/tests/enchanted_repo_smoke.gd +++ b/hoppys-adventure-arcade/tests/enchanted_repo_smoke.gd @@ -284,6 +284,5 @@ func shutdown() -> void: paused=false change_scene_to_file("res://scenes/platformer/platformer_menu.tscn") await frames(2) - root.get_node("AudioService").shutdown() - await create_timer(.3).timeout + await root.get_node("AudioService").settle() quit(failures) diff --git a/hoppys-adventure-arcade/tests/heroes_villains_assets.gd b/hoppys-adventure-arcade/tests/heroes_villains_assets.gd index 1a164754..c08a9615 100644 --- a/hoppys-adventure-arcade/tests/heroes_villains_assets.gd +++ b/hoppys-adventure-arcade/tests/heroes_villains_assets.gd @@ -95,6 +95,5 @@ func run() -> void: else: printerr(failures, " asset checks failed") resources.clear() - root.get_node("AudioService").shutdown() - await create_timer(0.3).timeout + await root.get_node("AudioService").settle() quit(0 if failures == 0 else 1) diff --git a/hoppys-adventure-arcade/tests/heroes_villains_logic.gd b/hoppys-adventure-arcade/tests/heroes_villains_logic.gd index 1544d8f0..354dadb6 100644 --- a/hoppys-adventure-arcade/tests/heroes_villains_logic.gd +++ b/hoppys-adventure-arcade/tests/heroes_villains_logic.gd @@ -427,7 +427,6 @@ func run() -> void: game.victory._input(advance_key()) check(game.leaving,"Dedicated advance presses Return once the count has landed") game.queue_free();await process_frame - root.get_node("AudioService").shutdown() - await create_timer(.3).timeout + await root.get_node("AudioService").settle() print("Heroes vs. Villains logic failures: ",failures) quit(0 if failures==0 else 1) diff --git a/hoppys-adventure-arcade/tests/heroes_villains_motion.gd b/hoppys-adventure-arcade/tests/heroes_villains_motion.gd index 640f4b96..42f650b1 100644 --- a/hoppys-adventure-arcade/tests/heroes_villains_motion.gd +++ b/hoppys-adventure-arcade/tests/heroes_villains_motion.gd @@ -422,9 +422,9 @@ func run() -> void: print("Heroes vs. Villains motion failures: ",failures) for voice in game.sound_pool: voice.stop() game.music.stop() - root.get_node("AudioService").shutdown() game.queue_free() await process_frame;await process_frame + await root.get_node("AudioService").settle() quit(1 if failures else 0) const ENCOUNTER := "ENCOUNTER 1 / 3 — clear the blockers" diff --git a/hoppys-adventure-arcade/tests/heroes_villains_smoke.gd b/hoppys-adventure-arcade/tests/heroes_villains_smoke.gd index 7074c6ba..c3921a33 100644 --- a/hoppys-adventure-arcade/tests/heroes_villains_smoke.gd +++ b/hoppys-adventure-arcade/tests/heroes_villains_smoke.gd @@ -168,6 +168,6 @@ func run() -> void: if is_instance_valid(current_scene) and current_scene.scene_file_path.ends_with("beatem_menu.tscn") and not root.get_node("AppRouter").busy: break check(current_scene.scene_file_path.ends_with("beatem_menu.tscn") and not paused,"Victory returns to an unpaused mode menu") current_scene.queue_free();await process_frame - root.get_node("AudioService").shutdown();await create_timer(.3).timeout + await root.get_node("AudioService").settle() print("Heroes vs. Villains smoke failures: ",failures) quit(0 if failures==0 else 1) diff --git a/hoppys-adventure-arcade/tests/menu_hoppy_smoke.gd b/hoppys-adventure-arcade/tests/menu_hoppy_smoke.gd index 428d60ad..4144fbb7 100644 --- a/hoppys-adventure-arcade/tests/menu_hoppy_smoke.gd +++ b/hoppys-adventure-arcade/tests/menu_hoppy_smoke.gd @@ -51,6 +51,5 @@ func run() -> void: check(hoppy.runner.process_mode == Node.PROCESS_MODE_DISABLED, "Hidden rig stops processing") check(hoppy.get_child_count() == 3, "Switching reuses portrait nodes") print("MENU HOPPY SMOKE: ", "PASS" if failures == 0 else "FAIL", " (", failures, " failures)") - root.get_node("AudioService").shutdown() - await wait(0.3) + await root.get_node("AudioService").settle() quit(failures) diff --git a/hoppys-adventure-arcade/tests/menu_return_layout.gd b/hoppys-adventure-arcade/tests/menu_return_layout.gd index e23e2685..5ba08927 100644 --- a/hoppys-adventure-arcade/tests/menu_return_layout.gd +++ b/hoppys-adventure-arcade/tests/menu_return_layout.gd @@ -104,6 +104,5 @@ func run() -> void: await settle() await check_menu("retry-platformer-hub") print("MENU RETURN: ", "PASS" if failures == 0 else "FAIL", " (", failures, " failures)") - root.get_node("AudioService").shutdown() - await wait(0.3) + await root.get_node("AudioService").settle() quit(failures) diff --git a/hoppys-adventure-arcade/tests/orbital_assets.gd b/hoppys-adventure-arcade/tests/orbital_assets.gd index e19672e0..2a24bcdb 100644 --- a/hoppys-adventure-arcade/tests/orbital_assets.gd +++ b/hoppys-adventure-arcade/tests/orbital_assets.gd @@ -98,8 +98,7 @@ func run() -> void: await finish() func finish(code: int = 0) -> void: - root.get_node("AudioService").shutdown() - await create_timer(0.3).timeout + await root.get_node("AudioService").settle() quit(code) func plate(key: String, pos: Vector2, size: Vector2, opacity: float = 1.0) -> void: diff --git a/hoppys-adventure-arcade/tests/orbital_ops_smoke.gd b/hoppys-adventure-arcade/tests/orbital_ops_smoke.gd index 56e9fefb..99fb1ec3 100644 --- a/hoppys-adventure-arcade/tests/orbital_ops_smoke.gd +++ b/hoppys-adventure-arcade/tests/orbital_ops_smoke.gd @@ -286,6 +286,5 @@ func shutdown() -> void: paused=false change_scene_to_file("res://scenes/platformer/platformer_menu.tscn") await frames(2) - root.get_node("AudioService").shutdown() - await create_timer(.3).timeout + await root.get_node("AudioService").settle() quit(failures) diff --git a/hoppys-adventure-arcade/tests/perf_probe.gd b/hoppys-adventure-arcade/tests/perf_probe.gd index 73b2b320..27599dc7 100644 --- a/hoppys-adventure-arcade/tests/perf_probe.gd +++ b/hoppys-adventure-arcade/tests/perf_probe.gd @@ -92,6 +92,5 @@ func run() -> void: router.go("hub") await worst(60) report("swap_to_hub") - root.get_node("AudioService").shutdown() - await create_timer(0.2).timeout + await root.get_node("AudioService").settle() quit(0) diff --git a/hoppys-adventure-arcade/tests/platformer_boss.gd b/hoppys-adventure-arcade/tests/platformer_boss.gd index ddfef26d..38634eb5 100644 --- a/hoppys-adventure-arcade/tests/platformer_boss.gd +++ b/hoppys-adventure-arcade/tests/platformer_boss.gd @@ -135,6 +135,5 @@ func run() -> void: print("BOSS TEST: ", "PASS" if failures == 0 else "FAIL", " (", failures, " failures)") change_scene_to_file("res://scenes/platformer/platformer_menu.tscn") await frames(2) - root.get_node("AudioService").shutdown() - await create_timer(0.4).timeout + await root.get_node("AudioService").settle() quit(failures) diff --git a/hoppys-adventure-arcade/tests/platformer_smoke.gd b/hoppys-adventure-arcade/tests/platformer_smoke.gd index d2ed2760..6f897efd 100644 --- a/hoppys-adventure-arcade/tests/platformer_smoke.gd +++ b/hoppys-adventure-arcade/tests/platformer_smoke.gd @@ -182,6 +182,5 @@ func run() -> void: check(not root.get_node("AudioService").music.stream_paused, "Menu music restored") print("PLATFORMER SMOKE: ", "PASS" if failures == 0 else "FAIL", " (", failures, " failures)") await frames(35) - root.get_node("AudioService").shutdown() - await create_timer(0.3).timeout + await root.get_node("AudioService").settle() quit(failures) diff --git a/hoppys-adventure-arcade/tests/platformer_traversal.gd b/hoppys-adventure-arcade/tests/platformer_traversal.gd index 8915c0a2..fefcc02c 100644 --- a/hoppys-adventure-arcade/tests/platformer_traversal.gd +++ b/hoppys-adventure-arcade/tests/platformer_traversal.gd @@ -90,8 +90,7 @@ func run() -> void: child.stream = null level.music.stop() level.music.stream = null - root.get_node("AudioService").shutdown() - await create_timer(0.3).timeout - # Audio cleanup uses wall time even in accelerated --fixed-fps test runs. + await root.get_node("AudioService").settle() + # The level's own player still releases on wall time in accelerated --fixed-fps runs. OS.delay_msec(300) quit(failures) diff --git a/hoppys-adventure-arcade/tests/runner_boss_validator.gd b/hoppys-adventure-arcade/tests/runner_boss_validator.gd index 99c78b91..5f3bca6c 100644 --- a/hoppys-adventure-arcade/tests/runner_boss_validator.gd +++ b/hoppys-adventure-arcade/tests/runner_boss_validator.gd @@ -915,4 +915,8 @@ func _report() -> void: player = null tool_node = null print("RUNNER BOSS: ", "PASS" if failures == 0 else "FAIL", " (", failures, " failures / ", checks, " checks)") - quit(failures) + _quit_quietly(failures) + +func _quit_quietly(code: int) -> void: + await root.get_node("AudioService").settle() + quit(code) diff --git a/hoppys-adventure-arcade/tests/runner_chunk_validator.gd b/hoppys-adventure-arcade/tests/runner_chunk_validator.gd index aa5258da..b42183af 100644 --- a/hoppys-adventure-arcade/tests/runner_chunk_validator.gd +++ b/hoppys-adventure-arcade/tests/runner_chunk_validator.gd @@ -1133,4 +1133,8 @@ func _report() -> void: junk.clear() wall_scene = null print("RUNNER CHUNKS: ", "PASS" if failures == 0 else "FAIL", " (", failures, " failures / ", checks, " checks)") - quit(failures) + _quit_quietly(failures) + +func _quit_quietly(code: int) -> void: + await root.get_node("AudioService").settle() + quit(code) diff --git a/hoppys-adventure-arcade/tests/runner_logic.gd b/hoppys-adventure-arcade/tests/runner_logic.gd index c77281ad..de7af293 100644 --- a/hoppys-adventure-arcade/tests/runner_logic.gd +++ b/hoppys-adventure-arcade/tests/runner_logic.gd @@ -3787,4 +3787,8 @@ func _report() -> void: spawner_script = null player_script = null print("RUNNER LOGIC: ", "PASS" if failures == 0 else "FAIL", " (", failures, " failures / ", checks, " checks)") - quit(failures) + _quit_quietly(failures) + +func _quit_quietly(code: int) -> void: + await root.get_node("AudioService").settle() + quit(code) diff --git a/hoppys-adventure-arcade/tests/runner_visual_smoke.gd b/hoppys-adventure-arcade/tests/runner_visual_smoke.gd index 9726348e..477a59af 100644 --- a/hoppys-adventure-arcade/tests/runner_visual_smoke.gd +++ b/hoppys-adventure-arcade/tests/runner_visual_smoke.gd @@ -336,8 +336,7 @@ func run() -> void: await check_runner_audio() await check_runner_full_loop() await check_runner_menu_reset() - root.get_node("AudioService").shutdown() - await create_timer(0.3).timeout + await root.get_node("AudioService").settle() failures += _logger.errors() print("RUNNER VISUAL SMOKE: ", "PASS" if failures == 0 else "FAIL", " (", failures, " failures)") OS.remove_logger(_logger) diff --git a/hoppys-adventure-arcade/tests/stage_flow.gd b/hoppys-adventure-arcade/tests/stage_flow.gd index b59624cb..50ea84f7 100644 --- a/hoppys-adventure-arcade/tests/stage_flow.gd +++ b/hoppys-adventure-arcade/tests/stage_flow.gd @@ -104,8 +104,7 @@ func run() -> void: await check_armed_overlays() print("Stage flow failures: ", failures) if is_instance_valid(current_scene): current_scene.queue_free() - root.get_node("AudioService").shutdown() - await create_timer(.4).timeout + await root.get_node("AudioService").settle() quit(1 if failures else 0) ## Accumulated 1/60 steps land one ulp under 4.5, so the page turns on the next tick. func cross_intro(page: int) -> void: diff --git a/hoppys-adventure-arcade/tools/blender/bosses/boss_kit.py b/hoppys-adventure-arcade/tools/blender/bosses/boss_kit.py index 03abf461..79147346 100644 --- a/hoppys-adventure-arcade/tools/blender/bosses/boss_kit.py +++ b/hoppys-adventure-arcade/tools/blender/bosses/boss_kit.py @@ -7,6 +7,7 @@ """ import hashlib import json +import math import os from pathlib import Path import random @@ -17,14 +18,14 @@ import tempfile import bpy -from mathutils import Matrix, Vector +from mathutils import Euler, Matrix, Vector HERE = Path(__file__).resolve().parent DEFAULT_ASSET_DIR = HERE.parent.parent.parent / 'Assets' / 'runner' / 'bosses' sys.path.insert(0, str(HERE.parent)) sys.path.insert(0, str(HERE.parent / 'props')) import props_kit as P -from props_kit import Part as Part, linear, xform, G2B, B2G +from props_kit import Part, linear, xform, G2B, B2G from glb_atlas import consolidate_atlases SEED = 73019 diff --git a/hoppys-adventure-arcade/tools/blender/bosses/build_legacy_monolith.py b/hoppys-adventure-arcade/tools/blender/bosses/build_legacy_monolith.py index 4b081dc1..ccf299f8 100644 --- a/hoppys-adventure-arcade/tools/blender/bosses/build_legacy_monolith.py +++ b/hoppys-adventure-arcade/tools/blender/bosses/build_legacy_monolith.py @@ -7,7 +7,7 @@ import sys sys.path.insert(0, str(Path(__file__).resolve().parent)) -from boss_kit import DEFAULT_ASSET_DIR, BossBuilder, Part, xform +from boss_kit import DEFAULT_ASSET_DIR, BossBuilder, Part, xform, require RECIPE_REVISION = 3 RACK_Y = (-0.47, 0.34, 1.32, 2.68) diff --git a/hoppys-adventure-arcade/tools/blender/bosses/build_merge_conflict_hydra.py b/hoppys-adventure-arcade/tools/blender/bosses/build_merge_conflict_hydra.py index deb2702e..350d6728 100644 --- a/hoppys-adventure-arcade/tools/blender/bosses/build_merge_conflict_hydra.py +++ b/hoppys-adventure-arcade/tools/blender/bosses/build_merge_conflict_hydra.py @@ -2,10 +2,11 @@ import argparse import bpy import hashlib +import json import math from pathlib import Path import sys -from mathutils import Vector +from mathutils import Matrix, Vector sys.path.insert(0, str(Path(__file__).resolve().parent)) from boss_kit import DEFAULT_ASSET_DIR, BossBuilder, Part, xform, B2G diff --git a/ruff.toml b/ruff.toml index 72b777e8..3b983d38 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,3 +1,9 @@ # Baseline: pyflakes errors only (undefined names, unused imports). Tighten over time. [lint] select = ["F"] + +[lint.per-file-ignores] +# Each boss GLB records the SHA-256 of these recipe sources (boss_kit.py export), and +# tests/assets/boss_legacy_monolith_check.gd fails when a source no longer matches. Leave their +# unused imports until the next rebake, so a lint fix doesn't drift them from the committed models. +"hoppys-adventure-arcade/tools/blender/bosses/{boss_kit,build_legacy_monolith,build_merge_conflict_hydra}.py" = ["F401"] diff --git a/tools/tests/run_tests.py b/tools/tests/run_tests.py new file mode 100644 index 00000000..a6cd8df8 --- /dev/null +++ b/tools/tests/run_tests.py @@ -0,0 +1,318 @@ +"""Run every Godot test suite headless and fail on anything the engine reports. + +Usage: + python3 tools/tests/run_tests.py # all suites, in parallel + python3 tools/tests/run_tests.py -k runner -j 2 # suites whose name contains "runner" + python3 tools/tests/run_tests.py --list # what would run, and what is skipped + +Every .gd file under hoppys-adventure-arcade/tests/ that extends SceneTree (or MainLoop) is a +suite and runs by default; new tests need no registration. Other scripts there are helpers. +tools/tests/suites.toml is the only way to keep a suite out, and each skip carries a reason. +A suite fails if it exits nonzero, times out, or prints an engine/script error or an exit-time +leak. Errors a suite provokes on purpose are allowed only through its expected_errors patterns, +and every listed pattern must still match. Suites listed under [serial] (timing budgets) run one +at a time after the parallel batch. + +Each suite gets its own HOME, so user:// writes (settings, leaderboards) never touch the +developer's real files or another suite running in parallel. Standard library only. +""" +from __future__ import annotations + +import argparse +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import time +import tomllib +from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import dataclass, field +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +PROJECT = ROOT / "hoppys-adventure-arcade" +TESTS = PROJECT / "tests" +CONFIG = Path(__file__).with_name("suites.toml") +TIMEOUT = 600 +ANSI = re.compile(r"\x1b\[[0-9;]*m") +# Lines that mean something went wrong even when a suite's own checks all passed. +PROBLEM = re.compile( + r"^(?:ERROR:|SCRIPT ERROR|USER ERROR|Parse Error)" + r"|Failed to load script|leaked at exit|still in use at exit" +) + + +@dataclass +class Result: + name: str + seconds: float + exit_code: int | None + problems: list[str] = field(default_factory=list) + log: str = "" + notes: list[str] = field(default_factory=list) + + @property + def passed(self) -> bool: + """A clean exit with nothing in the log that fails a suite.""" + return self.exit_code == 0 and not self.problems + + +# Godot can only run a script that extends SceneTree or MainLoop with --script; anything else +# under tests/ (shared helpers, fixtures) is support code, and running it pops a modal alert. +SUITE_BASE = re.compile(r"^extends\s+(?:SceneTree|MainLoop)\b", re.M) + + +def discover() -> list[str]: + """Suite names are paths under tests/ without .gd, e.g. "runner_logic" or "assets/a1_check".""" + return sorted( + path.relative_to(TESTS).with_suffix("").as_posix() + for path in TESTS.rglob("*.gd") + if SUITE_BASE.search(path.read_text(encoding="utf-8", errors="replace")) + ) + + +@dataclass +class Config: + skip: dict[str, str] + serial: dict[str, str] + expected: dict[str, list[re.Pattern[str]]] + errors: list[str] + + +def load_config(suites: list[str], text: str | None = None) -> Config: + """Read suites.toml (or `text`); every entry must name a suite, and skips and serials need a reason.""" + raw = tomllib.loads(CONFIG.read_text(encoding="utf-8") if text is None else text) + skip: dict[str, str] = raw.get("skip", {}) + serial: dict[str, str] = raw.get("serial", {}) + expected = {name: [re.compile(p) for p in patterns] for name, patterns in raw.get("expected_errors", {}).items()} + errors = [f"{CONFIG.name}: [{section}] names {name!r}, which is not a suite" + for section, names in (("skip", skip), ("serial", serial), ("expected_errors", expected)) + for name in names if name not in suites] + errors += [f"{CONFIG.name}: [{section}] entry {name!r} needs a reason" + for section, table in (("skip", skip), ("serial", serial)) + for name, reason in table.items() if not reason.strip()] + return Config(skip, serial, expected, errors) + + +def isolated_env(home: str) -> dict[str, str]: + """Point every platform's user-data root (user://) at a private directory.""" + return {**os.environ, "HOME": home, "USERPROFILE": home, + "XDG_DATA_HOME": f"{home}/.local/share", "XDG_CONFIG_HOME": f"{home}/.config", + "XDG_CACHE_HOME": f"{home}/.cache", + "APPDATA": str(Path(home, "AppData", "Roaming")), "LOCALAPPDATA": str(Path(home, "AppData", "Local"))} + + +def check_isolation(godot: str) -> str | None: + """Ask Godot where user:// resolves under the isolated environment; None when it is private.""" + home = tempfile.mkdtemp(prefix="arcade-test-probe-") + try: + probe = Path(home, "probe.gd") + probe.write_text('extends SceneTree\nfunc _init() -> void:\n\tprint("USER_DIR=", OS.get_user_data_dir())\n' + '\tquit()\n', encoding="utf-8") + try: + output = subprocess.run([godot, "--headless", "--path", str(PROJECT), "--script", str(probe)], + env=isolated_env(home), capture_output=True, text=True, timeout=120).stdout + except (OSError, subprocess.TimeoutExpired) as error: + return f"the user:// probe could not run ({error})" + match = re.search(r"^USER_DIR=(.+)$", ANSI.sub("", output), re.M) + user_dir = Path(match.group(1).strip()).resolve() if match else None + if user_dir is None or Path(home).resolve() not in user_dir.parents: + return f"user:// resolved to {user_dir or 'nothing'}, outside the isolated home {home}" + return None + finally: + shutil.rmtree(home, ignore_errors=True) + + +LEAK = re.compile(r"leaked at exit|still in use at exit") +# The engine's two summary lines whose --verbose detail lines classify_leaks() reads. Any other exit +# leak report (RID allocations, for one) has no such detail and is never tolerated. +LEAK_SUMMARY = re.compile(r"ObjectDB instances were leaked at exit|resources still in use at exit") +LEAKED_INSTANCE = re.compile(r"^Leaked instance: (\w+):", re.M) +RESOURCE_IN_USE = re.compile(r"^Resource still in use: (.+) \((\w+)\)$", re.M) +# The engine releases audio playbacks on its audio thread. When a suite quits while that thread +# is starved (parallel 3D suites on a small CI runner), the shared music and UI voices are +# reported as leaked even after AudioService.settle(). Only that exact shape is tolerated: +# audio stream classes (AudioStream*, and Ogg Vorbis's internal OggPacketSequence*) and audio +# files still in use. Any other leaked object fails the suite. +AUDIO_CLASS = re.compile(r"^(?:AudioStream\w*|OggPacketSequence\w*)$") +VERBOSE_NOISE = re.compile(r"^(?:Loading resource:|Completed load for:|Loaded builtin|Loading | ->|ResourceFormat)") + + +def classify_leaks(log: str) -> tuple[bool, list[str]]: + """(is the audio-teardown race, detail lines naming what leaked) from a --verbose log.""" + instances = LEAKED_INSTANCE.findall(log) + resources = RESOURCE_IN_USE.findall(log) + unclassified = [line.strip() for line in log.splitlines() if LEAK.search(line) and not LEAK_SUMMARY.search(line)] + details = [f"leaked: {name}" for name in instances] + [f"still in use: {path} ({kind})" for path, kind in resources] + audio_only = bool(details) and not unclassified and all(AUDIO_CLASS.match(name) for name in instances) \ + and all(AUDIO_CLASS.match(kind) for _, kind in resources) + return audio_only, details + unclassified + + +def run_suite(godot: str, name: str, expected: list[re.Pattern[str]]) -> Result: + """Run one suite and classify any exit leak: only the audio-teardown race becomes a note.""" + result = run_once(godot, name, expected) + if any(LEAK.search(problem) for problem in result.problems): + audio_only, details = classify_leaks(result.log) + if audio_only: + result.problems = [problem for problem in result.problems if not LEAK_SUMMARY.search(problem)] + result.notes.append("engine audio-teardown race tolerated: " + ", ".join(details)) + else: + result.problems += details[:20] or ["leak with no --verbose detail"] + result.log = "\n".join(line for line in result.log.splitlines() if not VERBOSE_NOISE.match(line)) + return result + + +def run_once(godot: str, name: str, expected: list[re.Pattern[str]]) -> Result: + """One headless, --verbose run of res://tests/.gd with a private user-data root.""" + home = tempfile.mkdtemp(prefix="arcade-test-home-") + env = isolated_env(home) + # --verbose makes Godot name what leaked in the same run, so a leak can be classified. + command = [godot, "--headless", "--verbose", "--path", str(PROJECT), "--script", f"res://tests/{name}.gd"] + started = time.monotonic() + try: + completed = subprocess.run(command, env=env, capture_output=True, text=True, timeout=TIMEOUT) + exit_code, log = completed.returncode, ANSI.sub("", completed.stdout + completed.stderr) + except subprocess.TimeoutExpired as timeout: + output = timeout.stdout or b"" + exit_code, log = None, ANSI.sub("", output.decode() if isinstance(output, bytes) else output) + finally: + shutil.rmtree(home, ignore_errors=True) + return Result(name, time.monotonic() - started, exit_code, find_problems(log, exit_code, expected), log) + + +def find_problems(log: str, exit_code: int | None, expected: list[re.Pattern[str]]) -> list[str]: + """Everything that fails a suite besides its exit code: engine errors, leaks, stale expectations.""" + problems = [] if exit_code is not None else [f"timed out after {TIMEOUT}s"] + lines = log.splitlines() + problems += [line.strip() for line in lines + if PROBLEM.search(line) and not any(pattern.search(line) for pattern in expected)] + problems += [f"expected error no longer occurs; remove it from {CONFIG.name}: {pattern.pattern}" + for pattern in expected if not any(pattern.search(line) for line in lines)] + return problems + + +def annotate(result: Result) -> None: + """Report a failed suite as a GitHub Actions error annotation (no-op outside Actions).""" + if os.environ.get("GITHUB_ACTIONS") != "true": + return + detail = "%0A".join(dict.fromkeys(result.problems))[:4000] or f"exit {result.exit_code}" + print(f"::error title=tests/{result.name}.gd::{detail}") + + +def write_summary(path: Path, seconds: float, skipped: dict[str, str], results: list[Result], + setup_error: str | None = None) -> None: + """Write the JSON results summary. A setup failure (isolation, import) is recorded with no suites.""" + summary: dict[str, object] = { + "seconds": round(seconds, 1), + "skipped": skipped, + "suites": [{"name": r.name, "passed": r.passed, "exit_code": r.exit_code, "seconds": round(r.seconds, 1), + "problems": list(dict.fromkeys(r.problems)), "notes": r.notes} + for r in sorted(results, key=lambda r: r.name)], + } + if setup_error is not None: + summary["setup_error"] = setup_error + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8") + + +def main() -> int: + """Discover, filter and run the suites; returns 0 if all pass, 1 on a failure, 2 on a usage or setup error.""" + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--godot", default=os.environ.get("GODOT", "godot"), + help="Godot 4.7.1 binary (default: $GODOT or godot)") + parser.add_argument("-k", dest="match", action="append", default=[], + help="run suites whose name contains this text") + parser.add_argument("-j", "--jobs", type=int, default=min(4, os.cpu_count() or 1), help="suites to run at once") + parser.add_argument("--no-import", action="store_true", help="skip the resource import step") + parser.add_argument("--list", action="store_true", help="print the plan and exit") + parser.add_argument("--results", type=Path, default=ROOT / "build/test-results/summary.json") + args = parser.parse_args() + + suites = discover() + config = load_config(suites) + if config.errors: + print("\n".join(config.errors), file=sys.stderr) + return 2 + skips, expected = config.skip, config.expected + selected = [s for s in suites if not args.match or any(m in s for m in args.match)] + planned = [s for s in selected if s not in skips] + if args.list: + for suite in selected: + if suite in skips: + print(f"skip {suite}: {skips[suite]}") + elif suite in config.serial: + print(f"serial {suite}: {config.serial[suite]}") + else: + print(f"run {suite}") + return 0 + if not planned: + print("No suites selected.", file=sys.stderr) + return 2 + skipped = {name: skips[name] for name in selected if name in skips} + + leak = check_isolation(args.godot) + if leak: + print(f"Refusing to run: {leak}. Suites must not touch real settings or leaderboards.", file=sys.stderr) + write_summary(args.results, 0.0, skipped, [], f"user:// isolation: {leak}") + return 2 + + if not args.no_import: + home = tempfile.mkdtemp(prefix="arcade-test-import-") + try: + imported = subprocess.run([args.godot, "--headless", "--path", str(PROJECT), "--import"], + env=isolated_env(home), capture_output=True, text=True, timeout=TIMEOUT) + except (OSError, subprocess.TimeoutExpired) as error: + print(f"Resource import failed: {error}", file=sys.stderr) + write_summary(args.results, 0.0, skipped, [], f"resource import failed: {error}") + return 1 + finally: + shutil.rmtree(home, ignore_errors=True) + import_log = ANSI.sub("", imported.stdout + imported.stderr) + import_problems = find_problems(import_log, imported.returncode, []) + if imported.returncode != 0 or import_problems: + print(import_log[-4000:], file=sys.stderr) + print("Resource import failed:", *dict.fromkeys(import_problems), sep="\n ", file=sys.stderr) + reasons = list(dict.fromkeys(import_problems)) or [f"exit {imported.returncode}"] + write_summary(args.results, 0.0, skipped, [], "resource import failed: " + "; ".join(reasons)) + return 1 + + parallel = [s for s in planned if s not in config.serial] + serial = [s for s in planned if s in config.serial] + print(f"Running {len(planned)} suites ({len(selected) - len(planned)} skipped): " + f"{len(parallel)} at {args.jobs} a time, then {len(serial)} alone.") + started = time.monotonic() + results: list[Result] = [] + + def record(result: Result) -> None: + results.append(result) + note = f" [{result.notes[0][:120]}]" if result.notes else "" + print(f"{'PASS' if result.passed else 'FAIL'} {result.name} ({result.seconds:.0f}s){note}", flush=True) + + with ThreadPoolExecutor(max_workers=max(1, args.jobs)) as pool: + futures = [pool.submit(run_suite, args.godot, s, expected.get(s, [])) for s in parallel] + for future in as_completed(futures): + record(future.result()) + # Suites with timing budgets run on an otherwise idle machine. + for suite in serial: + record(run_suite(args.godot, suite, expected.get(suite, []))) + + failed = sorted((r for r in results if not r.passed), key=lambda r: r.name) + for result in failed: + print(f"\n--- {result.name}: exit {result.exit_code}") + for problem in dict.fromkeys(result.problems): + print(f" {problem}") + print(" last output:\n" + "\n".join(f" {line}" for line in result.log.splitlines()[-25:])) + annotate(result) + + write_summary(args.results, time.monotonic() - started, skipped, results) + print(f"\n{len(results) - len(failed)} passed, {len(failed)} failed, " + f"{len(selected) - len(planned)} skipped in {time.monotonic() - started:.0f}s") + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/tests/suites.toml b/tools/tests/suites.toml new file mode 100644 index 00000000..5f3f47e9 --- /dev/null +++ b/tools/tests/suites.toml @@ -0,0 +1,27 @@ +# Configuration for tools/tests/run_tests.py. +# +# Every suite (a script under hoppys-adventure-arcade/tests/ that extends SceneTree or MainLoop) +# runs headless in CI by default. +# A [skip] entry is the only way to keep a suite out, and it must say why. + +[skip] +capture_runner_menu_previews = "Writes menu preview screenshots; needs a window and GPU. Run it locally without --headless." +perf_probe = "Measures GPU frame times, which are meaningless headless. Run it locally." +runner_perf_probe = "Measures runner frame times on a GPU. Run it locally." +runner_pickup_hitch = "Measures GPU hitches during pickups. Run it locally." +menu_smoke = "Drives the menu with pointer hover, which a headless run never produces." +menu_hoppy_smoke = "Drives the menu with pointer hover, which a headless run never produces." +capture_code_city_bosses = "Writes the Code City boss stills into docs/previews/; a graphical run only." +exit_leak = "Needs a --scenario per process; CI runs every scenario through tools/checks/exit_leak.py." + +# Suites with wall-clock budgets, or that occupy every core, run alone after the parallel +# batch, so CPU contention from other suites can't fail them (or them, the others). +[serial] +"assets/prewarm_check" = "Asserts the runner's synchronous cold warm finishes under 1000 ms." +"assets/a2_check" = "Waits at most 15 s for the asynchronous prewarm, which the parallel batch starves on a 2-core runner." +runner_synth_threads = "Parks a thread on every core and releases them together; it needs idle cores and starves other suites." + +# Engine errors a suite triggers on purpose. Each pattern must match at least one line of +# that suite's output, so the list can't outlive the behavior it describes. +[expected_errors] +runner_logic = ['^ERROR: ConfigFile parse error at user://test_leaderboard_'] diff --git a/tools/tests/test_run_tests.py b/tools/tests/test_run_tests.py new file mode 100644 index 00000000..62628f85 --- /dev/null +++ b/tools/tests/test_run_tests.py @@ -0,0 +1,166 @@ +"""Unit tests for run_tests.py's discovery, configuration checks and failure detection.""" +import contextlib +import io +import json +import re +import subprocess +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +import run_tests + + +class FindProblems(unittest.TestCase): + def test_clean_log_has_no_problems(self): + self.assertEqual(run_tests.find_problems("PASS: thing\nAll good\n", 0, []), []) + + def test_engine_and_script_errors_fail(self): + log = "SCRIPT ERROR: Invalid call\nERROR: Condition failed\nParse Error: bad\n" + self.assertEqual(len(run_tests.find_problems(log, 0, [])), 3) + + def test_exit_leaks_fail(self): + log = ("WARNING: 3 ObjectDB instances were leaked at exit (run with `--verbose` for details).\n" + "ERROR: 1 resources still in use at exit (run with --verbose for details).\n") + self.assertEqual(len(run_tests.find_problems(log, 0, [])), 2) + + def test_ordinary_warnings_and_file_names_pass(self): + log = "WARNING: leaderboard schema newer\nStoring File: sfx_sounds_error1.wav\n" + self.assertEqual(run_tests.find_problems(log, 0, []), []) + + def test_expected_errors_are_allowed_but_must_still_occur(self): + expected = [re.compile(r"^ERROR: ConfigFile parse error")] + self.assertEqual(run_tests.find_problems("ERROR: ConfigFile parse error at x\n", 0, expected), []) + stale = run_tests.find_problems("all fine\n", 0, expected) + self.assertEqual(len(stale), 1) + self.assertIn("no longer occurs", stale[0]) + + def test_timeout_is_a_problem(self): + self.assertIn("timed out", run_tests.find_problems("", None, [])[0]) + + +class ClassifyLeaks(unittest.TestCase): + AUDIO = ("Leaked instance: AudioStreamPlaybackMP3:1 - Reference count: 1\n" + "Leaked instance: AudioStreamPlaybackPolyphonic:2 - Reference count: 1\n" + "Resource still in use: res://Assets/Audio/Main Menu/track.mp3 (AudioStreamMP3)\n") + + def test_ogg_internals_count_as_audio(self): + log = ("Leaked instance: OggPacketSequence:1 - Reference count: 3\n" + "Leaked instance: OggPacketSequencePlayback:2 - Reference count: 1\n" + "Leaked instance: AudioStreamOggVorbis:3 - Reference count: 1\n") + self.assertTrue(run_tests.classify_leaks(log)[0]) + + def test_audio_teardown_race_is_recognized(self): + audio_only, details = run_tests.classify_leaks(self.AUDIO) + self.assertTrue(audio_only) + self.assertEqual(len(details), 3) + + def test_any_other_leak_is_not_tolerated(self): + audio_only, details = run_tests.classify_leaks(self.AUDIO + "Leaked instance: Node3D:9 - Node path:\n") + self.assertFalse(audio_only) + self.assertIn("leaked: Node3D", details) + texture = "Resource still in use: res://a.png (CompressedTexture2D)\n" + self.assertFalse(run_tests.classify_leaks(self.AUDIO + texture)[0]) + + def test_a_leak_without_detail_is_not_tolerated(self): + self.assertEqual(run_tests.classify_leaks("WARNING: 3 ObjectDB instances were leaked at exit\n"), (False, [])) + + def test_an_rid_leak_beside_the_audio_race_is_not_tolerated(self): + rid = "ERROR: 2 RID allocations of type 'N12MeshStorage4MeshE' were leaked at exit." + audio_only, details = run_tests.classify_leaks(self.AUDIO + rid + "\n") + self.assertFalse(audio_only) + self.assertIn(rid, details) + + +class RunSuite(unittest.TestCase): + """run_suite() turns only the audio-teardown race into a note; every other leak still fails.""" + SUMMARY = ("WARNING: 3 ObjectDB instances were leaked at exit (run with `--verbose` for details).\n" + "ERROR: 1 resources still in use at exit (run with --verbose for details).\n") + RID = "ERROR: 2 RID allocations of type 'N12MeshStorage4MeshE' were leaked at exit." + + def run_with_log(self, log): + result = run_tests.Result("fake", 1.0, 0, run_tests.find_problems(log, 0, []), log) + with mock.patch.object(run_tests, "run_once", return_value=result): + return run_tests.run_suite("godot", "fake", []) + + def test_the_audio_race_alone_passes_with_a_note(self): + result = self.run_with_log(self.SUMMARY + ClassifyLeaks.AUDIO) + self.assertTrue(result.passed, result.problems) + self.assertIn("audio-teardown race", result.notes[0]) + + def test_an_rid_leak_mixed_with_the_audio_race_still_fails(self): + result = self.run_with_log(self.SUMMARY + self.RID + "\n" + ClassifyLeaks.AUDIO) + self.assertFalse(result.passed) + self.assertIn(self.RID, result.problems) + self.assertEqual(result.notes, []) + + +class SetupFailures(unittest.TestCase): + """A run that stops before any suite still writes a results summary saying why.""" + + def run_main(self, results, **patches): + argv = ["run_tests.py", "--godot", "godot", "-k", "runner_logic", "--results", str(results)] + with mock.patch("sys.argv", argv), contextlib.redirect_stderr(io.StringIO()), \ + contextlib.ExitStack() as stack: + for name, value in patches.items(): + stack.enter_context(mock.patch.object(run_tests, name, value)) + return run_tests.main() + + def test_an_isolation_failure_writes_the_summary(self): + with tempfile.TemporaryDirectory() as tmp: + results = Path(tmp, "summary.json") + code = self.run_main(results, check_isolation=mock.Mock(return_value="user:// resolved to /real")) + self.assertEqual(code, 2) + summary = json.loads(results.read_text(encoding="utf-8")) + self.assertIn("/real", summary["setup_error"]) + self.assertEqual(summary["suites"], []) + + def test_a_probe_that_cannot_start_is_an_isolation_failure(self): + with mock.patch.object(run_tests.subprocess, "run", side_effect=FileNotFoundError("no godot")): + self.assertIn("no godot", run_tests.check_isolation("missing-godot")) + timeout = subprocess.TimeoutExpired(["godot"], 120) + with mock.patch.object(run_tests.subprocess, "run", side_effect=timeout): + self.assertIn("timed out", run_tests.check_isolation("godot")) + + def test_an_import_timeout_writes_the_summary(self): + timeout = subprocess.TimeoutExpired(["godot", "--import"], run_tests.TIMEOUT) + with tempfile.TemporaryDirectory() as tmp, \ + mock.patch.object(run_tests.subprocess, "run", side_effect=timeout): + results = Path(tmp, "summary.json") + code = self.run_main(results, check_isolation=mock.Mock(return_value=None)) + self.assertEqual(code, 1) + summary = json.loads(results.read_text(encoding="utf-8")) + self.assertIn("timed out", summary["setup_error"]) + + def test_an_import_failure_writes_the_summary(self): + imported = subprocess.CompletedProcess([], 0, stdout="ERROR: Cannot import x.png\n", stderr="") + with tempfile.TemporaryDirectory() as tmp, \ + mock.patch.object(run_tests.subprocess, "run", return_value=imported): + results = Path(tmp, "summary.json") + code = self.run_main(results, check_isolation=mock.Mock(return_value=None)) + self.assertEqual(code, 1) + summary = json.loads(results.read_text(encoding="utf-8")) + self.assertIn("Cannot import x.png", summary["setup_error"]) + + +class LoadConfig(unittest.TestCase): + def test_unknown_suites_and_missing_reasons_are_errors(self): + text = ('[skip]\nghost = "gone"\nreal = " "\n[serial]\nspecter = "timing"\nreal = ""\n' + '[expected_errors]\nphantom = ["x"]\n') + self.assertEqual(len(run_tests.load_config(["real"], text).errors), 5) + + def test_committed_config_matches_the_suites_on_disk(self): + self.assertEqual(run_tests.load_config(run_tests.discover()).errors, []) + + +class Discover(unittest.TestCase): + def test_finds_top_level_and_nested_suites_but_not_helpers(self): + suites = run_tests.discover() + self.assertIn("runner_logic", suites) + self.assertIn("assets/a1_check", suites) + self.assertNotIn("support/armed_measure", suites) # extends RefCounted: a helper, not a suite + + +if __name__ == "__main__": + unittest.main()