test: run every suite in CI, fail on engine errors, and settle audio deterministically

.github/workflows/ci.yml

@@ -44,74 +44,27 @@ jobs:
           unzip -q /tmp/godot.zip -d ~/godot
           mv ~/godot/Godot_v${GODOT_VERSION}_linux.x86_64 ~/godot/godot
 
-      - name: Import project
-        run: ~/godot/godot --headless --path "$PROJECT" --import
+      - name: Cache imported resources
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ${{ env.PROJECT }}/.godot
+          key: godot-import-${{ env.GODOT_VERSION }}-${{ hashFiles('hoppys-adventure-arcade/project.godot', 'hoppys-adventure-arcade/**/*.import') }}
+          # A stale cache is safe: Godot reimports any source whose hash changed.
+          restore-keys: godot-import-${{ env.GODOT_VERSION }}-
 
-      # Headless suites only. Not listed here:
-      # - capture_runner_menu_previews, perf_probe, runner_perf_probe, runner_pickup_hitch
-      #   need a real window/GPU and are run locally (see CONTRIBUTING.md).
-      # - menu_smoke and menu_hoppy_smoke rely on pointer hover, which doesn't happen headless
-      #   (2 failures each, also on main).
-      - name: Run headless suites
-        run: |
-          suites=(
-            code_city_assets
-            code_city_celebration
-            code_city_logic
-            code_city_mini_boss
-            code_city_motion
-            code_city_smoke
-            controller_service
-            stage_flow
-            beatemup_knockout
-            beatemup_story
-            beatemup_minimap
-            heroes_villains_assets
-            heroes_villains_logic
-            heroes_villains_motion
-            heroes_villains_smoke
-            hoppy_armed_assets
-            enchanted_assets
-            enchanted_boundaries
-            enchanted_repo_smoke
-            menu_return_layout
-            orbital_assets
-            orbital_ops_smoke
-            platformer_boss
-            platformer_knockout
-            platformer_smoke
-            platformer_traversal
-            runner_boss_validator
-            runner_chunk_validator
-            runner_logic
-            runner_run_intro
-            runner_run_intro_ui
-            runner_scaffold_check
-            runner_smoke
-            runner_synth_threads
-            runner_visual_smoke
-            runner_warmup_check
-          )
-          failed=()
-          for suite in "${suites[@]}"; do
-            echo "::group::$suite"
-            log="/tmp/$suite.log"
-            set +e
-            timeout 600 ~/godot/godot --headless --path "$PROJECT" \
-              --script "res://tests/$suite.gd" 2>&1 | tee "$log"
-            rc=${PIPESTATUS[0]}
-            set -e
-            echo "::endgroup::"
-            # A script that fails to parse can still exit 0, so treat SCRIPT ERROR as a failure too.
-            if [ "$rc" -ne 0 ] || grep -q "SCRIPT ERROR" "$log"; then
-              echo "::error title=$suite::failed (exit $rc)"
-              failed+=("$suite")
-            fi
-          done
-          if [ ${#failed[@]} -gt 0 ]; then
-            echo "Failed suites: ${failed[*]}"
-            exit 1
-          fi
+      # The runner imports first (failing on any engine error), then runs every tests/**/*.gd;
+      # tools/tests/suites.toml lists the few skipped or serial suites and why.
+      - name: Import and run headless suites
+        run: python3 tools/tests/run_tests.py --godot ~/godot/godot --jobs 4
+
+      - name: Upload test results
+        if: always()
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: test-results
+          path: build/test-results/
+          retention-days: 7
+          if-no-files-found: ignore
 
       # Quits from the hub, fights, the Code City celebration, the runner intro and mid-transition
       # with their audio playing. Godot prints its exit-leak lines only after the script is gone,
@@ -133,6 +86,8 @@ jobs:
       - run: pip install ruff gdtoolkit==4.*
       - name: Ruff (tools)
         run: ruff check tools hoppys-adventure-arcade/tools
+      - name: Test runner unit tests
+        run: python -m unittest discover -s tools/tests -v
       # Advisory until the existing code is formatted; make it blocking afterwards.
       - name: gdlint (advisory)
         continue-on-error: true

.gitignore

@@ -24,5 +24,5 @@ __pycache__/
 .vscode/
 .DS_Store
 
-# Exported game builds (tools/build/export_game.py)
+# Build outputs: exported games (tools/build) and test results (tools/tests)
 /build/

CONTRIBUTING.md

@@ -29,14 +29,26 @@ Thanks for helping out! This guide covers setup, workflow, testing and the asset
 
 ## Tests
 
-The tests are standalone `SceneTree` scripts. Each one exits nonzero on failure:
+The tests are standalone `SceneTree` scripts under `hoppys-adventure-arcade/tests/`. Run them all the way CI does:
+
+```sh
+python3 tools/tests/run_tests.py --godot /path/to/Godot      # or set $GODOT
+python3 tools/tests/run_tests.py -k runner                   # suites whose name contains "runner"
+python3 tools/tests/run_tests.py --list                      # what runs and what is skipped
+```
+
+- **Every new `.gd` file under `tests/` that `extends SceneTree` or `MainLoop` runs automatically.** Shared helpers that extend anything else, such as `tests/support/`, are never run as suites. There's no list to update. To keep a suite out of CI (it needs a GPU, a window or pointer hover), add it to `[skip]` in [`tools/tests/suites.toml`](tools/tests/suites.toml) with the reason.
+- **A suite fails if it exits nonzero, times out, or prints an engine or script error or an exit-time leak.** Suites run with `--verbose`, so a leak names the objects involved. The one tolerated shape is the engine's audio-teardown race: only `AudioStream*` playbacks and audio files, reported as a note. Anything else that leaks fails. That includes `ERROR:`, `SCRIPT ERROR`, parse errors and "leaked at exit". If a test provokes an engine error on purpose, list a pattern for it under `[expected_errors]`; the runner fails if that pattern stops matching.
+- **Finish a suite with `await root.get_node("AudioService").settle()` before `quit()`.** It stops the shared music and waits until the audio thread has released it. A fixed delay is not enough under parallel load.
+- Each suite gets a private user-data root (`HOME`, the XDG directories, and `APPDATA` on Windows), so `user://` files (settings, leaderboards) never touch your real ones. The runner checks this with Godot before starting and refuses to run if `user://` would escape. Suites listed under `[serial]` (wall-clock budgets, or suites that occupy every core) run alone after the parallel batch. Results are written to `build/test-results/summary.json`.
+
+To run a single suite directly:
 
 ```sh
 godot --headless --path hoppys-adventure-arcade --script res://tests/runner_logic.gd
-godot --headless --path hoppys-adventure-arcade --script res://tests/platformer_smoke.gd
 ```
 
-CI runs every headless suite listed in `.github/workflows/ci.yml`. It also runs `python3 tools/checks/exit_leak.py --godot <godot>`, which quits from screens with audio playing and fails on Godot's "leaked at exit" lines. Add a focused regression test when you change behavior. Some suites capture screenshots when run without `--headless`; include those in the PR for visual changes. On macOS, pass `--always-on-top` for graphical runs so the capture doesn't stall when the window is covered.
+CI also runs `python3 tools/checks/exit_leak.py --godot <godot>`, which quits from screens with audio playing and fails on Godot's "leaked at exit" lines. Add a focused regression test when you change behavior. Some suites capture screenshots when run without `--headless`; include those in the PR for visual changes. On macOS, pass `--always-on-top` for graphical runs so the capture doesn't stall when the window is covered.
 
 Per-mode notes and test details live in [`docs/`](docs/).
 

README.md

@@ -43,11 +43,10 @@ Versioned macOS, Windows, Linux and Web builds come from the [Build workflow](.g
 ## Tests
 
 ```sh
-godot --headless --path hoppys-adventure-arcade --import
-godot --headless --path hoppys-adventure-arcade --script res://tests/runner_logic.gd
+python3 tools/tests/run_tests.py --godot /path/to/Godot
 ```
 
-Each file in `hoppys-adventure-arcade/tests/` is a standalone script that exits nonzero on failure. CI runs the headless suites on every PR. See [CONTRIBUTING.md](CONTRIBUTING.md) for the full workflow.
+CI discovers the suites in `hoppys-adventure-arcade/tests/`: every script that extends `SceneTree` or `MainLoop`. Helpers such as `tests/support/` are not suites. It runs all of them on every PR, except the few listed with a reason in [`tools/tests/suites.toml`](tools/tests/suites.toml). A suite fails on a nonzero exit, an engine or script error, or a leak. See [CONTRIBUTING.md](CONTRIBUTING.md#tests) for details.
 
 ## Contributing
 

hoppys-adventure-arcade/Assets/runner/hoppy/hoppy_accessory_albedo.png.import

@@ -3,10 +3,9 @@
 importer="texture"
 type="CompressedTexture2D"
 uid="uid://0smaw6gfvyj5"
-path.s3tc="res://.godot/imported/hoppy_accessory_albedo.png-c122b57ea96ce5ad67ded19ac527ca90.s3tc.ctex"
+path="res://.godot/imported/hoppy_accessory_albedo.png-c122b57ea96ce5ad67ded19ac527ca90.ctex"
 metadata={
-"imported_formats": ["s3tc_bptc"],
-"vram_texture": true
+"vram_texture": false
 }
 generator_parameters={
 "md5": "12ea3843bd9e25b4241e0d5e33b33f0e"
@@ -15,11 +14,11 @@ generator_parameters={
 [deps]
 
 source_file="res://Assets/runner/hoppy/hoppy_accessory_albedo.png"
-dest_files=["res://.godot/imported/hoppy_accessory_albedo.png-c122b57ea96ce5ad67ded19ac527ca90.s3tc.ctex"]
+dest_files=["res://.godot/imported/hoppy_accessory_albedo.png-c122b57ea96ce5ad67ded19ac527ca90.ctex"]
 
 [params]
 
-compress/mode=2
+compress/mode=0
 compress/high_quality=false
 compress/lossy_quality=0.7
 compress/uastc_level=0

hoppys-adventure-arcade/Assets/runner/hoppy/hoppy_accessory_emissive.png.import

@@ -3,10 +3,9 @@
 importer="texture"
 type="CompressedTexture2D"
 uid="uid://dk6fxo7twe0us"
-path.s3tc="res://.godot/imported/hoppy_accessory_emissive.png-12050a52c862b1593e1bc121f8f122fa.s3tc.ctex"
+path="res://.godot/imported/hoppy_accessory_emissive.png-12050a52c862b1593e1bc121f8f122fa.ctex"
 metadata={
-"imported_formats": ["s3tc_bptc"],
-"vram_texture": true
+"vram_texture": false
 }
 generator_parameters={
 "md5": "ee72f210ceb66159b7d3d224a51ebf7a"
@@ -15,11 +14,11 @@ generator_parameters={
 [deps]
 
 source_file="res://Assets/runner/hoppy/hoppy_accessory_emissive.png"
-dest_files=["res://.godot/imported/hoppy_accessory_emissive.png-12050a52c862b1593e1bc121f8f122fa.s3tc.ctex"]
+dest_files=["res://.godot/imported/hoppy_accessory_emissive.png-12050a52c862b1593e1bc121f8f122fa.ctex"]
 
 [params]
 
-compress/mode=2
+compress/mode=0
 compress/high_quality=false
 compress/lossy_quality=0.7
 compress/uastc_level=0

hoppys-adventure-arcade/Assets/runner/hoppy/hoppy_accessory_normal.png.import

@@ -3,10 +3,9 @@
 importer="texture"
 type="CompressedTexture2D"
 uid="uid://b27dm5dn5qhlj"
-path.s3tc="res://.godot/imported/hoppy_accessory_normal.png-8498ac34154d1c36039fcb9428c418ee.s3tc.ctex"
+path="res://.godot/imported/hoppy_accessory_normal.png-8498ac34154d1c36039fcb9428c418ee.ctex"
 metadata={
-"imported_formats": ["s3tc_bptc"],
-"vram_texture": true
+"vram_texture": false
 }
 generator_parameters={
 "md5": "ca837196a548101c4ce1d30a9644f167"
@@ -15,11 +14,11 @@ generator_parameters={
 [deps]
 
 source_file="res://Assets/runner/hoppy/hoppy_accessory_normal.png"
-dest_files=["res://.godot/imported/hoppy_accessory_normal.png-8498ac34154d1c36039fcb9428c418ee.s3tc.ctex"]
+dest_files=["res://.godot/imported/hoppy_accessory_normal.png-8498ac34154d1c36039fcb9428c418ee.ctex"]
 
 [params]
 
-compress/mode=2
+compress/mode=0
 compress/high_quality=false
 compress/lossy_quality=0.7
 compress/uastc_level=0

hoppys-adventure-arcade/Assets/runner/hoppy/hoppy_accessory_orm.png.import

@@ -3,10 +3,9 @@
 importer="texture"
 type="CompressedTexture2D"
 uid="uid://bq52lv7petqa7"
-path.s3tc="res://.godot/imported/hoppy_accessory_orm.png-a24d2bbfa7539917534e8ce2748bee64.s3tc.ctex"
+path="res://.godot/imported/hoppy_accessory_orm.png-a24d2bbfa7539917534e8ce2748bee64.ctex"
 metadata={
-"imported_formats": ["s3tc_bptc"],
-"vram_texture": true
+"vram_texture": false
 }
 generator_parameters={
 "md5": "16227a4c91bb686a5186ac40be1c526f"
@@ -15,11 +14,11 @@ generator_parameters={
 [deps]
 
 source_file="res://Assets/runner/hoppy/hoppy_accessory_orm.png"
-dest_files=["res://.godot/imported/hoppy_accessory_orm.png-a24d2bbfa7539917534e8ce2748bee64.s3tc.ctex"]
+dest_files=["res://.godot/imported/hoppy_accessory_orm.png-a24d2bbfa7539917534e8ce2748bee64.ctex"]
 
 [params]
 
-compress/mode=2
+compress/mode=0
 compress/high_quality=false
 compress/lossy_quality=0.7
 compress/uastc_level=0

hoppys-adventure-arcade/Assets/runner/hoppy/hoppy_body_albedo.png.import

@@ -3,10 +3,9 @@
 importer="texture"
 type="CompressedTexture2D"
 uid="uid://dk1u3h8nede88"
-path.s3tc="res://.godot/imported/hoppy_body_albedo.png-a4c37408013e37c8de18072982be504d.s3tc.ctex"
+path="res://.godot/imported/hoppy_body_albedo.png-a4c37408013e37c8de18072982be504d.ctex"
 metadata={
-"imported_formats": ["s3tc_bptc"],
-"vram_texture": true
+"vram_texture": false
 }
 generator_parameters={
 "md5": "8d1ce8c41029007e13f41ee644bd020e"
@@ -15,11 +14,11 @@ generator_parameters={
 [deps]
 
 source_file="res://Assets/runner/hoppy/hoppy_body_albedo.png"
-dest_files=["res://.godot/imported/hoppy_body_albedo.png-a4c37408013e37c8de18072982be504d.s3tc.ctex"]
+dest_files=["res://.godot/imported/hoppy_body_albedo.png-a4c37408013e37c8de18072982be504d.ctex"]
 
 [params]
 
-compress/mode=2
+compress/mode=0
 compress/high_quality=false
 compress/lossy_quality=0.7
 compress/uastc_level=0

hoppys-adventure-arcade/Assets/runner/hoppy/hoppy_body_emissive.png.import

@@ -3,10 +3,9 @@
 importer="texture"
 type="CompressedTexture2D"
 uid="uid://cnwbddowongvs"
-path.s3tc="res://.godot/imported/hoppy_body_emissive.png-15b9193f78dd32e62b3bc8d4f2eb5591.s3tc.ctex"
+path="res://.godot/imported/hoppy_body_emissive.png-15b9193f78dd32e62b3bc8d4f2eb5591.ctex"
 metadata={
-"imported_formats": ["s3tc_bptc"],
-"vram_texture": true
+"vram_texture": false
 }
 generator_parameters={
 "md5": "efeebdda98ec1d7fb2ad83d23f0713bf"
@@ -15,11 +14,11 @@ generator_parameters={
 [deps]
 
 source_file="res://Assets/runner/hoppy/hoppy_body_emissive.png"
-dest_files=["res://.godot/imported/hoppy_body_emissive.png-15b9193f78dd32e62b3bc8d4f2eb5591.s3tc.ctex"]
+dest_files=["res://.godot/imported/hoppy_body_emissive.png-15b9193f78dd32e62b3bc8d4f2eb5591.ctex"]
 
 [params]
 
-compress/mode=2
+compress/mode=0
 compress/high_quality=false
 compress/lossy_quality=0.7
 compress/uastc_level=0

hoppys-adventure-arcade/Assets/runner/hoppy/hoppy_body_normal.png.import

@@ -3,10 +3,9 @@
 importer="texture"
 type="CompressedTexture2D"
 uid="uid://d2rcbm37d6nu4"
-path.s3tc="res://.godot/imported/hoppy_body_normal.png-904e19f0aa8db2d232ac2f9e1ac495c8.s3tc.ctex"
+path="res://.godot/imported/hoppy_body_normal.png-904e19f0aa8db2d232ac2f9e1ac495c8.ctex"
 metadata={
-"imported_formats": ["s3tc_bptc"],
-"vram_texture": true
+"vram_texture": false
 }
 generator_parameters={
 "md5": "d338760e3eb0d148b22941e704792983"
@@ -15,11 +14,11 @@ generator_parameters={
 [deps]
 
 source_file="res://Assets/runner/hoppy/hoppy_body_normal.png"
-dest_files=["res://.godot/imported/hoppy_body_normal.png-904e19f0aa8db2d232ac2f9e1ac495c8.s3tc.ctex"]
+dest_files=["res://.godot/imported/hoppy_body_normal.png-904e19f0aa8db2d232ac2f9e1ac495c8.ctex"]
 
 [params]
 
-compress/mode=2
+compress/mode=0
 compress/high_quality=false
 compress/lossy_quality=0.7
 compress/uastc_level=0

hoppys-adventure-arcade/Assets/runner/hoppy/hoppy_body_orm.png.import

@@ -3,10 +3,9 @@
 importer="texture"
 type="CompressedTexture2D"
 uid="uid://bjkoyu6nxsyci"
-path.s3tc="res://.godot/imported/hoppy_body_orm.png-a36dae46ff5d3b78351e386b721c189f.s3tc.ctex"
+path="res://.godot/imported/hoppy_body_orm.png-a36dae46ff5d3b78351e386b721c189f.ctex"
 metadata={
-"imported_formats": ["s3tc_bptc"],
-"vram_texture": true
+"vram_texture": false
 }
 generator_parameters={
 "md5": "f684053ef742dca3be2d514aacf3a5a3"
@@ -15,11 +14,11 @@ generator_parameters={
 [deps]
 
 source_file="res://Assets/runner/hoppy/hoppy_body_orm.png"
-dest_files=["res://.godot/imported/hoppy_body_orm.png-a36dae46ff5d3b78351e386b721c189f.s3tc.ctex"]
+dest_files=["res://.godot/imported/hoppy_body_orm.png-a36dae46ff5d3b78351e386b721c189f.ctex"]
 
 [params]
 
-compress/mode=2
+compress/mode=0
 compress/high_quality=false
 compress/lossy_quality=0.7
 compress/uastc_level=0

hoppys-adventure-arcade/app/audio_service.gd

@@ -96,6 +96,19 @@ func shutdown() -> void:
 	music.stream = null
 	sfx.stream = null
 
+## Stops all audio, then waits until the audio thread has let go of every playback, so the
+## caller can quit without "leaked at exit" reports. A fixed delay races the mixer under load.
+func settle(max_seconds := 2.0) -> void:
+	var held: Array[AudioStreamPlayback] = []
+	for player: AudioStreamPlayer in [music, sfx]:
+		if player.has_stream_playback():
+			held.append(player.get_stream_playback())
+	shutdown()
+	var deadline := Time.get_ticks_msec() + int(max_seconds * 1000.0)
+	while held.any(func(playback: AudioStreamPlayback) -> bool: return playback.get_reference_count() > 1) \
+			and Time.get_ticks_msec() < deadline:
+		await get_tree().create_timer(0.02, true, false, true).timeout
+
 ## The autoload only leaves the tree when the app exits, after the current scene has left it.
 ## A voice the mixer still holds when the engine shuts down is reported as leaked ("ObjectDB
 ## instances were leaked", "resources still in use"), and a stopped voice is let go only by the

hoppys-adventure-arcade/tests/assets/a3_check.gd

@@ -247,6 +247,7 @@ func _run() -> void:
 	check(runtime_errors.count == 0,"no runtime errors during asset checks")
 	preload("res://scenes/runner/hazards/visuals/hazard_art.gd").release_caches()
 	print("ASSET A3: %s (%d checks)" % ["PASS" if failures == 0 else "FAIL",checks])
+	await root.get_node("AudioService").settle()
 	quit(0 if failures == 0 else 1)
 
 func _round_two(visual: Base, title: String) -> void: