fix(ci): run each fork test in its own process

The whole-file batch still crashes in Python 3.9 after the first test passes. Give each existing fork test a fresh interpreter, preserving marker filters and assertions. Both test nodes passed 30 individual runs on Linux with Python 3.9 and 3.12.

Signed-off-by: mday-io <mdaytn@gmail.com>

Author: mday-io

Makefile

@@ -132,13 +132,13 @@ engine-up: engine-clickhouse-up engine-mssql-up engine-mysql-up engine-postgres-
 engine-down: engine-clickhouse-down engine-mssql-down engine-mysql-down engine-postgres-down engine-spark-down engine-trino-down
 
 fast-test:
-	pytest -n auto -m "fast and not cicdonly and not isolated" --junitxml=test-results/junit-fast-test.xml && pytest -m "isolated and not slow and not fork_isolated" && pytest tests/test_forking.py -m "isolated and fork_isolated and not slow" && pytest -m "registry_isolation" && pytest -m "dialect_isolated"
+	pytest -n auto -m "fast and not cicdonly and not isolated" --junitxml=test-results/junit-fast-test.xml && pytest -m "isolated and not slow and not fork_isolated" && pytest tests/test_forking.py::test_parallel_load -m "isolated and fork_isolated and not slow" && pytest tests/test_forking.py::test_parallel_load_multi_repo -m "isolated and fork_isolated and not slow" && pytest -m "registry_isolation" && pytest -m "dialect_isolated"
 
 slow-test:
-	pytest -n auto -m "(fast or slow) and not cicdonly and not isolated" && pytest -m "isolated and not fork_isolated" && pytest tests/test_forking.py -m "isolated and fork_isolated" && pytest -m "registry_isolation" && pytest -m "dialect_isolated"
+	pytest -n auto -m "(fast or slow) and not cicdonly and not isolated" && pytest -m "isolated and not fork_isolated" && pytest tests/test_forking.py::test_parallel_load -m "isolated and fork_isolated" && pytest tests/test_forking.py::test_parallel_load_multi_repo -m "isolated and fork_isolated" && pytest -m "registry_isolation" && pytest -m "dialect_isolated"
 
 cicd-test:
-	pytest -n auto -m "(fast or slow) and not pyspark and not isolated" --junitxml=test-results/junit-cicd.xml && pytest -m "pyspark" && pytest -m "isolated and not pyspark and not fork_isolated" && pytest tests/test_forking.py -m "isolated and not pyspark and fork_isolated" && pytest -m "registry_isolation" && pytest -m "dialect_isolated"
+	pytest -n auto -m "(fast or slow) and not pyspark and not isolated" --junitxml=test-results/junit-cicd.xml && pytest -m "pyspark" && pytest -m "isolated and not pyspark and not fork_isolated" && pytest tests/test_forking.py::test_parallel_load -m "isolated and not pyspark and fork_isolated" && pytest tests/test_forking.py::test_parallel_load_multi_repo -m "isolated and not pyspark and fork_isolated" && pytest -m "registry_isolation" && pytest -m "dialect_isolated"
 
 core-fast-test:
 	pytest -n auto -m "fast and not web and not github and not dbt and not jupyter"

pyproject.toml

@@ -257,7 +257,7 @@ markers = [
     "remote: test that involves interacting with a remote DB",
     "cicdonly: test that only runs on CI/CD",
     "isolated: tests that need to run sequentially usually because they use fork",
-    "fork_isolated: tests that need a fresh pytest process because they use fork",
+    "fork_isolated: tests that need their own fresh pytest process because they use fork",
     "dialect_isolated: tests that need to run separately due to global dialect overrides",
 
     # Test Domain Markers

tests/test_forking.py

@@ -6,8 +6,8 @@
 import concurrent.futures
 
 
-# Run in a fresh pytest process so these fork tests do not inherit process-local state created
-# by earlier test modules.
+# Run each fork test in a fresh pytest process to avoid inheriting native process state from
+# earlier tests.
 pytestmark = [pytest.mark.isolated, pytest.mark.fork_isolated]