From d95fd735396f999669b642def7d24ed9d494186d Mon Sep 17 00:00:00 2001 From: Nico Loesch Date: Mon, 3 Aug 2026 00:06:46 +0000 Subject: [PATCH 1/3] Adapt to new oa-config changes, rename Resource -> Database --- pyproject.toml | 4 ++-- src/orm_loader/config.py | 25 ++++-------------------- tests/conftest.py | 4 ++-- tests/loaders/test_pg_loader.py | 34 ++++++++++++++++----------------- 4 files changed, 25 insertions(+), 42 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a92d203..131aa44 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ authors = [ requires-python = ">=3.12" dependencies = [ "chardet>=5.2.0", - "oa-configurator==0.1.2", + "oa-configurator==0.1.2", # TODO: bump to >=1.0.0 once oa-configurator 1.0 is published (source already migrated) "pandas>=2.3.3", "pyarrow>=23.0.1", "sqlalchemy>=2.0.45", @@ -49,7 +49,7 @@ postgres = [ "psycopg[binary]>=3.2", ] dev = [ - "oa-configurator[postgres]==0.1.2", + "oa-configurator[postgres]==0.1.2", # TODO: bump to >=1.0.0 once oa-configurator 1.0 is published "pytest>=9.0.3", "ty>=0.0.59", "ruff>=0.14.11", diff --git a/src/orm_loader/config.py b/src/orm_loader/config.py index 0c1f3b6..2c559ad 100644 --- a/src/orm_loader/config.py +++ b/src/orm_loader/config.py @@ -4,7 +4,7 @@ from typing import ClassVar -from oa_configurator import DatabaseConfig, PackageConfigBase, ResourceSpec +from oa_configurator import PackageConfigBase, with_test_prefix class OrmLoaderConfig(PackageConfigBase): @@ -13,27 +13,10 @@ class OrmLoaderConfig(PackageConfigBase): orm-loader is connection-agnostic — it accepts SQLAlchemy sessions/engines as parameters and owns no database resources. This class exists to register orm-loader in the oa-configurator ecosystem, provide a canonical - ``configure_logging()`` entry point, and declare the test database resource - used by the integration test suite. + ``configure_logging()`` entry point, and name the test database used by the + integration test suite. """ - TEST_DB: ClassVar[ResourceSpec] = ResourceSpec( - semantic_name="test_orm_db", - display_name="ORM Loader Test Database", - description="PostgreSQL database for running orm-loader integration tests.", - connection_name_hint="pg_test_orm", - is_cdm_database=False, - cdm_schema_default="public", - connection_defaults=DatabaseConfig( - dialect="postgresql+psycopg", - host="localhost", - port=55432, - user="test", - password="test", - database_name="test", - ), - ) - tool_name: ClassVar[str] = "orm_loader" extra_logging_namespaces: ClassVar[tuple[str, ...]] = () - test_resources: ClassVar[tuple[ResourceSpec, ...]] = (TEST_DB,) + TEST_DB: ClassVar[str] = with_test_prefix("orm_db") diff --git a/tests/conftest.py b/tests/conftest.py index a5a9abe..0b812cd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -30,10 +30,10 @@ def session(engine): @pytest.fixture(scope="session") def pg_engine(): - from oa_configurator.pytest_plugin import ensure_test_db_exists, resolve_test_resource + from oa_configurator.pytest_plugin import ensure_test_db_exists, resolve_test_database from orm_loader.config import OrmLoaderConfig - url = resolve_test_resource(OrmLoaderConfig.TEST_DB) + url = resolve_test_database(OrmLoaderConfig.TEST_DB) try: ensure_test_db_exists(url) diff --git a/tests/loaders/test_pg_loader.py b/tests/loaders/test_pg_loader.py index 9877e5e..60742b4 100644 --- a/tests/loaders/test_pg_loader.py +++ b/tests/loaders/test_pg_loader.py @@ -7,7 +7,7 @@ from tests.models import SimpleTable -@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database(OrmLoaderConfig.TEST_DB) def test_copy_into_staging_with_extra_identity_column(pg_session, tmp_path): """COPY must succeed when the staging table has a _rownum identity column.""" csv = tmp_path / "test_table.csv" @@ -30,7 +30,7 @@ def test_copy_into_staging_with_extra_identity_column(pg_session, tmp_path): assert rownums == [1, 2], "_rownum must be auto-populated by IDENTITY sequence" -@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database(OrmLoaderConfig.TEST_DB) def test_copy_and_orm_path_equivalence(pg_session, tmp_path): csv = tmp_path / "test_table.csv" @@ -52,7 +52,7 @@ def test_copy_and_orm_path_equivalence(pg_session, tmp_path): -@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database(OrmLoaderConfig.TEST_DB) def test_postgres_copy_fast_path(pg_session, tmp_path): csv = tmp_path / "test_table.csv" pd.DataFrame([{"id": 1, "name": "alpha"}]).to_csv(csv, index=False) @@ -62,7 +62,7 @@ def test_postgres_copy_fast_path(pg_session, tmp_path): assert inserted == 1 -@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database(OrmLoaderConfig.TEST_DB) def test_postgres_copy_fast_path_is_used(pg_session, tmp_path, monkeypatch): csv = tmp_path / "test_table.csv" pd.DataFrame([{"id": 1, "name": "alpha"}]).to_csv(csv, index=False) @@ -82,7 +82,7 @@ def fake_quick_load_pg(*args, **kwargs): assert called["copy"] is True assert inserted == 1 -@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database(OrmLoaderConfig.TEST_DB) def test_copy_failure_falls_back_to_orm(pg_session, tmp_path, monkeypatch): csv = tmp_path / "test_table.csv" pd.DataFrame([{"id": 1, "name": "alpha"}]).to_csv(csv, index=False) @@ -102,7 +102,7 @@ def broken_copy(*args, **kwargs): assert [(r.id, r.name) for r in rows] == [(1, "alpha")] -@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database(OrmLoaderConfig.TEST_DB) def test_postgres_upsert_does_not_update(pg_session, tmp_path): csv = tmp_path / "test_table.csv" @@ -119,7 +119,7 @@ def test_postgres_upsert_does_not_update(pg_session, tmp_path): assert [(r.id, r.name) for r in rows] == [(1, "alpha")] -@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database(OrmLoaderConfig.TEST_DB) def test_postgres_insert_if_empty(pg_session, tmp_path): csv = tmp_path / "test_table.csv" @@ -146,7 +146,7 @@ def test_postgres_insert_if_empty(pg_session, tmp_path): ] -@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database(OrmLoaderConfig.TEST_DB) def test_postgres_insert_if_empty_raises_on_non_empty_target(pg_session, tmp_path): csv = tmp_path / "test_table.csv" @@ -164,7 +164,7 @@ def test_postgres_insert_if_empty_raises_on_non_empty_target(pg_session, tmp_pat ) -@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database(OrmLoaderConfig.TEST_DB) def test_postgres_copy_large_batch(pg_session, tmp_path): csv = tmp_path / "test_table.csv" @@ -181,7 +181,7 @@ def test_postgres_copy_large_batch(pg_session, tmp_path): assert inserted == 9999 -@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database(OrmLoaderConfig.TEST_DB) def test_staging_schema_matches_target(pg_session, tmp_path): csv = tmp_path / "test_table.csv" pd.DataFrame([{"id": 1, "name": "alpha"}]).to_csv(csv, index=False) @@ -251,7 +251,7 @@ def test_check_line_ending_unknown(caplog): assert "Unable to detect line ending" in caplog.text -@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database(OrmLoaderConfig.TEST_DB) def test_quick_load_pg_basic(pg_session, tmp_path): csv = tmp_path / "test_table.csv" csv.write_text("id,name\n1,alpha\n2,beta\n") @@ -267,7 +267,7 @@ def test_quick_load_pg_basic(pg_session, tmp_path): assert rows == [(1, "alpha"), (2, "beta")] -@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database(OrmLoaderConfig.TEST_DB) def test_quick_load_pg_lowercases_header(pg_session, tmp_path): csv = tmp_path / "test_table.csv" csv.write_text("ID,NAME\n1,alpha\n") @@ -279,7 +279,7 @@ def test_quick_load_pg_lowercases_header(pg_session, tmp_path): assert row == (1, "alpha") -@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database(OrmLoaderConfig.TEST_DB) def test_quick_load_pg_tab_delimiter(pg_session, tmp_path): csv = tmp_path / "test_table.csv" csv.write_text("id\tname\n1\talpha\n2\tbeta\n") @@ -291,7 +291,7 @@ def test_quick_load_pg_tab_delimiter(pg_session, tmp_path): assert rows == [(1, "alpha"), (2, "beta")] -@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database(OrmLoaderConfig.TEST_DB) def test_quick_load_pg_rollback_on_error(pg_session, tmp_path): csv = tmp_path / "test_table.csv" csv.write_text("id,name\n1,alpha\n2,\n") # violates NOT NULL @@ -303,7 +303,7 @@ def test_quick_load_pg_rollback_on_error(pg_session, tmp_path): assert rows == 0 -@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database(OrmLoaderConfig.TEST_DB) def test_quick_load_pg_equivalence_with_orm(pg_session, tmp_path): csv = tmp_path / "test_table.csv" csv.write_text("id,name\n1,alpha\n2,beta\n") @@ -327,7 +327,7 @@ def test_quick_load_pg_equivalence_with_orm(pg_session, tmp_path): assert rows_pg == rows_orm -@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database(OrmLoaderConfig.TEST_DB) def test_quick_load_pg_trailing_blank_lines(pg_session, tmp_path): csv = tmp_path / "test_table.csv" @@ -346,7 +346,7 @@ def test_quick_load_pg_trailing_blank_lines(pg_session, tmp_path): assert total == 2 assert rows == [(1, "alpha"), (2, "beta")] -@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database(OrmLoaderConfig.TEST_DB) def test_copy_fails_with_raw_carriage_returns_but_succeeds_after_normalisation(pg_session, tmp_path): csv = tmp_path / "test_table.csv" From 248e7eafb2495efd0b5fc81d156f8f2f0ae798a9 Mon Sep 17 00:00:00 2001 From: Nico Loesch Date: Mon, 3 Aug 2026 03:44:10 +0000 Subject: [PATCH 2/3] Drop ClassVar to prevent duplication, remove the prefix system --- src/orm_loader/config.py | 15 +++++++------- tests/conftest.py | 2 +- tests/loaders/test_pg_loader.py | 35 ++++++++++++++++----------------- 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/src/orm_loader/config.py b/src/orm_loader/config.py index 2c559ad..10062aa 100644 --- a/src/orm_loader/config.py +++ b/src/orm_loader/config.py @@ -2,21 +2,22 @@ from __future__ import annotations -from typing import ClassVar +from typing import Annotated, ClassVar -from oa_configurator import PackageConfigBase, with_test_prefix +from oa_configurator import DatabaseConfig, PackageConfigBase, RefTo class OrmLoaderConfig(PackageConfigBase): """oa-configurator config class for orm-loader. orm-loader is connection-agnostic — it accepts SQLAlchemy sessions/engines - as parameters and owns no database resources. This class exists to register - orm-loader in the oa-configurator ecosystem, provide a canonical - ``configure_logging()`` entry point, and name the test database used by the - integration test suite. + as parameters and owns no production database resource of its own. This + class exists to register orm-loader in the oa-configurator ecosystem, + provide a canonical ``configure_logging()`` entry point, and declare the + test database used by the integration test suite. """ tool_name: ClassVar[str] = "orm_loader" extra_logging_namespaces: ClassVar[tuple[str, ...]] = () - TEST_DB: ClassVar[str] = with_test_prefix("orm_db") + + test_orm_db: Annotated[str | None, RefTo(DatabaseConfig, is_test=True)] = None diff --git a/tests/conftest.py b/tests/conftest.py index 0b812cd..40a07d4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -33,7 +33,7 @@ def pg_engine(): from oa_configurator.pytest_plugin import ensure_test_db_exists, resolve_test_database from orm_loader.config import OrmLoaderConfig - url = resolve_test_database(OrmLoaderConfig.TEST_DB) + url = resolve_test_database(OrmLoaderConfig, "test_orm_db") try: ensure_test_db_exists(url) diff --git a/tests/loaders/test_pg_loader.py b/tests/loaders/test_pg_loader.py index 60742b4..d75387b 100644 --- a/tests/loaders/test_pg_loader.py +++ b/tests/loaders/test_pg_loader.py @@ -2,12 +2,11 @@ import pandas as pd import pytest from orm_loader.loaders.loading_helpers import infer_encoding, infer_delim, check_line_ending, quick_load_pg -from orm_loader.config import OrmLoaderConfig from tests.models import SimpleTable -@pytest.mark.requires_database(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database("test_orm_db") def test_copy_into_staging_with_extra_identity_column(pg_session, tmp_path): """COPY must succeed when the staging table has a _rownum identity column.""" csv = tmp_path / "test_table.csv" @@ -30,7 +29,7 @@ def test_copy_into_staging_with_extra_identity_column(pg_session, tmp_path): assert rownums == [1, 2], "_rownum must be auto-populated by IDENTITY sequence" -@pytest.mark.requires_database(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database("test_orm_db") def test_copy_and_orm_path_equivalence(pg_session, tmp_path): csv = tmp_path / "test_table.csv" @@ -52,7 +51,7 @@ def test_copy_and_orm_path_equivalence(pg_session, tmp_path): -@pytest.mark.requires_database(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database("test_orm_db") def test_postgres_copy_fast_path(pg_session, tmp_path): csv = tmp_path / "test_table.csv" pd.DataFrame([{"id": 1, "name": "alpha"}]).to_csv(csv, index=False) @@ -62,7 +61,7 @@ def test_postgres_copy_fast_path(pg_session, tmp_path): assert inserted == 1 -@pytest.mark.requires_database(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database("test_orm_db") def test_postgres_copy_fast_path_is_used(pg_session, tmp_path, monkeypatch): csv = tmp_path / "test_table.csv" pd.DataFrame([{"id": 1, "name": "alpha"}]).to_csv(csv, index=False) @@ -82,7 +81,7 @@ def fake_quick_load_pg(*args, **kwargs): assert called["copy"] is True assert inserted == 1 -@pytest.mark.requires_database(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database("test_orm_db") def test_copy_failure_falls_back_to_orm(pg_session, tmp_path, monkeypatch): csv = tmp_path / "test_table.csv" pd.DataFrame([{"id": 1, "name": "alpha"}]).to_csv(csv, index=False) @@ -102,7 +101,7 @@ def broken_copy(*args, **kwargs): assert [(r.id, r.name) for r in rows] == [(1, "alpha")] -@pytest.mark.requires_database(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database("test_orm_db") def test_postgres_upsert_does_not_update(pg_session, tmp_path): csv = tmp_path / "test_table.csv" @@ -119,7 +118,7 @@ def test_postgres_upsert_does_not_update(pg_session, tmp_path): assert [(r.id, r.name) for r in rows] == [(1, "alpha")] -@pytest.mark.requires_database(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database("test_orm_db") def test_postgres_insert_if_empty(pg_session, tmp_path): csv = tmp_path / "test_table.csv" @@ -146,7 +145,7 @@ def test_postgres_insert_if_empty(pg_session, tmp_path): ] -@pytest.mark.requires_database(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database("test_orm_db") def test_postgres_insert_if_empty_raises_on_non_empty_target(pg_session, tmp_path): csv = tmp_path / "test_table.csv" @@ -164,7 +163,7 @@ def test_postgres_insert_if_empty_raises_on_non_empty_target(pg_session, tmp_pat ) -@pytest.mark.requires_database(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database("test_orm_db") def test_postgres_copy_large_batch(pg_session, tmp_path): csv = tmp_path / "test_table.csv" @@ -181,7 +180,7 @@ def test_postgres_copy_large_batch(pg_session, tmp_path): assert inserted == 9999 -@pytest.mark.requires_database(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database("test_orm_db") def test_staging_schema_matches_target(pg_session, tmp_path): csv = tmp_path / "test_table.csv" pd.DataFrame([{"id": 1, "name": "alpha"}]).to_csv(csv, index=False) @@ -251,7 +250,7 @@ def test_check_line_ending_unknown(caplog): assert "Unable to detect line ending" in caplog.text -@pytest.mark.requires_database(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database("test_orm_db") def test_quick_load_pg_basic(pg_session, tmp_path): csv = tmp_path / "test_table.csv" csv.write_text("id,name\n1,alpha\n2,beta\n") @@ -267,7 +266,7 @@ def test_quick_load_pg_basic(pg_session, tmp_path): assert rows == [(1, "alpha"), (2, "beta")] -@pytest.mark.requires_database(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database("test_orm_db") def test_quick_load_pg_lowercases_header(pg_session, tmp_path): csv = tmp_path / "test_table.csv" csv.write_text("ID,NAME\n1,alpha\n") @@ -279,7 +278,7 @@ def test_quick_load_pg_lowercases_header(pg_session, tmp_path): assert row == (1, "alpha") -@pytest.mark.requires_database(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database("test_orm_db") def test_quick_load_pg_tab_delimiter(pg_session, tmp_path): csv = tmp_path / "test_table.csv" csv.write_text("id\tname\n1\talpha\n2\tbeta\n") @@ -291,7 +290,7 @@ def test_quick_load_pg_tab_delimiter(pg_session, tmp_path): assert rows == [(1, "alpha"), (2, "beta")] -@pytest.mark.requires_database(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database("test_orm_db") def test_quick_load_pg_rollback_on_error(pg_session, tmp_path): csv = tmp_path / "test_table.csv" csv.write_text("id,name\n1,alpha\n2,\n") # violates NOT NULL @@ -303,7 +302,7 @@ def test_quick_load_pg_rollback_on_error(pg_session, tmp_path): assert rows == 0 -@pytest.mark.requires_database(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database("test_orm_db") def test_quick_load_pg_equivalence_with_orm(pg_session, tmp_path): csv = tmp_path / "test_table.csv" csv.write_text("id,name\n1,alpha\n2,beta\n") @@ -327,7 +326,7 @@ def test_quick_load_pg_equivalence_with_orm(pg_session, tmp_path): assert rows_pg == rows_orm -@pytest.mark.requires_database(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database("test_orm_db") def test_quick_load_pg_trailing_blank_lines(pg_session, tmp_path): csv = tmp_path / "test_table.csv" @@ -346,7 +345,7 @@ def test_quick_load_pg_trailing_blank_lines(pg_session, tmp_path): assert total == 2 assert rows == [(1, "alpha"), (2, "beta")] -@pytest.mark.requires_database(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database("test_orm_db") def test_copy_fails_with_raw_carriage_returns_but_succeeds_after_normalisation(pg_session, tmp_path): csv = tmp_path / "test_table.csv" From bd55de6d27abc474c824b85dc2b6c06efad613aa Mon Sep 17 00:00:00 2001 From: Nico Loesch Date: Thu, 6 Aug 2026 05:13:38 +0000 Subject: [PATCH 3/3] Push versioning and docstring for config --- pyproject.toml | 4 ++-- src/orm_loader/config.py | 9 +++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 131aa44..5a02ce6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ authors = [ requires-python = ">=3.12" dependencies = [ "chardet>=5.2.0", - "oa-configurator==0.1.2", # TODO: bump to >=1.0.0 once oa-configurator 1.0 is published (source already migrated) + "oa-configurator==0.1.2", # TODO: bump to >=1.0.0,<2.0.0 "pandas>=2.3.3", "pyarrow>=23.0.1", "sqlalchemy>=2.0.45", @@ -49,7 +49,7 @@ postgres = [ "psycopg[binary]>=3.2", ] dev = [ - "oa-configurator[postgres]==0.1.2", # TODO: bump to >=1.0.0 once oa-configurator 1.0 is published + "oa-configurator[postgres]==0.1.2", # TODO: bump to >=1.0.0,<2.0.0 "pytest>=9.0.3", "ty>=0.0.59", "ruff>=0.14.11", diff --git a/src/orm_loader/config.py b/src/orm_loader/config.py index 10062aa..497c719 100644 --- a/src/orm_loader/config.py +++ b/src/orm_loader/config.py @@ -4,7 +4,7 @@ from typing import Annotated, ClassVar -from oa_configurator import DatabaseConfig, PackageConfigBase, RefTo +from oa_configurator import CDMDatabaseConfig, PackageConfigBase, RefTo class OrmLoaderConfig(PackageConfigBase): @@ -15,9 +15,14 @@ class OrmLoaderConfig(PackageConfigBase): class exists to register orm-loader in the oa-configurator ecosystem, provide a canonical ``configure_logging()`` entry point, and declare the test database used by the integration test suite. + + Notes + ----- + By design, this config is for internal use only and must not be + imported or resolved by any other package. """ tool_name: ClassVar[str] = "orm_loader" extra_logging_namespaces: ClassVar[tuple[str, ...]] = () - test_orm_db: Annotated[str | None, RefTo(DatabaseConfig, is_test=True)] = None + test_orm_db: Annotated[str | None, RefTo(CDMDatabaseConfig, is_test=True)] = None