diff --git a/pyproject.toml b/pyproject.toml index a92d203..5a02ce6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ authors = [ requires-python = ">=3.12" dependencies = [ "chardet>=5.2.0", - "oa-configurator==0.1.2", + "oa-configurator==0.1.2", # TODO: bump to >=1.0.0,<2.0.0 "pandas>=2.3.3", "pyarrow>=23.0.1", "sqlalchemy>=2.0.45", @@ -49,7 +49,7 @@ postgres = [ "psycopg[binary]>=3.2", ] dev = [ - "oa-configurator[postgres]==0.1.2", + "oa-configurator[postgres]==0.1.2", # TODO: bump to >=1.0.0,<2.0.0 "pytest>=9.0.3", "ty>=0.0.59", "ruff>=0.14.11", diff --git a/src/orm_loader/config.py b/src/orm_loader/config.py index 0c1f3b6..497c719 100644 --- a/src/orm_loader/config.py +++ b/src/orm_loader/config.py @@ -2,38 +2,27 @@ from __future__ import annotations -from typing import ClassVar +from typing import Annotated, ClassVar -from oa_configurator import DatabaseConfig, PackageConfigBase, ResourceSpec +from oa_configurator import CDMDatabaseConfig, PackageConfigBase, RefTo class OrmLoaderConfig(PackageConfigBase): """oa-configurator config class for orm-loader. orm-loader is connection-agnostic — it accepts SQLAlchemy sessions/engines - as parameters and owns no database resources. This class exists to register - orm-loader in the oa-configurator ecosystem, provide a canonical - ``configure_logging()`` entry point, and declare the test database resource - used by the integration test suite. + as parameters and owns no production database resource of its own. This + class exists to register orm-loader in the oa-configurator ecosystem, + provide a canonical ``configure_logging()`` entry point, and declare the + test database used by the integration test suite. + + Notes + ----- + By design, this config is for internal use only and must not be + imported or resolved by any other package. """ - TEST_DB: ClassVar[ResourceSpec] = ResourceSpec( - semantic_name="test_orm_db", - display_name="ORM Loader Test Database", - description="PostgreSQL database for running orm-loader integration tests.", - connection_name_hint="pg_test_orm", - is_cdm_database=False, - cdm_schema_default="public", - connection_defaults=DatabaseConfig( - dialect="postgresql+psycopg", - host="localhost", - port=55432, - user="test", - password="test", - database_name="test", - ), - ) - tool_name: ClassVar[str] = "orm_loader" extra_logging_namespaces: ClassVar[tuple[str, ...]] = () - test_resources: ClassVar[tuple[ResourceSpec, ...]] = (TEST_DB,) + + test_orm_db: Annotated[str | None, RefTo(CDMDatabaseConfig, is_test=True)] = None diff --git a/tests/conftest.py b/tests/conftest.py index a5a9abe..40a07d4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -30,10 +30,10 @@ def session(engine): @pytest.fixture(scope="session") def pg_engine(): - from oa_configurator.pytest_plugin import ensure_test_db_exists, resolve_test_resource + from oa_configurator.pytest_plugin import ensure_test_db_exists, resolve_test_database from orm_loader.config import OrmLoaderConfig - url = resolve_test_resource(OrmLoaderConfig.TEST_DB) + url = resolve_test_database(OrmLoaderConfig, "test_orm_db") try: ensure_test_db_exists(url) diff --git a/tests/loaders/test_pg_loader.py b/tests/loaders/test_pg_loader.py index 9877e5e..d75387b 100644 --- a/tests/loaders/test_pg_loader.py +++ b/tests/loaders/test_pg_loader.py @@ -2,12 +2,11 @@ import pandas as pd import pytest from orm_loader.loaders.loading_helpers import infer_encoding, infer_delim, check_line_ending, quick_load_pg -from orm_loader.config import OrmLoaderConfig from tests.models import SimpleTable -@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database("test_orm_db") def test_copy_into_staging_with_extra_identity_column(pg_session, tmp_path): """COPY must succeed when the staging table has a _rownum identity column.""" csv = tmp_path / "test_table.csv" @@ -30,7 +29,7 @@ def test_copy_into_staging_with_extra_identity_column(pg_session, tmp_path): assert rownums == [1, 2], "_rownum must be auto-populated by IDENTITY sequence" -@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database("test_orm_db") def test_copy_and_orm_path_equivalence(pg_session, tmp_path): csv = tmp_path / "test_table.csv" @@ -52,7 +51,7 @@ def test_copy_and_orm_path_equivalence(pg_session, tmp_path): -@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database("test_orm_db") def test_postgres_copy_fast_path(pg_session, tmp_path): csv = tmp_path / "test_table.csv" pd.DataFrame([{"id": 1, "name": "alpha"}]).to_csv(csv, index=False) @@ -62,7 +61,7 @@ def test_postgres_copy_fast_path(pg_session, tmp_path): assert inserted == 1 -@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database("test_orm_db") def test_postgres_copy_fast_path_is_used(pg_session, tmp_path, monkeypatch): csv = tmp_path / "test_table.csv" pd.DataFrame([{"id": 1, "name": "alpha"}]).to_csv(csv, index=False) @@ -82,7 +81,7 @@ def fake_quick_load_pg(*args, **kwargs): assert called["copy"] is True assert inserted == 1 -@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database("test_orm_db") def test_copy_failure_falls_back_to_orm(pg_session, tmp_path, monkeypatch): csv = tmp_path / "test_table.csv" pd.DataFrame([{"id": 1, "name": "alpha"}]).to_csv(csv, index=False) @@ -102,7 +101,7 @@ def broken_copy(*args, **kwargs): assert [(r.id, r.name) for r in rows] == [(1, "alpha")] -@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database("test_orm_db") def test_postgres_upsert_does_not_update(pg_session, tmp_path): csv = tmp_path / "test_table.csv" @@ -119,7 +118,7 @@ def test_postgres_upsert_does_not_update(pg_session, tmp_path): assert [(r.id, r.name) for r in rows] == [(1, "alpha")] -@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database("test_orm_db") def test_postgres_insert_if_empty(pg_session, tmp_path): csv = tmp_path / "test_table.csv" @@ -146,7 +145,7 @@ def test_postgres_insert_if_empty(pg_session, tmp_path): ] -@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database("test_orm_db") def test_postgres_insert_if_empty_raises_on_non_empty_target(pg_session, tmp_path): csv = tmp_path / "test_table.csv" @@ -164,7 +163,7 @@ def test_postgres_insert_if_empty_raises_on_non_empty_target(pg_session, tmp_pat ) -@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database("test_orm_db") def test_postgres_copy_large_batch(pg_session, tmp_path): csv = tmp_path / "test_table.csv" @@ -181,7 +180,7 @@ def test_postgres_copy_large_batch(pg_session, tmp_path): assert inserted == 9999 -@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database("test_orm_db") def test_staging_schema_matches_target(pg_session, tmp_path): csv = tmp_path / "test_table.csv" pd.DataFrame([{"id": 1, "name": "alpha"}]).to_csv(csv, index=False) @@ -251,7 +250,7 @@ def test_check_line_ending_unknown(caplog): assert "Unable to detect line ending" in caplog.text -@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database("test_orm_db") def test_quick_load_pg_basic(pg_session, tmp_path): csv = tmp_path / "test_table.csv" csv.write_text("id,name\n1,alpha\n2,beta\n") @@ -267,7 +266,7 @@ def test_quick_load_pg_basic(pg_session, tmp_path): assert rows == [(1, "alpha"), (2, "beta")] -@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database("test_orm_db") def test_quick_load_pg_lowercases_header(pg_session, tmp_path): csv = tmp_path / "test_table.csv" csv.write_text("ID,NAME\n1,alpha\n") @@ -279,7 +278,7 @@ def test_quick_load_pg_lowercases_header(pg_session, tmp_path): assert row == (1, "alpha") -@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database("test_orm_db") def test_quick_load_pg_tab_delimiter(pg_session, tmp_path): csv = tmp_path / "test_table.csv" csv.write_text("id\tname\n1\talpha\n2\tbeta\n") @@ -291,7 +290,7 @@ def test_quick_load_pg_tab_delimiter(pg_session, tmp_path): assert rows == [(1, "alpha"), (2, "beta")] -@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database("test_orm_db") def test_quick_load_pg_rollback_on_error(pg_session, tmp_path): csv = tmp_path / "test_table.csv" csv.write_text("id,name\n1,alpha\n2,\n") # violates NOT NULL @@ -303,7 +302,7 @@ def test_quick_load_pg_rollback_on_error(pg_session, tmp_path): assert rows == 0 -@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database("test_orm_db") def test_quick_load_pg_equivalence_with_orm(pg_session, tmp_path): csv = tmp_path / "test_table.csv" csv.write_text("id,name\n1,alpha\n2,beta\n") @@ -327,7 +326,7 @@ def test_quick_load_pg_equivalence_with_orm(pg_session, tmp_path): assert rows_pg == rows_orm -@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database("test_orm_db") def test_quick_load_pg_trailing_blank_lines(pg_session, tmp_path): csv = tmp_path / "test_table.csv" @@ -346,7 +345,7 @@ def test_quick_load_pg_trailing_blank_lines(pg_session, tmp_path): assert total == 2 assert rows == [(1, "alpha"), (2, "beta")] -@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB) +@pytest.mark.requires_database("test_orm_db") def test_copy_fails_with_raw_carriage_returns_but_succeeds_after_normalisation(pg_session, tmp_path): csv = tmp_path / "test_table.csv"