Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ authors = [
requires-python = ">=3.12"
dependencies = [
"chardet>=5.2.0",
"oa-configurator==0.1.2",
"oa-configurator==0.1.2", # TODO: bump to >=1.0.0,<2.0.0
"pandas>=2.3.3",
"pyarrow>=23.0.1",
"sqlalchemy>=2.0.45",
Expand Down Expand Up @@ -49,7 +49,7 @@ postgres = [
"psycopg[binary]>=3.2",
]
dev = [
"oa-configurator[postgres]==0.1.2",
"oa-configurator[postgres]==0.1.2", # TODO: bump to >=1.0.0,<2.0.0
"pytest>=9.0.3",
"ty>=0.0.59",
"ruff>=0.14.11",
Expand Down
37 changes: 13 additions & 24 deletions src/orm_loader/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,38 +2,27 @@

from __future__ import annotations

from typing import ClassVar
from typing import Annotated, ClassVar

from oa_configurator import DatabaseConfig, PackageConfigBase, ResourceSpec
from oa_configurator import CDMDatabaseConfig, PackageConfigBase, RefTo


class OrmLoaderConfig(PackageConfigBase):
"""oa-configurator config class for orm-loader.

orm-loader is connection-agnostic — it accepts SQLAlchemy sessions/engines
as parameters and owns no database resources. This class exists to register
orm-loader in the oa-configurator ecosystem, provide a canonical
``configure_logging()`` entry point, and declare the test database resource
used by the integration test suite.
as parameters and owns no production database resource of its own. This
class exists to register orm-loader in the oa-configurator ecosystem,
provide a canonical ``configure_logging()`` entry point, and declare the
test database used by the integration test suite.

Notes
-----
By design, this config is for internal use only and must not be
imported or resolved by any other package.
"""

TEST_DB: ClassVar[ResourceSpec] = ResourceSpec(
semantic_name="test_orm_db",
display_name="ORM Loader Test Database",
description="PostgreSQL database for running orm-loader integration tests.",
connection_name_hint="pg_test_orm",
is_cdm_database=False,
cdm_schema_default="public",
connection_defaults=DatabaseConfig(
dialect="postgresql+psycopg",
host="localhost",
port=55432,
user="test",
password="test",
database_name="test",
),
)

tool_name: ClassVar[str] = "orm_loader"
extra_logging_namespaces: ClassVar[tuple[str, ...]] = ()
test_resources: ClassVar[tuple[ResourceSpec, ...]] = (TEST_DB,)

test_orm_db: Annotated[str | None, RefTo(CDMDatabaseConfig, is_test=True)] = None
4 changes: 2 additions & 2 deletions tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,10 @@ def session(engine):

@pytest.fixture(scope="session")
def pg_engine():
from oa_configurator.pytest_plugin import ensure_test_db_exists, resolve_test_resource
from oa_configurator.pytest_plugin import ensure_test_db_exists, resolve_test_database
from orm_loader.config import OrmLoaderConfig

url = resolve_test_resource(OrmLoaderConfig.TEST_DB)
url = resolve_test_database(OrmLoaderConfig, "test_orm_db")

try:
ensure_test_db_exists(url)
Expand Down
35 changes: 17 additions & 18 deletions tests/loaders/test_pg_loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,11 @@
import pandas as pd
import pytest
from orm_loader.loaders.loading_helpers import infer_encoding, infer_delim, check_line_ending, quick_load_pg
from orm_loader.config import OrmLoaderConfig

from tests.models import SimpleTable


@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB)
@pytest.mark.requires_database("test_orm_db")
def test_copy_into_staging_with_extra_identity_column(pg_session, tmp_path):
"""COPY must succeed when the staging table has a _rownum identity column."""
csv = tmp_path / "test_table.csv"
Expand All @@ -30,7 +29,7 @@ def test_copy_into_staging_with_extra_identity_column(pg_session, tmp_path):
assert rownums == [1, 2], "_rownum must be auto-populated by IDENTITY sequence"


@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB)
@pytest.mark.requires_database("test_orm_db")
def test_copy_and_orm_path_equivalence(pg_session, tmp_path):
csv = tmp_path / "test_table.csv"

Expand All @@ -52,7 +51,7 @@ def test_copy_and_orm_path_equivalence(pg_session, tmp_path):



@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB)
@pytest.mark.requires_database("test_orm_db")
def test_postgres_copy_fast_path(pg_session, tmp_path):
csv = tmp_path / "test_table.csv"
pd.DataFrame([{"id": 1, "name": "alpha"}]).to_csv(csv, index=False)
Expand All @@ -62,7 +61,7 @@ def test_postgres_copy_fast_path(pg_session, tmp_path):

assert inserted == 1

@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB)
@pytest.mark.requires_database("test_orm_db")
def test_postgres_copy_fast_path_is_used(pg_session, tmp_path, monkeypatch):
csv = tmp_path / "test_table.csv"
pd.DataFrame([{"id": 1, "name": "alpha"}]).to_csv(csv, index=False)
Expand All @@ -82,7 +81,7 @@ def fake_quick_load_pg(*args, **kwargs):
assert called["copy"] is True
assert inserted == 1

@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB)
@pytest.mark.requires_database("test_orm_db")
def test_copy_failure_falls_back_to_orm(pg_session, tmp_path, monkeypatch):
csv = tmp_path / "test_table.csv"
pd.DataFrame([{"id": 1, "name": "alpha"}]).to_csv(csv, index=False)
Expand All @@ -102,7 +101,7 @@ def broken_copy(*args, **kwargs):
assert [(r.id, r.name) for r in rows] == [(1, "alpha")]


@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB)
@pytest.mark.requires_database("test_orm_db")
def test_postgres_upsert_does_not_update(pg_session, tmp_path):
csv = tmp_path / "test_table.csv"

Expand All @@ -119,7 +118,7 @@ def test_postgres_upsert_does_not_update(pg_session, tmp_path):
assert [(r.id, r.name) for r in rows] == [(1, "alpha")]


@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB)
@pytest.mark.requires_database("test_orm_db")
def test_postgres_insert_if_empty(pg_session, tmp_path):
csv = tmp_path / "test_table.csv"

Expand All @@ -146,7 +145,7 @@ def test_postgres_insert_if_empty(pg_session, tmp_path):
]


@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB)
@pytest.mark.requires_database("test_orm_db")
def test_postgres_insert_if_empty_raises_on_non_empty_target(pg_session, tmp_path):
csv = tmp_path / "test_table.csv"

Expand All @@ -164,7 +163,7 @@ def test_postgres_insert_if_empty_raises_on_non_empty_target(pg_session, tmp_pat
)


@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB)
@pytest.mark.requires_database("test_orm_db")
def test_postgres_copy_large_batch(pg_session, tmp_path):
csv = tmp_path / "test_table.csv"

Expand All @@ -181,7 +180,7 @@ def test_postgres_copy_large_batch(pg_session, tmp_path):
assert inserted == 9999


@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB)
@pytest.mark.requires_database("test_orm_db")
def test_staging_schema_matches_target(pg_session, tmp_path):
csv = tmp_path / "test_table.csv"
pd.DataFrame([{"id": 1, "name": "alpha"}]).to_csv(csv, index=False)
Expand Down Expand Up @@ -251,7 +250,7 @@ def test_check_line_ending_unknown(caplog):
assert "Unable to detect line ending" in caplog.text


@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB)
@pytest.mark.requires_database("test_orm_db")
def test_quick_load_pg_basic(pg_session, tmp_path):
csv = tmp_path / "test_table.csv"
csv.write_text("id,name\n1,alpha\n2,beta\n")
Expand All @@ -267,7 +266,7 @@ def test_quick_load_pg_basic(pg_session, tmp_path):
assert rows == [(1, "alpha"), (2, "beta")]


@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB)
@pytest.mark.requires_database("test_orm_db")
def test_quick_load_pg_lowercases_header(pg_session, tmp_path):
csv = tmp_path / "test_table.csv"
csv.write_text("ID,NAME\n1,alpha\n")
Expand All @@ -279,7 +278,7 @@ def test_quick_load_pg_lowercases_header(pg_session, tmp_path):
assert row == (1, "alpha")


@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB)
@pytest.mark.requires_database("test_orm_db")
def test_quick_load_pg_tab_delimiter(pg_session, tmp_path):
csv = tmp_path / "test_table.csv"
csv.write_text("id\tname\n1\talpha\n2\tbeta\n")
Expand All @@ -291,7 +290,7 @@ def test_quick_load_pg_tab_delimiter(pg_session, tmp_path):
assert rows == [(1, "alpha"), (2, "beta")]


@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB)
@pytest.mark.requires_database("test_orm_db")
def test_quick_load_pg_rollback_on_error(pg_session, tmp_path):
csv = tmp_path / "test_table.csv"
csv.write_text("id,name\n1,alpha\n2,\n") # violates NOT NULL
Expand All @@ -303,7 +302,7 @@ def test_quick_load_pg_rollback_on_error(pg_session, tmp_path):
assert rows == 0


@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB)
@pytest.mark.requires_database("test_orm_db")
def test_quick_load_pg_equivalence_with_orm(pg_session, tmp_path):
csv = tmp_path / "test_table.csv"
csv.write_text("id,name\n1,alpha\n2,beta\n")
Expand All @@ -327,7 +326,7 @@ def test_quick_load_pg_equivalence_with_orm(pg_session, tmp_path):
assert rows_pg == rows_orm


@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB)
@pytest.mark.requires_database("test_orm_db")
def test_quick_load_pg_trailing_blank_lines(pg_session, tmp_path):
csv = tmp_path / "test_table.csv"

Expand All @@ -346,7 +345,7 @@ def test_quick_load_pg_trailing_blank_lines(pg_session, tmp_path):
assert total == 2
assert rows == [(1, "alpha"), (2, "beta")]

@pytest.mark.requires_resource(OrmLoaderConfig.TEST_DB)
@pytest.mark.requires_database("test_orm_db")
def test_copy_fails_with_raw_carriage_returns_but_succeeds_after_normalisation(pg_session, tmp_path):
csv = tmp_path / "test_table.csv"

Expand Down
Loading