diff --git a/.gitignore b/.gitignore index 2d1c4ffb..0a1cc067 100644 --- a/.gitignore +++ b/.gitignore @@ -124,6 +124,8 @@ venv.bak/ docs/* !docs/extract_ai_user_guide.md !docs/github-app-deployment.md +!docs/testing-strategy.md +!pytest-integration.ini .DS_Store # Spyder project settings diff --git a/AGENTS.md b/AGENTS.md index accbb60d..f855d5be 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -29,6 +29,31 @@ production Python version; verify that repository before making production runtime claims. +## Running tests locally + +- Use `.\scripts\test-local.ps1` for the default local verification run. It + clears live credentials and runs `pytest -c pytest-local.ini` with an isolated + temp root, so local and AI-agent runs do not accidentally call OpenAI, + WrangleWorks, AWS, Gemini, SerpAPI, or other live providers. +- Use focused `python -m pytest -c pytest-local.ini ` commands + while iterating. Do not use bare `pytest` as the default local command; it can + collect credentialed legacy tests that are intentionally outside the local + loop. +- New AI-generated tests should be `unit` or `contract` tests unless the live + service behavior is the feature under test. Mock provider transports, + WrangleWorks model APIs, and credential lookups with `monkeypatch` or + `mocker` for unit/contract coverage. +- Mark tests that need deployed services with the most specific pytest markers: + `integration`, plus `live_ai`, `live_wrangleworks`, `live_s3`, or `slow` as + applicable. Live tests must be opt-in and must not be added to + `pytest-local.ini` without mocking the external dependency. +- Prefer `tmp_path` for new file-writing tests. Avoid adding new shared + `tests/temp` outputs unless compatibility with an existing recipe fixture + requires that path. +- Any integration test that creates a model or external resource must register + it for cleanup immediately and delete it in a fixture finalizer or `finally` + block, even when assertions fail. + ## Code Review Rules ### Make the required action explicit diff --git a/docs/testing-strategy.md b/docs/testing-strategy.md new file mode 100644 index 00000000..94dd7bdd --- /dev/null +++ b/docs/testing-strategy.md @@ -0,0 +1,182 @@ +# WranglesPY Testing Strategy + +## Goals + +WranglesPY needs high-confidence tests without making local development or +AI-assisted coding slow, flaky, or dependent on live production state. + +The default testing loop should be fast, deterministic, and credential-free. +Live-service tests still matter, but they should be explicit, opt-in, and +responsible for cleaning up anything they create. + +## Test Categories + +Use pytest markers to make test intent visible: + +- `unit`: pure local tests with no network, credentials, or persistent service + state. +- `contract`: mocked provider/API tests that verify request payloads, response + parsing, retries, errors, caching, and schema contracts. +- `integration`: tests that exercise live services or deployed infrastructure. +- `live_ai`: tests that call a live AI provider. +- `live_wrangleworks`: tests that call live WrangleWorks services. +- `live_s3`: tests that call live AWS/S3 resources. +- `slow`: intentionally slow tests that should be excluded from tight local + loops. + +Unit and contract tests are the default for local development and AI-generated +test additions. Integration tests are opt-in. + +## Default Local Workflow + +Use: + +```powershell +.\scripts\test-local.ps1 +``` + +For focused iteration: + +```powershell +python -m pytest -c pytest-local.ini +``` + +The local runner clears live credentials before running pytest. This prevents +local and AI-agent runs from accidentally calling OpenAI, WrangleWorks, AWS, +Gemini, SerpAPI, or other external providers. + +Live integration tests are opt-in: + +```powershell +python -m pytest -c pytest-integration.ini +``` + +Run them only in an environment that intentionally provides the required live +credentials and cleanup permissions. + +Validate marker/config drift with: + +```powershell +python scripts/check_pytest_markers.py +``` + +## AI-Generated Test Standard + +AI-generated tests should follow the same bar as human-authored tests: + +- Prefer unit or contract coverage. +- Mock provider transports and WrangleWorks APIs. +- Assert behavior and externally visible contracts, not incidental + implementation details. +- Include failure-path assertions when the change affects retries, validation, + parsing, cleanup, or error reporting. +- Use deterministic examples and stable fake responses. +- Use `tmp_path` for newly written files. +- Do not add new `pytest-local.ini` ignores or deselects unless the test is + intentionally live-only and marked accordingly. + +## Example Fixtures + +Shared fixtures should make the common path easy: + +- fake OpenAI Responses API success response; +- malformed OpenAI response body; +- transient transport error followed by success; +- rate-limit response with retry metadata; +- fake WrangleWorks model metadata; +- fake WrangleWorks model content; +- fake model creation response with a returned model id; +- cleanup registry for live-created models. + +Keep fixtures small and explicit. Test-specific payloads can override the +default fake body rather than creating unrelated helper types in each file. + +## Mocks Versus Live Calls + +Use mocks for: + +- payload shape; +- schema compilation; +- request headers and query parameters; +- retry and timeout behavior; +- cache key behavior; +- parsing and validation failures; +- model creation/update/delete request contracts. + +Use live integration tests for: + +- provider behavior that cannot be represented by a stable contract test; +- deployed WrangleWorks authentication and authorization behavior; +- model lifecycle behavior after the backend accepts a created model; +- S3 behavior that depends on real AWS object semantics. + +Live tests should be narrow smoke tests. They should not duplicate broad unit or +contract coverage. + +## Model Creation And Cleanup + +Any test that creates a live model must: + +1. Generate a unique model name with a test prefix, timestamp or run id, and a + short random suffix. +2. Register the model id for cleanup immediately after creation. +3. Delete the model in a fixture finalizer or `finally` block. +4. Log the created model id clearly enough for manual cleanup if the process is + interrupted. + +Integration jobs should also include a periodic cleanup/audit path for stale +test models by prefix and age. + +## Unit Test Writing Agent Evaluation + +Microsoft's Unit Test Writing Agent framework is a good fit for WranglesPY's +unit and contract layers because it is designed to inspect project conventions, +generate tests, run them, and iterate on failures. Its published guidance also +explicitly discourages unit tests that call external URLs, open ports, depend on +exact timing, or exercise infrastructure. + +Use it first as a focused assistant, not as a whole-suite rewrite tool. + +Pilot scope: + +- `wrangles/ai_cache.py` +- `wrangles/ai_definition.py` +- `wrangles/openai_responses.py` + +Evaluation checklist: + +- Does it follow existing pytest style? +- Does it reuse shared fixtures? +- Does it avoid live provider calls? +- Does it produce meaningful assertions? +- Does it cover both success and failure behavior? +- Does it run `pytest -c pytest-local.ini` or a focused equivalent? +- Does it avoid adding brittle sleeps or broad deselects? + +References: + +- Microsoft blog post: https://devblogs.microsoft.com/dotnet/polyglot-unit-testing-agent/ +- Plugin repository: https://github.com/dotnet/skills/tree/main/plugins/dotnet-test + +## Rollout Plan: Unit And Contract Tests + +1. Register pytest markers and document local testing policy. +2. Add shared fake provider and model fixtures. +3. Pilot Unit Test Writing Agent on one AI-adjacent module. +4. Review generated tests for assertion quality and fixture reuse. +5. Convert repeated inline fake OpenAI/WrangleWorks responses to shared + fixtures when doing nearby work. +6. Add or restore contract tests before touching `pytest-local.ini` ignores. +7. Require AI-authored PRs to report the focused pytest command they ran. + +## Rollout Plan: Integration Tests + +1. Mark existing live-service tests with `integration` and the relevant + provider marker. +2. Move integration tests behind an explicit manual or scheduled command. +3. Standardize live model creation and cleanup fixtures. +4. Keep normal PR CI focused on unit and contract tests. +5. Add a credentialed scheduled integration job once cleanup is reliable. +6. Track flaky integration tests separately from product regressions. +7. Periodically audit stale live resources and remove or repair obsolete + integration coverage. diff --git a/pytest-integration.ini b/pytest-integration.ini new file mode 100644 index 00000000..a155db5e --- /dev/null +++ b/pytest-integration.ini @@ -0,0 +1,16 @@ +[pytest] +markers = + unit: pure local tests with no network, credentials, or persistent service state + contract: mocked provider/API tests that verify request/response contracts + integration: tests that exercise live services or deployed infrastructure + live_ai: tests that call a live AI provider + live_wrangleworks: tests that call live WrangleWorks services + live_s3: tests that call live AWS/S3 resources + slow: intentionally slow tests that should be excluded from tight local loops +testpaths = + tests +addopts = + -p no:cacheprovider + -m integration +filterwarnings = + ignore:'imghdr' is deprecated and slated for removal in Python 3.13:DeprecationWarning:apprise\.utils\.pgp diff --git a/pytest-local.ini b/pytest-local.ini index 11c64814..9fbdf6e4 100644 --- a/pytest-local.ini +++ b/pytest-local.ini @@ -1,4 +1,12 @@ [pytest] +markers = + unit: pure local tests with no network, credentials, or persistent service state + contract: mocked provider/API tests that verify request/response contracts + integration: tests that exercise live services or deployed infrastructure + live_ai: tests that call a live AI provider + live_wrangleworks: tests that call live WrangleWorks services + live_s3: tests that call live AWS/S3 resources + slow: intentionally slow tests that should be excluded from tight local loops testpaths = tests/test_ai_cache.py tests/test_ai_definition.py @@ -6,6 +14,7 @@ testpaths = tests/test_data.py tests/test_dataframe.py tests/test_openai_extract_ai.py + tests/test_pytest_marker_config.py tests/recipes tests/connectors/test_access.py tests/connectors/test_concurrent.py @@ -27,28 +36,6 @@ testpaths = tests/connectors/test_test.py addopts = -p no:cacheprovider - --ignore=tests/recipes/wrangles/test_extract.py - --ignore=tests/recipes/wrangles/test_extract_unit_conversion.py - --ignore=tests/recipes/wrangles/test_generate.py - --ignore=tests/recipes/wrangles/test_search.py - --deselect=tests/recipes/wrangles/test_create.py::TestCreateEmbeddings - --deselect=tests/recipes/wrangles/test_main.py::TestClassify - --deselect=tests/recipes/wrangles/test_main.py::TestLookup - --deselect=tests/recipes/wrangles/test_main.py::TestMatrix::test_extract_ai - --deselect=tests/recipes/wrangles/test_main.py::TestStandardize - --deselect=tests/recipes/wrangles/test_main.py::TestTranslate - --deselect=tests/recipes/test_custom_functions.py::test_local_takes_priority - --deselect=tests/recipes/test_custom_functions.py::test_model_with_custom_functions - --deselect=tests/recipes/test_recipes.py::test_recipe_by_latest_version - --deselect=tests/recipes/test_recipes.py::test_recipe_by_production_version - --deselect=tests/recipes/test_recipes.py::test_recipe_by_version_id - --deselect=tests/recipes/test_recipes.py::test_recipe_by_version_latest - --deselect=tests/recipes/test_recipes.py::test_recipe_by_version_tag - --deselect=tests/recipes/test_recipes.py::test_recipe_from_url - --deselect=tests/recipes/test_recipes.py::test_recipe_from_url_not_found - --deselect=tests/recipes/test_recipes.py::test_recipe_model - --deselect=tests/recipes/test_recipes.py::test_recipe_wrong_model - --deselect=tests/connectors/test_recipe.py::test_model_id - --deselect=tests/connectors/test_recipe.py::test_model_with_custom_functions + -m "not integration and not slow" filterwarnings = ignore:'imghdr' is deprecated and slated for removal in Python 3.13:DeprecationWarning:apprise\.utils\.pgp diff --git a/scripts/check_pytest_markers.py b/scripts/check_pytest_markers.py new file mode 100644 index 00000000..bf97573f --- /dev/null +++ b/scripts/check_pytest_markers.py @@ -0,0 +1,149 @@ +""" +Validate pytest marker configuration against marker usage in tests. + +This keeps the local/offline and integration pytest configs aligned and makes +new live-test markers explicit instead of letting them drift silently. +""" +from __future__ import annotations + +import ast +import configparser +import re +import sys +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[1] +PYTEST_CONFIGS = [ + REPO_ROOT / "setup.cfg", + REPO_ROOT / "pytest-local.ini", + REPO_ROOT / "pytest-integration.ini", +] +REQUIRED_LOCAL_EXPRESSION_PARTS = ("not integration", "not slow") +REQUIRED_INTEGRATION_EXPRESSION = "integration" +BUILTIN_MARKERS = { + "anyio", + "filterwarnings", + "parametrize", + "skip", + "skipif", + "tryfirst", + "trylast", + "usefixtures", + "xfail", +} + + +def _pytest_section_name(path: Path) -> str: + return "tool:pytest" if path.name == "setup.cfg" else "pytest" + + +def _read_pytest_config(path: Path) -> configparser.SectionProxy: + parser = configparser.ConfigParser() + parser.read(path, encoding="utf-8") + section = _pytest_section_name(path) + if not parser.has_section(section): + raise AssertionError(f"{path.relative_to(REPO_ROOT)} is missing [{section}]") + return parser[section] + + +def marker_definitions(path: Path) -> dict[str, str]: + section = _read_pytest_config(path) + if "markers" not in section: + raise AssertionError(f"{path.relative_to(REPO_ROOT)} is missing pytest markers") + + markers = {} + for raw_line in section["markers"].splitlines(): + line = raw_line.strip() + if not line: + continue + name, _, description = line.partition(":") + name = name.strip() + description = description.strip() + if not name or not description: + raise AssertionError( + f"{path.relative_to(REPO_ROOT)} has an invalid marker line: {raw_line!r}" + ) + markers[name] = description + return markers + + +def marker_calls(path: Path) -> set[str]: + source = path.read_text(encoding="utf-8") + tree = ast.parse(source, filename=str(path)) + names = set() + for node in ast.walk(tree): + if not isinstance(node, ast.Attribute): + continue + parent = node.value + if ( + isinstance(parent, ast.Attribute) + and parent.attr == "mark" + and isinstance(parent.value, ast.Name) + and parent.value.id == "pytest" + ): + names.add(node.attr) + + # Some legacy skipif decorators use the compact string form: + # @pytest.mark.skipif("OPENAI_API_KEY" not in os.environ, ...) + for match in re.finditer(r"pytest\.mark\.([A-Za-z_][A-Za-z0-9_]*)", source): + names.add(match.group(1)) + return names + + +def used_markers() -> dict[str, list[Path]]: + usages: dict[str, list[Path]] = {} + for path in sorted((REPO_ROOT / "tests").rglob("test*.py")): + for marker in marker_calls(path): + usages.setdefault(marker, []).append(path) + return usages + + +def _addopts(path: Path) -> str: + section = _read_pytest_config(path) + return " ".join(section.get("addopts", "").split()) + + +def validate() -> list[str]: + errors = [] + definitions_by_config = {path: marker_definitions(path) for path in PYTEST_CONFIGS} + + baseline = definitions_by_config[PYTEST_CONFIGS[0]] + for path, definitions in definitions_by_config.items(): + if definitions != baseline: + errors.append( + f"{path.relative_to(REPO_ROOT)} marker definitions differ from " + f"{PYTEST_CONFIGS[0].relative_to(REPO_ROOT)}" + ) + + declared = set(baseline) | BUILTIN_MARKERS + for marker, paths in used_markers().items(): + if marker in declared: + continue + formatted_paths = ", ".join(str(path.relative_to(REPO_ROOT)) for path in paths) + errors.append(f"pytest marker '{marker}' is used but not registered: {formatted_paths}") + + local_addopts = _addopts(REPO_ROOT / "pytest-local.ini") + for expression in REQUIRED_LOCAL_EXPRESSION_PARTS: + if expression not in local_addopts: + errors.append(f"pytest-local.ini addopts must exclude '{expression}'") + + integration_addopts = _addopts(REPO_ROOT / "pytest-integration.ini") + if REQUIRED_INTEGRATION_EXPRESSION not in integration_addopts: + errors.append("pytest-integration.ini addopts must select integration tests") + + return errors + + +def main() -> int: + errors = validate() + if errors: + for error in errors: + print(f"ERROR: {error}", file=sys.stderr) + return 1 + print("pytest marker configuration is consistent") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/setup.cfg b/setup.cfg index fa522ab7..d838d28a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,5 +2,13 @@ description_file = README.md [tool:pytest] +markers = + unit: pure local tests with no network, credentials, or persistent service state + contract: mocked provider/API tests that verify request/response contracts + integration: tests that exercise live services or deployed infrastructure + live_ai: tests that call a live AI provider + live_wrangleworks: tests that call live WrangleWorks services + live_s3: tests that call live AWS/S3 resources + slow: intentionally slow tests that should be excluded from tight local loops filterwarnings = ignore:'imghdr' is deprecated and slated for removal in Python 3.13:DeprecationWarning:apprise\.utils\.pgp diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..dfc527e7 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,105 @@ +import json + +import pytest + + +class JsonResponse: + """Minimal requests-like response for contract tests.""" + + def __init__(self, body, ok=True, status_code=200, headers=None, text=None): + self._body = body + self.ok = ok + self.status_code = status_code + self.headers = headers or {} + self.text = text if text is not None else json.dumps(body) + + def json(self): + return self._body + + +class MalformedJsonResponse(JsonResponse): + def __init__(self, ok=True, status_code=200, headers=None, text="not json"): + super().__init__(None, ok=ok, status_code=status_code, headers=headers, text=text) + + def json(self): + raise json.JSONDecodeError("Malformed response body", self.text, 0) + + +@pytest.fixture +def json_response_factory(): + return JsonResponse + + +@pytest.fixture +def malformed_json_response_factory(): + return MalformedJsonResponse + + +@pytest.fixture +def openai_output_text_body(): + def make(payload): + return { + "output": [{ + "type": "message", + "content": [{ + "type": "output_text", + "text": json.dumps(payload), + }], + }] + } + + return make + + +@pytest.fixture +def openai_success_response(json_response_factory, openai_output_text_body): + def make(payload): + return json_response_factory(openai_output_text_body(payload)) + + return make + + +@pytest.fixture +def openai_rate_limit_response(json_response_factory): + def make(retry_after="1"): + return json_response_factory( + {"error": {"message": "Rate limit reached", "type": "requests"}}, + ok=False, + status_code=429, + headers={"retry-after": retry_after}, + ) + + return make + + +@pytest.fixture +def fake_model_id(): + return "12345678-1234-1234-1234-123456789abc" + + +@pytest.fixture +def model_creation_response(json_response_factory, fake_model_id): + return json_response_factory({"model_id": fake_model_id}) + + +@pytest.fixture +def created_model_ids(): + model_ids = [] + yield model_ids + + if not model_ids: + return + + import wrangles + + cleanup_errors = [] + for model_id in model_ids: + try: + wrangles.train.delete(model_id) + except Exception as exc: + cleanup_errors.append(f"{model_id}: {exc}") + + if cleanup_errors: + raise RuntimeError( + "Failed to clean up live test model(s): " + "; ".join(cleanup_errors) + ) diff --git a/tests/connectors/test_recipe.py b/tests/connectors/test_recipe.py index 10799612..3759063d 100644 --- a/tests/connectors/test_recipe.py +++ b/tests/connectors/test_recipe.py @@ -1,5 +1,6 @@ import wrangles import pandas as pd +import pytest from wrangles.connectors import memory @@ -59,7 +60,7 @@ def test_run_recipe_connector(): assert run(recipe) == None -def test_function_sub_recipe(): +def test_function_sub_recipe(tmp_path): """ Test that custom functions are able to be called by sub-recipes. @@ -83,7 +84,7 @@ def custom_func_1(df, case): return df def write_1(df, type): - df.to_excel(f"tests/temp/excel.{type}") + df.to_excel(tmp_path / f"excel.{type}") df = wrangles.recipe.run( recipe=main_recipe, @@ -277,6 +278,8 @@ def test_run(): ) assert memory.dataframes["recipe_run"]["data"][0][0] == "VALUE1" +@pytest.mark.integration +@pytest.mark.live_wrangleworks def test_model_id(): """ Test reading a recipe with a model ID @@ -293,6 +296,8 @@ def test_model_id(): list(df.columns[:3]) == ["Part Number", "Description", "Brand"] ) +@pytest.mark.integration +@pytest.mark.live_wrangleworks def test_model_with_custom_functions(): """ Test a model that includes custom functions diff --git a/tests/connectors/test_s3.py b/tests/connectors/test_s3.py index 8d367ad3..7074cf98 100644 --- a/tests/connectors/test_s3.py +++ b/tests/connectors/test_s3.py @@ -5,6 +5,8 @@ import pytest import time +pytestmark = [pytest.mark.integration, pytest.mark.live_s3] + s3_key = os.getenv('AWS_ACCESS_KEY_ID', '...') s3_secret = os.getenv('AWS_SECRET_ACCESS_KEY', '...') diff --git a/tests/recipes/test_custom_functions.py b/tests/recipes/test_custom_functions.py index 846404f3..213c6a14 100644 --- a/tests/recipes/test_custom_functions.py +++ b/tests/recipes/test_custom_functions.py @@ -1211,10 +1211,44 @@ def wrangle_stuff(): "did not return a dataframe" in info.value.args[0] ) -def test_model_with_custom_functions(): +def _mock_custom_function_model(monkeypatch): + recipe = """ + read: + - test: + rows: 1 + values: + header1: value1 + header2: value2 + wrangles: + - custom.convert_case: + input: header2 + output: header3 + """ + functions = """ +def convert_case(header2: str): + return header2.upper() +""" + monkeypatch.setattr( + wrangles.data, + "model", + lambda model_id: {"purpose": "recipe"}, + ) + monkeypatch.setattr( + wrangles.data, + "model_content", + lambda model_id, version_id=None: { + "recipe": recipe, + "functions": functions, + }, + ) + + +def test_model_with_custom_functions(monkeypatch): """ Test a model that includes custom functions """ + _mock_custom_function_model(monkeypatch) + df = wrangles.recipe.run("42f319a8-0849-4177") assert ( df['header1'][0] == "value1" and @@ -1222,11 +1256,14 @@ def test_model_with_custom_functions(): df['header3'][0] == "VALUE2" ) -def test_local_takes_priority(): + +def test_local_takes_priority(monkeypatch): """ Ensure a locally passed custom function overrides a remote function of the same name """ + _mock_custom_function_model(monkeypatch) + def convert_case(header2: str): return header2.title() diff --git a/tests/recipes/test_recipes.py b/tests/recipes/test_recipes.py index 172d9fe5..5934b791 100644 --- a/tests/recipes/test_recipes.py +++ b/tests/recipes/test_recipes.py @@ -39,6 +39,8 @@ def test_recipe_from__recipe_file(): ) assert df.columns.tolist() == ['ID', 'Find2'] +@pytest.mark.integration +@pytest.mark.live_wrangleworks def test_recipe_from_url(): """ Testing reading a recipe from an https:// source @@ -51,6 +53,8 @@ def test_recipe_from_url(): ) assert df.iloc[0]['out1'] == 'HELLO WORLD' +@pytest.mark.integration +@pytest.mark.live_wrangleworks def test_recipe_from_url_not_found(): """ Test that if a user passes in a recipe as a URL and the @@ -95,6 +99,8 @@ def test_recipe_special_character(): df = wrangles.recipe.run("tests/samples/recipe_special_character.wrgl.yml") assert df.iloc[0]['column'] == 'this is a ° symbol' +@pytest.mark.integration +@pytest.mark.live_wrangleworks def test_recipe_model(): """ Test running a recipe using a model ID @@ -105,6 +111,8 @@ def test_recipe_model(): list(df.columns[:3]) == ["Part Number", "Description", "Brand"] ) +@pytest.mark.integration +@pytest.mark.live_wrangleworks def test_recipe_by_version_id(): """ Test running a recipe using a model ID and version ID @@ -115,6 +123,8 @@ def test_recipe_by_version_id(): list(df.columns) == ["header"] ) +@pytest.mark.integration +@pytest.mark.live_wrangleworks def test_recipe_by_version_tag(): """ Test running a recipe using a model ID and version tag @@ -127,6 +137,8 @@ def test_recipe_by_version_tag(): ) +@pytest.mark.integration +@pytest.mark.live_wrangleworks def test_recipe_by_production_version(): """ Test running a recipe using a model ID and production version @@ -138,52 +150,59 @@ def test_recipe_by_production_version(): ) -def test_recipe_by_production_semantic_version(mocker): +def test_recipe_by_production_semantic_version(monkeypatch): """ Test running a recipe using the production semantic version """ - mocker.patch( - "wrangles.data.model", - return_value={ + monkeypatch.setattr( + wrangles.data, + "model", + lambda model_id: { "purpose": "recipe", "production_version_id": "production-version-id" - } - ) - model_content = mocker.patch( - "wrangles.data.model_content", - return_value={"recipe": "{}"} + }, ) + calls = [] + + def model_content(model_id, version_id=None): + calls.append((model_id, version_id)) + return {"recipe": "{}"} + + monkeypatch.setattr(wrangles.data, "model_content", model_content) wrangles.recipe.run("e954717c-fb9c-4c47:production") - model_content.assert_called_once_with( - "e954717c-fb9c-4c47", - "production-version-id" - ) + assert calls == [("e954717c-fb9c-4c47", "production-version-id")] def test_recipe_by_production_semantic_version_falls_back_to_latest( - mocker, + monkeypatch, caplog ): """ Test the production semantic version falls back when none exists """ - mocker.patch( - "wrangles.data.model", - return_value={"purpose": "recipe"} - ) - model_content = mocker.patch( - "wrangles.data.model_content", - return_value={"recipe": "{}"} + monkeypatch.setattr( + wrangles.data, + "model", + lambda model_id: {"purpose": "recipe"}, ) + calls = [] + + def model_content(model_id, version_id=None): + calls.append((model_id, version_id)) + return {"recipe": "{}"} + + monkeypatch.setattr(wrangles.data, "model_content", model_content) wrangles.recipe.run("e954717c-fb9c-4c47:production") - model_content.assert_called_once_with("e954717c-fb9c-4c47", None) + assert calls == [("e954717c-fb9c-4c47", None)] assert "No production version exists, defaulting to latest version" in caplog.text +@pytest.mark.integration +@pytest.mark.live_wrangleworks def test_recipe_by_version_latest(): """ Test running a recipe using a model ID and latest version @@ -194,6 +213,8 @@ def test_recipe_by_version_latest(): list(df.columns) == ["header"] ) +@pytest.mark.integration +@pytest.mark.live_wrangleworks def test_recipe_by_latest_version(): """ Test running a recipe using a model ID and latest version @@ -204,6 +225,8 @@ def test_recipe_by_latest_version(): list(df.columns) == ["header"] ) +@pytest.mark.integration +@pytest.mark.live_wrangleworks def test_recipe_wrong_model(): """ Test the error message when a model is incorrect type diff --git a/tests/recipes/wrangles/test_create.py b/tests/recipes/wrangles/test_create.py index 9c3e2ae8..67fbfa36 100644 --- a/tests/recipes/wrangles/test_create.py +++ b/tests/recipes/wrangles/test_create.py @@ -1444,6 +1444,8 @@ def test_create_embeddings_rejects_invalid_timeout(self, timeout): with pytest.raises(ValueError, match="timeout must be a positive finite number"): wrangles.openai.embeddings(["test"], api_key="fake-key", timeout=timeout) + @pytest.mark.integration + @pytest.mark.live_ai def test_create_embeddings(self): """ Test generating openai embeddings @@ -1468,6 +1470,8 @@ def test_create_embeddings(self): len(df["embedding"][0]) == 1536 ) + @pytest.mark.integration + @pytest.mark.live_ai def test_create_embeddings_batching(self): """ Test generating openai embeddings @@ -1499,6 +1503,8 @@ def test_create_embeddings_batching(self): len(df) == 150 ) + @pytest.mark.integration + @pytest.mark.live_ai def test_create_embeddings_empty(self): """ Test generating openai embeddings with an empty string @@ -1523,6 +1529,8 @@ def test_create_embeddings_empty(self): len(df["embedding"][0]) == 1536 ) + @pytest.mark.integration + @pytest.mark.live_ai def test_create_embeddings_python_list(self): """ Test generating openai embeddings as a python list @@ -1548,6 +1556,8 @@ def test_create_embeddings_python_list(self): len(df["embedding"][0]) == 1536 ) + @pytest.mark.integration + @pytest.mark.live_ai def test_create_embeddings_np_array(self): """ Test generating openai embeddings as a numpy array @@ -1590,11 +1600,13 @@ def test_create_embeddings_invalid_output_type(self): - create.embeddings: input: text output: embedding - api_key: ${OPENAI_API_KEY} + api_key: fake-key output_type: Something here is not right """ ) + @pytest.mark.integration + @pytest.mark.live_ai def test_create_embeddings_multi_column(self): """ Test generating openai embeddings @@ -1627,6 +1639,8 @@ def test_create_embeddings_multi_column(self): len(df["embedding2"][0]) == 1536 ) + @pytest.mark.integration + @pytest.mark.live_ai def test_create_embeddings_np_array_with_where(self): """ Test using where with a numpy array @@ -1657,6 +1671,8 @@ def test_create_embeddings_np_array_with_where(self): df["text"].values.tolist() == ['This is a test', 'THIS IS NOT A TEST'] ) + @pytest.mark.integration + @pytest.mark.live_ai def test_create_embeddings_kwargs(self): """ Test passing through any unspecified @@ -1705,6 +1721,8 @@ def test_create_embeddings_missing_apikey(self): """ ) + @pytest.mark.integration + @pytest.mark.live_ai def test_create_embeddings_invalid_apikey(self): """ Test create.embeddings gives a clear @@ -1727,6 +1745,8 @@ def test_create_embeddings_invalid_apikey(self): """ ) + @pytest.mark.integration + @pytest.mark.live_ai def test_create_embeddings_where(self): """ Test generating openai embeddings with where @@ -1752,6 +1772,8 @@ def test_create_embeddings_where(self): ) assert df['embedding'][0] == '' and len(df['embedding'][2]) == 1536 + @pytest.mark.integration + @pytest.mark.live_ai def test_float16_precision(self): """ Test generating half precision embeddings @@ -1787,6 +1809,8 @@ def test_float16_precision(self): round(float(df['embeddings_32'][0][0]), 3) == round(float(df['embeddings_16'][0][0]), 3) ) + @pytest.mark.integration + @pytest.mark.live_ai def test_create_embeddings_empty_dataframe(self): """ Test create.embeddings with an empty dataframe @@ -1804,6 +1828,8 @@ def test_create_embeddings_empty_dataframe(self): ) assert df.empty and list(df.columns) == ['text', 'embedding'] + @pytest.mark.integration + @pytest.mark.live_ai def test_create_embeddings_jina(self): """ Test create.embeddings with Jina provider returns the correct shape. @@ -1888,6 +1914,8 @@ def test_create_embeddings_invalid_provider(self): provider="unsupported-provider", ) + @pytest.mark.integration + @pytest.mark.live_ai def test_create_embeddings_jina_infer_provider_from_url(self): """ Test that when the Jina URL is provided without an explicit provider, @@ -2088,6 +2116,8 @@ def test_create_embeddings_task_warns_for_non_jina(self): task="retrieval.query", ) + @pytest.mark.integration + @pytest.mark.live_ai def test_create_embeddings_jina_return_value(self): """ Test that the Python API returns a list of numpy arrays when using the Jina provider. diff --git a/tests/recipes/wrangles/test_extract.py b/tests/recipes/wrangles/test_extract.py index 4904b526..22478769 100644 --- a/tests/recipes/wrangles/test_extract.py +++ b/tests/recipes/wrangles/test_extract.py @@ -4,6 +4,8 @@ import pandas as pd from unittest.mock import patch +pytestmark = [pytest.mark.integration, pytest.mark.live_wrangleworks] + class TestExtractAIWebSearch: @patch("wrangles.recipe_wrangles.extract._extract.ai") @@ -3664,6 +3666,8 @@ def test_ai_output_format_concatenate_with_char(self): result = wrangles.recipe.run(recipe, dataframe=df) assert result.iloc[0]["tags"] == "wrench | 25mm" + @pytest.mark.integration + @pytest.mark.live_ai def test_ai(self): """ Test openai extract with a single input and output @@ -3701,6 +3705,8 @@ def test_ai(self): ]) assert matches >= 2 + @pytest.mark.integration + @pytest.mark.live_ai def test_ai_only_description(self): """ Test with only a description instead of a JSON schema object @@ -3736,6 +3742,8 @@ def test_ai_only_description(self): ]) assert matches >= 2 + @pytest.mark.integration + @pytest.mark.live_ai def test_ai_formatted_as_a_list(self): """ Test with output as a list instead of an object @@ -3771,6 +3779,8 @@ def test_ai_formatted_as_a_list(self): ]) assert matches >= 2 + @pytest.mark.integration + @pytest.mark.live_ai def test_ai_list_keys_only(self): """ Test with output as a list instead of an object @@ -3805,6 +3815,8 @@ def test_ai_list_keys_only(self): ]) assert matches >= 1 + @pytest.mark.integration + @pytest.mark.live_ai def test_ai_output_key_only(self): """ Test with output as a single string @@ -3838,6 +3850,8 @@ def test_ai_output_key_only(self): ]) assert matches >= 1 + @pytest.mark.integration + @pytest.mark.live_ai def test_ai_multiple_output(self): """ Test AI extract with multiple outputs @@ -3883,6 +3897,8 @@ def test_ai_multiple_output(self): ]) assert matches >= 4 + @pytest.mark.integration + @pytest.mark.live_ai def test_ai_multiple_input(self): """ Test AI extract with multiple inputs @@ -3926,6 +3942,8 @@ def test_ai_multiple_input(self): ]) assert matches >= 2 + @pytest.mark.integration + @pytest.mark.live_ai def test_ai_enum(self): """ Test AI extract with an enum defined @@ -3965,6 +3983,8 @@ def test_ai_enum(self): ]) assert matches >= 2 + @pytest.mark.integration + @pytest.mark.live_ai def test_ai_timeout(self): df = wrangles.recipe.run( """ @@ -3996,6 +4016,8 @@ def test_ai_timeout(self): df['length'][2] == 'Timed Out' ) + @pytest.mark.integration + @pytest.mark.live_ai def test_ai_timeout_multiple_output(self): """ Test AI extract with multiple outputs @@ -4039,6 +4061,8 @@ def test_ai_timeout_multiple_output(self): df['type'][2] == 'Timed Out' ) + @pytest.mark.integration + @pytest.mark.live_ai def test_ai_messages(self): """ Test openai extract with a header level prompt @@ -4078,6 +4102,8 @@ def test_ai_messages(self): ]) assert matches >= 2 + @pytest.mark.integration + @pytest.mark.live_ai def test_ai_array_no_items(self): """ Test openai extract with array but items type is @@ -4109,6 +4135,8 @@ def test_ai_array_no_items(self): ("banana" in df['fruits'][0] or "bananas" in df['fruits'][0]) ) + @pytest.mark.integration + @pytest.mark.live_ai def test_ai_array_item_type_specified(self): """ Test openai extract with array where @@ -4165,6 +4193,8 @@ def test_ai_bad_schema(self): assert "Invalid extract.ai definition" in error.value.args[0] assert "unsupported JSON type" in error.value.args[0] + @pytest.mark.integration + @pytest.mark.live_ai def test_ai_invalid_apikey(self): """ Test that an appropriate error is returned @@ -4192,6 +4222,8 @@ def test_ai_invalid_apikey(self): ) assert "API Key" in error.value.args[0] + @pytest.mark.integration + @pytest.mark.live_ai def test_ai_where(self): """ Test using where with extract.ai @@ -4223,6 +4255,8 @@ def test_ai_where(self): ) ) + @pytest.mark.integration + @pytest.mark.live_ai def test_ai_string_examples(self): """ Test openai extract with examples passed as a string @@ -4261,6 +4295,8 @@ def test_ai_string_examples(self): ]) assert matches >= 2 + @pytest.mark.integration + @pytest.mark.live_ai def test_ai_empty(self): """ Test extract.ai with an empty input @@ -4287,6 +4323,8 @@ def test_ai_empty(self): ) assert df.empty and df.columns.to_list() == ['data', 'length'] + @pytest.mark.integration + @pytest.mark.live_ai def test_ai_o3_mini(self): """ Test openai extract with a single input and output @@ -4324,6 +4362,8 @@ def test_ai_o3_mini(self): ]) assert matches >= 2 + @pytest.mark.integration + @pytest.mark.live_ai def test_ai_reasoning_and_verbosity(self): """ Test extract.ai using the configured default model with @@ -4356,6 +4396,8 @@ def test_ai_reasoning_and_verbosity(self): ]) assert matches >= 1 + @pytest.mark.integration + @pytest.mark.live_ai def test_ai_pre_gpt5_reasoning_and_verbosity_ignored(self, caplog): """ Test extract.ai with a pre-gpt5 model that does not support @@ -4395,6 +4437,8 @@ def test_ai_pre_gpt5_reasoning_and_verbosity_ignored(self, caplog): assert "Ignoring 'reasoning' parameter" in caplog.text assert "Ignoring 'verbosity' parameter" in caplog.text + @pytest.mark.integration + @pytest.mark.live_ai def test_ai_invalid_model_fails_recipe(self): """ Test that a non-existent model fails the recipe after @@ -4421,6 +4465,8 @@ def test_ai_invalid_model_fails_recipe(self): }) ) + @pytest.mark.integration + @pytest.mark.live_ai def test_ai_legacy_chat_completions_endpoint(self): """ Test extract.ai using the legacy Chat Completions endpoint @@ -4451,6 +4497,9 @@ def test_ai_legacy_chat_completions_endpoint(self): ]) assert matches >= 1 + @pytest.mark.integration + @pytest.mark.live_ai + @pytest.mark.live_wrangleworks def test_model_id(self): """ Test using extract.ai with a saved model @@ -4478,6 +4527,9 @@ def test_model_id(self): ('square' in df['Shapes'].values or 'circle' in df['Shapes'].values or 'diamond' in df['Shapes'].values) ) + @pytest.mark.integration + @pytest.mark.live_ai + @pytest.mark.live_wrangleworks def test_model_id_additional_properties(self): """ Test non-explicitly passed properties, i.e. kwargs @@ -4500,6 +4552,9 @@ def test_model_id_additional_properties(self): ) assert 3 in df['Numbers'][0] or 2 in df['Numbers'][0] + @pytest.mark.integration + @pytest.mark.live_ai + @pytest.mark.live_wrangleworks def test_model_id_named_output_single_column(self): """ Test using a predefined model that specifies @@ -4529,6 +4584,9 @@ def test_model_id_named_output_single_column(self): 'Colors' in df['result'][0] ) + @pytest.mark.integration + @pytest.mark.live_ai + @pytest.mark.live_wrangleworks def test_model_id_named_output_multi_column(self): """ Test using a predefined model that specifies @@ -4560,6 +4618,9 @@ def test_model_id_named_output_multi_column(self): ('square' in df['Shapes'].values or 'circle' in df['Shapes'].values) ) + @pytest.mark.integration + @pytest.mark.live_ai + @pytest.mark.live_wrangleworks def test_model_id_object_with_properties(self): """ Test a model_id that contains an output of type object @@ -4583,6 +4644,9 @@ def test_model_id_object_with_properties(self): ) assert 'unit' in df['attributes'][0] and 'value' in df['attributes'][1] + @pytest.mark.integration + @pytest.mark.live_ai + @pytest.mark.live_wrangleworks def test_model_id_array_of_objects(self): """ Test a model_id that contains an output of type array @@ -4605,6 +4669,9 @@ def test_model_id_array_of_objects(self): ) assert 'unit' in df['Attributes'][0][0] and 'value' in df['Attributes'][0][1] + @pytest.mark.integration + @pytest.mark.live_ai + @pytest.mark.live_wrangleworks def test_strict_mode(self): """ Test strict mode with an easy question but a schema that contradicts the correct answer @@ -4711,6 +4778,8 @@ def _run_complex_recipe(self, extra_settings: str): """ return wrangles.recipe.run(recipe, dataframe=self.complex_data.copy()) + @pytest.mark.integration + @pytest.mark.live_ai def test_ai_reasoning_low_vs_high_complex_data(self): """ Test extract.ai on complex, multi-attribute data @@ -4727,6 +4796,8 @@ def test_ai_reasoning_low_vs_high_complex_data(self): assert "Acme" in df["Manufacturer"][0] assert "AFC-6150-WN-A105" in df["PartNumber"][0] + @pytest.mark.integration + @pytest.mark.live_ai def test_ai_verbosity_low_vs_high_complex_data(self): """ Test extract.ai on complex, multi-attribute data diff --git a/tests/recipes/wrangles/test_extract_unit_conversion.py b/tests/recipes/wrangles/test_extract_unit_conversion.py index 199087ba..e4c6bdd2 100644 --- a/tests/recipes/wrangles/test_extract_unit_conversion.py +++ b/tests/recipes/wrangles/test_extract_unit_conversion.py @@ -2,6 +2,8 @@ import wrangles import pytest +pytestmark = [pytest.mark.integration, pytest.mark.live_wrangleworks] + # # misc diff --git a/tests/recipes/wrangles/test_generate.py b/tests/recipes/wrangles/test_generate.py index 71122c96..7b25138f 100644 --- a/tests/recipes/wrangles/test_generate.py +++ b/tests/recipes/wrangles/test_generate.py @@ -4,6 +4,8 @@ import wrangles import wrangles.generate +pytestmark = [pytest.mark.integration, pytest.mark.live_ai] + @pytest.mark.skipif("OPENAI_API_KEY" not in os.environ, reason="needs live OpenAI access") def test_generate_ai_recipe_without_web_search_real_call(): diff --git a/tests/recipes/wrangles/test_main.py b/tests/recipes/wrangles/test_main.py index 10ce1f70..6018cc0f 100644 --- a/tests/recipes/wrangles/test_main.py +++ b/tests/recipes/wrangles/test_main.py @@ -28,6 +28,8 @@ def assert_lookup_equal(a, b, score_tol=0.01): assert a == b +@pytest.mark.integration +@pytest.mark.live_wrangleworks class TestClassify: """ Test classify @@ -907,12 +909,13 @@ def test_log_escaped_wildcard(self, caplog): wrangles.recipe.run(recipe, dataframe=data) assert caplog.messages[-1] == ': Dataframe ::\n\n Col*\n0 WrangleWorks!\n' - def test_log_write(self): + def test_log_write(self, tmp_path): """ Test using a connector as part of a log """ + file_name = (tmp_path / "temp.csv").as_posix() wrangles.recipe.run( - """ + f""" read: - test: rows: 5 @@ -923,24 +926,25 @@ def test_log_write(self): - log: write: - file: - name: tests/temp/temp.csv + name: {file_name} """ ) df = wrangles.recipe.run( - """ + f""" read: - file: - name: tests/temp/temp.csv + name: {file_name} """ ) assert len(df) == 5 and df['header'][0] == 'value' - - def test_log_write_multiple_files(self): + def test_log_write_multiple_files(self, tmp_path): """ Test writing multiple files from a log """ + file_name = (tmp_path / "temp.csv").as_posix() + file_name2 = (tmp_path / "temp2.csv").as_posix() wrangles.recipe.run( - """ + f""" read: - test: rows: 5 @@ -951,27 +955,26 @@ def test_log_write_multiple_files(self): - log: write: - file: - name: tests/temp/temp.csv + name: {file_name} - file: - name: tests/temp/temp2.csv + name: {file_name2} """ ) df = wrangles.recipe.run( - """ + f""" read: - file: - name: tests/temp/temp.csv + name: {file_name} """ ) df2 = wrangles.recipe.run( - """ + f""" read: - file: - name: tests/temp/temp2.csv + name: {file_name2} """ ) assert len(df) == 5 and df['header'][0] == 'value' and len(df2) == 5 and df2['header'][0] == 'value' - def test_log_length(self, caplog): """ Test default log @@ -1164,10 +1167,12 @@ def test_log_columns_variables(self, caplog): wrangles.recipe.run(recipe, dataframe=data, variables={'var': 'Col1'}) assert caplog.messages[-1] == ': Dataframe ::\n\n Col1\n0 Chicken\n' - def test_log_write_multiple_variable_files(self): + def test_log_write_multiple_variable_files(self, tmp_path): """ Test writing multiple files using variables """ + file_name = (tmp_path / "temp.csv").as_posix() + file_name2 = (tmp_path / "temp2.csv").as_posix() wrangles.recipe.run( """ read: @@ -1185,28 +1190,27 @@ def test_log_write_multiple_variable_files(self): name: ${var2} """, variables = { - 'var1': 'tests/temp/temp.csv', - 'var2': 'tests/temp/temp2.csv', + 'var1': file_name, + 'var2': file_name2, } ) df = wrangles.recipe.run( - """ + f""" read: - file: - name: tests/temp/temp.csv + name: {file_name} """ ) df2 = wrangles.recipe.run( - """ + f""" read: - file: - name: tests/temp/temp2.csv + name: {file_name2} """ ) assert len(df) == 5 and df['header'][0] == 'value' and len(df2) == 5 and df2['header'][0] == 'value' - class TestRemoveWords: """ Test remove_words @@ -2951,6 +2955,8 @@ def test_similarity_empty(self): assert df.empty and df.columns.to_list() == ['col1', 'col2', 'Cos Sim'] +@pytest.mark.integration +@pytest.mark.live_wrangleworks class TestStandardize: """ Test standardize @@ -3574,6 +3580,8 @@ def test_replace_dict(self): assert df.iloc[1]['Not Dictionaries'] == {'So': 'is this'} and isinstance(df.iloc[0]['Not Dictionaries'], dict) == True +@pytest.mark.integration +@pytest.mark.live_wrangleworks class TestTranslate: """ Test translate @@ -6301,6 +6309,8 @@ def fail_on_2nd_batch(df): +@pytest.mark.integration +@pytest.mark.live_wrangleworks class TestLookup: """ Test lookup wrangle @@ -7571,6 +7581,8 @@ def test_error(self): """ ) + @pytest.mark.integration + @pytest.mark.live_ai def test_extract_ai(self): """ Test using extract.ai within a matrix diff --git a/tests/recipes/wrangles/test_search.py b/tests/recipes/wrangles/test_search.py index dc2c6988..dd13e4ed 100644 --- a/tests/recipes/wrangles/test_search.py +++ b/tests/recipes/wrangles/test_search.py @@ -1,5 +1,8 @@ import wrangles import pandas as pd +import pytest + +pytestmark = [pytest.mark.integration, pytest.mark.live_ai] class TestFindLinks: @@ -534,4 +537,4 @@ def test_link_content(self): # # Error should be returned as part of results rather than raising # assert isinstance(results, list) # assert len(results) > 0 -# assert 'error' in results[0] \ No newline at end of file +# assert 'error' in results[0] diff --git a/tests/test_openai_extract_ai.py b/tests/test_openai_extract_ai.py index 3240e576..df780b24 100644 --- a/tests/test_openai_extract_ai.py +++ b/tests/test_openai_extract_ai.py @@ -8,6 +8,7 @@ from wrangles import ai_config from wrangles import ai_cache from wrangles import recipe +from wrangles import openai_responses @pytest.fixture(autouse=True) @@ -28,25 +29,15 @@ def json(self): return self._body -def test_extract_ai_uses_responses_structured_outputs(monkeypatch, caplog): +def test_extract_ai_uses_responses_structured_outputs( + monkeypatch, caplog, openai_success_response +): calls = [] - body = { - "output": [ - { - "type": "message", - "content": [ - { - "type": "output_text", - "text": '{"length":"25mm"}', - } - ], - } - ] - } + response = openai_success_response({"length": "25mm"}) def post(**kwargs): calls.append(kwargs) - return _Response(body) + return response monkeypatch.setattr(extract._openai_responses._requests, "post", post) @@ -87,18 +78,14 @@ def post(**kwargs): @pytest.mark.parametrize("store", [None, False, True]) -def test_extract_ai_recipe_preserves_response_storage_override(monkeypatch, store): +def test_extract_ai_recipe_preserves_response_storage_override( + monkeypatch, store, openai_success_response +): calls = [] - body = { - "output": [{ - "type": "message", - "content": [{"type": "output_text", "text": '{"length":"25mm"}'}], - }] - } monkeypatch.setattr( extract._openai_responses._requests, "post", - lambda **kwargs: calls.append(kwargs) or _Response(body), + lambda **kwargs: calls.append(kwargs) or openai_success_response({"length": "25mm"}), ) settings = { "input": "Description", @@ -122,7 +109,7 @@ def test_extract_ai_recipe_preserves_response_storage_override(monkeypatch, stor @pytest.mark.parametrize("configured_store", [True, False, None]) def test_extract_ai_storage_configuration_and_override_have_separate_caches( - monkeypatch, tmp_path, configured_store + monkeypatch, tmp_path, configured_store, openai_success_response ): config = ai_config.load() if configured_store is None: @@ -135,16 +122,10 @@ def test_extract_ai_storage_configuration_and_override_have_separate_caches( ai_config.clear_cache() calls = [] - body = { - "output": [{ - "type": "message", - "content": [{"type": "output_text", "text": '{"length":"25mm"}'}], - }] - } monkeypatch.setattr( extract._openai_responses._requests, "post", - lambda **kwargs: calls.append(kwargs) or _Response(body), + lambda **kwargs: calls.append(kwargs) or openai_success_response({"length": "25mm"}), ) arguments = { "input": "wrench 25mm", @@ -238,22 +219,14 @@ def test_extract_ai_web_search_returns_metadata_sources_and_caches_them(monkeypa assert "authorized evidence in addition to DATA" in payload["instructions"] -def test_extract_ai_web_search_preserves_expert_tool_settings(monkeypatch): +def test_extract_ai_web_search_preserves_expert_tool_settings( + monkeypatch, openai_success_response +): calls = [] - body = { - "output": [{ - "type": "message", - "content": [{ - "type": "output_text", - "text": '{"manufacturer":"Acme"}', - "annotations": [], - }], - }] - } monkeypatch.setattr( extract._openai_responses._requests, "post", - lambda **kwargs: calls.append(kwargs) or _Response(body), + lambda **kwargs: calls.append(kwargs) or openai_success_response({"manufacturer": "Acme"}), ) result = extract.ai( @@ -317,15 +290,13 @@ def test_extract_ai_web_search_failure_still_returns_empty_sources(monkeypatch): assert result["output"].startswith("Invalid structured response") -def test_extract_ai_malformed_response_json_returns_structured_failure(monkeypatch): - class MalformedResponse(_Response): - def json(self): - raise json.JSONDecodeError("Malformed response body", "", 0) - +def test_extract_ai_malformed_response_json_returns_structured_failure( + monkeypatch, malformed_json_response_factory +): monkeypatch.setattr( extract._openai_responses._requests, "post", - lambda **kwargs: MalformedResponse(None), + lambda **kwargs: malformed_json_response_factory(), ) result = extract.ai( @@ -341,11 +312,9 @@ def json(self): assert result["web_search_sources"] == [] -def test_extract_ai_malformed_retry_does_not_reuse_prior_response_sources(monkeypatch): - class MalformedResponse(_Response): - def json(self): - raise json.JSONDecodeError("Malformed response body", "", 0) - +def test_extract_ai_malformed_retry_does_not_reuse_prior_response_sources( + monkeypatch, malformed_json_response_factory +): responses = [ _Response({ "output": [ @@ -362,7 +331,7 @@ def json(self): }, ] }), - MalformedResponse(None), + malformed_json_response_factory(), ] monkeypatch.setattr( extract._openai_responses._requests, @@ -384,6 +353,134 @@ def json(self): assert result["web_search_sources"] == [] assert responses == [] +@pytest.mark.parametrize( + ("body", "message"), + [ + pytest.param( + {"output": [{"type": "message", "content": [{"type": "refusal", "refusal": "Cannot comply"}]}]}, + "Cannot comply", + id="refusal_content", + ), + pytest.param( + {"status": "incomplete", "incomplete_details": {"reason": "max_output_tokens"}}, + "The model response was incomplete: max_output_tokens.", + id="incomplete_with_reason", + ), + pytest.param( + {"error": {"message": "Bad request"}}, + "Bad request", + id="error_object", + ), + ], +) +def test_openai_responses_extract_response_text_reports_terminal_model_states(body, message): + with pytest.raises(ValueError, match=message): + openai_responses.extract_response_text(body) + + +def test_openai_responses_call_structured_formats_input_and_preserves_payload( + monkeypatch, openai_success_response +): + calls = [] + payload = { + "model": "gpt-5-mini", + "instructions": "Extract dimensions.", + "text": { + "format": { + "schema": { + "type": "object", + "properties": {"length": {"type": "string"}}, + } + } + }, + } + + def post(**kwargs): + calls.append(kwargs) + return openai_success_response({"length": "25mm"}) + + monkeypatch.setattr(openai_responses._requests, "post", post) + + result = openai_responses.call_structured( + data={"description": "wrench 25mm"}, + api_key="test-key", + payload=payload, + url="https://api.openai.test/v1/responses", + timeout=7, + retries=0, + required_fields=["length"], + ) + + assert result == {"length": "25mm"} + assert "input" not in payload + assert calls == [{ + "url": "https://api.openai.test/v1/responses", + "headers": { + "Authorization": "Bearer test-key", + "Content-Type": "application/json", + }, + "json": { + **payload, + "input": [{ + "role": "user", + "content": 'DATA:\n{\n "description": "wrench 25mm"\n}', + }], + }, + "timeout": 7, + }] + + +def test_openai_responses_call_structured_retries_invalid_json_without_stale_sources( + monkeypatch, openai_success_response, json_response_factory +): + responses = [ + json_response_factory({ + "output": [ + { + "type": "web_search_call", + "action": { + "sources": [{"url": "https://example.com/stale"}], + }, + }, + { + "type": "message", + "content": [{"type": "output_text", "text": "not json"}], + }, + ] + }), + openai_success_response({"manufacturer": "Acme"}), + ] + sleeps = [] + payload = { + "model": "gpt-5-mini", + "tools": [{"type": "web_search"}], + "text": { + "format": { + "schema": { + "type": "object", + "properties": {"manufacturer": {"type": "string"}}, + } + } + }, + } + + monkeypatch.setattr(openai_responses._requests, "post", lambda **kwargs: responses.pop(0)) + monkeypatch.setattr(openai_responses._time, "sleep", sleeps.append) + monkeypatch.setattr(openai_responses._random, "uniform", lambda *args: 0) + + result = openai_responses.call_structured( + data="Acme part", + api_key="test-key", + payload=payload, + url="https://api.openai.test/v1/responses", + timeout=7, + retries=1, + required_fields=["manufacturer"], + ) + + assert result == {"manufacturer": "Acme", "web_search_sources": []} + assert sleeps == [1] + assert responses == [] def test_extract_ai_web_search_validates_protocol_and_reserved_output(): with pytest.raises(ValueError, match="only with protocol='responses'"): diff --git a/tests/test_pytest_marker_config.py b/tests/test_pytest_marker_config.py new file mode 100644 index 00000000..6d4ded7b --- /dev/null +++ b/tests/test_pytest_marker_config.py @@ -0,0 +1,5 @@ +from scripts import check_pytest_markers + + +def test_pytest_marker_configuration_is_consistent(): + assert check_pytest_markers.validate() == [] diff --git a/tests/test_wrangles.py b/tests/test_wrangles.py index 91617523..248ca12c 100644 --- a/tests/test_wrangles.py +++ b/tests/test_wrangles.py @@ -5,6 +5,8 @@ import os import logging +pytestmark = [pytest.mark.integration, pytest.mark.live_wrangleworks, pytest.mark.live_ai] + # Classify def test_classify(): @@ -932,4 +934,4 @@ def test_compare_overlap_exact_match_custom(): Test compare.overlap with exact_match parameter """ result = wrangles.compare.overlap([['test', 'test']], exact_match='MATCH') - assert result == ['MATCH'] \ No newline at end of file + assert result == ['MATCH']