Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ dependencies = [
"httpx>=0.26.0",

# qsv runner (timeouts, typed errors) for the onboarding profiling passes
"qsv-client>=0.3.0",
"qsv-client>=0.4.0",

# Vector Search (Pinecone for CKAN semantic search)
# Note: Use the new `pinecone` package (v5+), NOT the legacy
Expand Down
7 changes: 3 additions & 4 deletions src/data_concierge/data_layer/qsv_profiling.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,6 @@
QsvError,
QsvNotFound,
QsvTimeout,
QsvVersionError,
)

from data_concierge.data_layer.storage import storage
Expand Down Expand Up @@ -80,10 +79,10 @@ async def download_csv(url: str, dest: Path) -> int:
"count": 120.0,
}

# A failed run, a missing or too-old binary, or output that does not parse. QsvNotFound and
# QsvVersionError are not QsvError subclasses, and the client raises QsvNotFound from its
# A failed run, a missing or too-old binary, or output that does not parse. A too-old qsv
# (QsvVersionError) is a QsvError; QsvNotFound is not, and the client raises it from its
# constructor.
_QSV_FAILURES = (QsvError, QsvNotFound, QsvVersionError, ValueError)
_QSV_FAILURES = (QsvError, QsvNotFound, ValueError)

# Binaries whose missing describegpt has already been reported, so a portal run warns once.
_describegpt_checked: set[str] = set()
Expand Down
57 changes: 43 additions & 14 deletions tests/unit/test_qsv_profiling.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,34 @@
posix_only = pytest.mark.skipif(sys.platform == "win32", reason="fake qsv is a POSIX script")


def _fake_qsv(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, body: str) -> Path:
"""Put a Python script named ``qsv`` first on PATH, ahead of any real qsv."""
QSV_CAPABILITIES = {
"binary": "qsv",
"version": "25.0.0",
"commands": ["count", "describegpt", "frequency", "stats"],
}


def _fake_qsv(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
body: str,
capabilities: dict | None = QSV_CAPABILITIES,
) -> Path:
"""Put a Python script named ``qsv`` first on PATH, ahead of any real qsv.

It answers ``--capabilities`` with ``capabilities`` (qsv-client checks that before the
first run); pass ``None`` to leave it to ``body``, like a qsv older than 25.0.0.
"""
fake = tmp_path / "bin" / "qsv"
fake.parent.mkdir(exist_ok=True)
fake.write_text(f"#!{sys.executable}\nimport json, os, sys, time\n{body}")
caps = (
""
if capabilities is None
else f"if sys.argv[1:] == ['--capabilities']:\n"
f" print({json.dumps(capabilities)!r})\n"
f" sys.exit(0)\n"
)
fake.write_text(f"#!{sys.executable}\nimport json, os, sys, time\n{caps}{body}")
fake.chmod(0o755)
monkeypatch.setenv("PATH", f"{fake.parent}:{os.environ['PATH']}")
monkeypatch.delenv("QSV_BIN", raising=False)
Expand Down Expand Up @@ -160,22 +183,28 @@ async def test_describegpt_failure_on_a_binary_without_it_says_so(tmp_path, monk
_fake_qsv(
tmp_path,
monkeypatch,
"arg = sys.argv[1]\n"
"if arg == '--version':\n"
" print('qsvlite 24.0.0-standard-16-16; (aarch64-apple-darwin compiled with Rust 1.99)')\n"
"elif arg == '--list':\n"
" print('Installed commands (2):')\n"
" print(' count Count records')\n"
" print(' stats Infer data types')\n"
"else:\n"
" sys.stderr.write('Invalid arguments.\\n')\n"
" sys.exit(1 if arg == '--capabilities' else 2)\n",
"sys.stderr.write('Invalid arguments.\\n')\nsys.exit(2)\n",
capabilities={"binary": "qsvlite", "version": "25.0.0", "commands": ["count", "stats"]},
)
out = tmp_path / "qsv_dict.json"

assert await run_qsv_describegpt(tmp_path / "data.csv", KEY, out) is None

printed = capsys.readouterr().out
assert "has no describegpt" in printed
assert "qsvlite 24.0.0" in printed
assert "qsvlite 25.0.0" in printed
assert not out.exists()


@posix_only
async def test_too_old_qsv_returns_none_instead_of_raising(tmp_path, monkeypatch, capsys):
# qsv before 25.0.0 rejects --capabilities, and qsv-client refuses to run it.
_fake_qsv(
tmp_path,
monkeypatch,
"sys.stderr.write(\"Unknown flag: '--capabilities'\\n\")\nsys.exit(1)\n",
capabilities=None,
)

assert await run_qsv_count(tmp_path / "data.csv") is None
assert "QsvVersionError" in capsys.readouterr().out
8 changes: 4 additions & 4 deletions uv.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading