From 314852233ced6010b1abff715a4528553d05e8c4 Mon Sep 17 00:00:00 2001 From: Joel Natividad <1980690+jqnatividad@users.noreply.github.com> Date: Thu, 8 Oct 2026 21:47:48 -0400 Subject: [PATCH] Run every qsv call through qsv-client No qsv run during onboarding had a timeout, so a stalled describegpt (an LLM that never answers) or a stats --everything on a huge CSV blocked the job forever. qsv-client kills a run's whole process group once it passes its timeout. The defaults are describegpt 900s, stats 900s, frequency 600s and count 120s, and VERIKAN_QSV__TIMEOUT overrides each one. The run_qsv_* helpers keep their contract: they return None on any failure, now including a missing binary or a timeout, and they log the error type and kind instead of truncated stderr. stats and frequency now stream to a sibling .part file that replaces the output only after a run succeeds, so a failed run leaves the previous qsv_stats.csv in place instead of a truncated one. The bytes on disk are still exactly qsv's stdout. qsv_dict.json is still re-dumped with indent=2, because it is published and hashed. Binary discovery honours QSV_BIN, in the onboarding health warning too. Discovery can fall back to qsvlite/qsvdp, which have no describegpt, so after a describegpt failure the binary's command list is checked once and a missing describegpt is reported. A successful run pays nothing for that check. The two inline `qsv count` blocks, which swallowed every exception, become run_qsv_count. The Dockerfile moves from qsv 22.0.1 to 24.0.0. Co-Authored-By: Claude Opus 5.5 --- Dockerfile | 2 +- pyproject.toml | 3 + scripts/onboard_ckan.py | 22 +- scripts/onboard_dcat.py | 15 +- .../data_layer/qsv_profiling.py | 194 +++++++++++++----- src/data_concierge/gateway/onboarding_jobs.py | 7 +- tests/unit/test_onboarding_jobs.py | 14 +- tests/unit/test_qsv_profiling.py | 161 ++++++++++++++- uv.lock | 11 + 9 files changed, 330 insertions(+), 99 deletions(-) diff --git a/Dockerfile b/Dockerfile index b6816f4..d507474 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ # TARGETARCH is supplied automatically by BuildKit. The release ships one zip # per architecture holding several build variants; only the full "qsv" binary # has describegpt, so that is the one extracted (qsvlite omits it). -ARG QSV_VERSION=22.0.1 +ARG QSV_VERSION=24.0.0 ARG TARGETARCH=amd64 RUN set -eux; \ case "${TARGETARCH}" in \ diff --git a/pyproject.toml b/pyproject.toml index d25df75..7c70180 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,9 @@ dependencies = [ # HTTP Client "httpx>=0.26.0", + # qsv runner (timeouts, typed errors) for the onboarding profiling passes + "qsv-client>=0.1.0", + # Vector Search (Pinecone for CKAN semantic search) # Note: Use the new `pinecone` package (v5+), NOT the legacy # `pinecone-client` — the code uses the new Pinecone class and diff --git a/scripts/onboard_ckan.py b/scripts/onboard_ckan.py index 7417594..f1f9200 100644 --- a/scripts/onboard_ckan.py +++ b/scripts/onboard_ckan.py @@ -14,11 +14,12 @@ import json import os import re -import subprocess import sys from io import StringIO from pathlib import Path +from qsv_client import QsvNotFound, find_qsv + sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src")) csv.field_size_limit(10 * 1024 * 1024) # 10 MB — qsv stats can produce large fields @@ -32,6 +33,7 @@ build_index, download_csv, merge_columns, + run_qsv_count, run_qsv_describegpt, run_qsv_frequency, run_qsv_stats, @@ -211,17 +213,9 @@ async def process_resource( # Step 7: Count rows from CSV if csv_path.exists(): - try: - proc = await asyncio.create_subprocess_exec( - "qsv", "count", str(csv_path), - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - stdout, _ = await proc.communicate() - if proc.returncode == 0: - result["row_count"] = int(stdout.decode().strip()) - except Exception: - pass + row_count = await run_qsv_count(csv_path) + if row_count is not None: + result["row_count"] = row_count # Save meta.json resource_dir.mkdir(parents=True, exist_ok=True) @@ -372,8 +366,8 @@ async def main() -> None: # Verify qsv is installed try: - subprocess.run(["qsv", "--version"], capture_output=True, check=True) - except (FileNotFoundError, subprocess.CalledProcessError): + find_qsv() + except QsvNotFound: print("Error: qsv is not installed or not in PATH.") print("Install from: https://github.com/dathere/qsv") sys.exit(1) diff --git a/scripts/onboard_dcat.py b/scripts/onboard_dcat.py index d1f54aa..4b84d0a 100755 --- a/scripts/onboard_dcat.py +++ b/scripts/onboard_dcat.py @@ -48,6 +48,7 @@ _extract_qsv_tags, _now_iso, merge_columns, + run_qsv_count, run_qsv_describegpt, run_qsv_frequency, run_qsv_stats, @@ -218,17 +219,9 @@ async def process_dataset( # Step 5: exact row count if csv_path.exists(): - try: - proc = await asyncio.create_subprocess_exec( - "qsv", "count", str(csv_path), - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - stdout, _ = await proc.communicate() - if proc.returncode == 0: - result["row_count"] = int(stdout.decode().strip()) - except Exception: - pass + row_count = await run_qsv_count(csv_path) + if row_count is not None: + result["row_count"] = row_count resource_dir.mkdir(parents=True, exist_ok=True) meta_path.write_text(json.dumps(result, indent=2)) diff --git a/src/data_concierge/data_layer/qsv_profiling.py b/src/data_concierge/data_layer/qsv_profiling.py index 04e4156..7d5a55f 100644 --- a/src/data_concierge/data_layer/qsv_profiling.py +++ b/src/data_concierge/data_layer/qsv_profiling.py @@ -19,6 +19,11 @@ ``count`` Exact row count. +Every pass runs through `qsv-client `_, which +kills a run that exceeds its timeout (see ``_DEFAULT_TIMEOUTS``; each one can be +overridden with ``VERIKAN_QSV__TIMEOUT``). A failed, timed-out, or missing +qsv makes the pass return ``None``, and the onboarding scripts carry on without it. + :func:`merge_columns` folds all of those, plus the portal's own field metadata when it has any, into the single column list that ``data_layer.onboard_index`` searches and that @@ -27,16 +32,21 @@ from __future__ import annotations -import asyncio import csv import json import os from datetime import UTC, datetime -from io import StringIO from pathlib import Path from typing import Any import httpx +from qsv_client import ( + AsyncQsv, + QsvError, + QsvNotFound, + QsvTimeout, + QsvVersionError, +) from data_concierge.data_layer.storage import storage @@ -61,6 +71,71 @@ async def download_csv(url: str, dest: Path) -> int: return size +# Default per-pass timeouts in seconds. ``VERIKAN_QSV__TIMEOUT`` overrides one, +# e.g. ``VERIKAN_QSV_STATS_TIMEOUT=1800`` for a portal with very large CSVs. +_DEFAULT_TIMEOUTS: dict[str, float] = { + "describegpt": 900.0, # several LLM round trips + "stats": 900.0, + "frequency": 600.0, + "count": 120.0, +} + +# A failed run, a missing or too-old binary, or output that does not parse. QsvNotFound and +# QsvVersionError are not QsvError subclasses, and the client raises QsvNotFound from its +# constructor. +_QSV_FAILURES = (QsvError, QsvNotFound, QsvVersionError, ValueError) + +# Binaries whose missing describegpt has already been reported, so a portal run warns once. +_describegpt_checked: set[str] = set() + + +def _timeout(command: str) -> float: + var = f"VERIKAN_QSV_{command.upper()}_TIMEOUT" + raw = os.environ.get(var) + if raw: + try: + return float(raw) + except ValueError: + print(f" ignoring {var}={raw!r}: not a number") + return _DEFAULT_TIMEOUTS[command] + + +def _qsv(api_key: str | None = None) -> AsyncQsv: + """The qsv client every profiling pass runs through. + + The binary is ``$QSV_BIN`` if set, else the first of qsv, qsvmcp, qsvdp, qsvlite on + ``PATH``. Each run is killed (with its whole process group) once it passes its timeout. + """ + return AsyncQsv(llm_api_key=api_key) + + +def _report_failure(command: str, exc: Exception) -> None: + kind = getattr(exc, "kind", None) + message = getattr(exc, "message", None) or str(exc) + label = f"{type(exc).__name__}, {kind}" if kind else type(exc).__name__ + print(f" qsv {command} failed ({label}): {message[:500]}") + + +async def _explain_missing_describegpt(qsv: AsyncQsv) -> None: + """After a describegpt failure, say so if the binary has no describegpt at all. + + qsvlite and qsvdp lack it, and binary discovery falls back to them. Checked lazily so a + successful run pays nothing for the probe. + """ + if qsv.binary in _describegpt_checked: + return + _describegpt_checked.add(qsv.binary) + try: + caps = await qsv.capabilities() + except _QSV_FAILURES: + return + if caps.commands and not caps.has_command("describegpt"): + print( + f" {qsv.binary} ({caps.binary} {caps.version}) has no describegpt command; " + "install the full qsv binary or point QSV_BIN at it" + ) + + async def run_qsv_describegpt( csv_path: Path, api_key: str, @@ -71,58 +146,66 @@ async def run_qsv_describegpt( The API key goes to qsv in ``QSV_LLM_APIKEY``, not ``--api-key``: describegpt copies its command line into the descriptions it generates, and argv is also visible to other local users in ``ps``. + + The output is re-serialised with ``json.dump(indent=2)`` rather than streamed to + ``output_path`` as qsv wrote it: ``qsv_dict.json`` is published and hashed, so its + bytes must not change with qsv's own formatting. """ - cmd = [ - "qsv", "describegpt", str(csv_path), - "--all", - "--format", "json", - "--base-url", "https://openrouter.ai/api/v1", - "--model", "google/gemini-2.5-flash-lite", - ] - - proc = await asyncio.create_subprocess_exec( - *cmd, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - env={**os.environ, "QSV_LLM_APIKEY": api_key}, - ) - stdout, stderr = await proc.communicate() - - if proc.returncode != 0: - print(f" qsv describegpt failed (exit {proc.returncode}): {stderr.decode()[:500]}") + qsv: AsyncQsv | None = None + try: + qsv = _qsv(api_key) + data = await qsv.describegpt( + csv_path, + "--all", + "--format", "json", + "--base-url", "https://openrouter.ai/api/v1", + "--model", "google/gemini-2.5-flash-lite", + timeout=_timeout("describegpt"), + ) + except _QSV_FAILURES as e: + _report_failure("describegpt", e) + if qsv is not None and isinstance(e, QsvError) and not isinstance(e, QsvTimeout): + await _explain_missing_describegpt(qsv) return None + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w") as f: + json.dump(data, f, indent=2) + return data + + +async def _run_qsv_csv( + command: str, csv_path: Path, output_path: Path, *args: str +) -> list[dict[str, str]] | None: + """Stream ``qsv ``'s CSV output to ``output_path`` and return its rows. + + qsv writes to a sibling ``.part`` file that replaces ``output_path`` only once the run + succeeds and parses, so a failed or killed run leaves the previous output in place + rather than a truncated file. + """ + part = output_path.with_name(output_path.name + ".part") try: - data = json.loads(stdout.decode()) - output_path.parent.mkdir(parents=True, exist_ok=True) - with open(output_path, "w") as f: - json.dump(data, f, indent=2) - return data - except json.JSONDecodeError as e: - print(f" qsv output not valid JSON: {e}") + res = await _qsv().run( + command, csv_path, *args, stdout_path=part, timeout=_timeout(command) + ) + rows = res.csv_rows() + os.replace(part, output_path) + return rows + except _QSV_FAILURES as e: + _report_failure(command, e) return None + finally: + part.unlink(missing_ok=True) async def run_qsv_stats(csv_path: Path, output_path: Path) -> dict[str, dict] | None: """Run qsv stats and return per-column stats keyed by column name.""" - proc = await asyncio.create_subprocess_exec( - "qsv", "stats", str(csv_path), "--everything", - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - stdout, stderr = await proc.communicate() - - if proc.returncode != 0: - print(f" qsv stats failed: {stderr.decode()[:300]}") + rows = await _run_qsv_csv("stats", csv_path, output_path, "--everything") + if rows is None: return None - raw = stdout.decode() - output_path.parent.mkdir(parents=True, exist_ok=True) - output_path.write_text(raw) - - reader = csv.DictReader(StringIO(raw)) stats_by_col: dict[str, dict] = {} - for row in reader: + for row in rows: col_name = row.get("field", "") if col_name: stats_by_col[col_name] = {k: v for k, v in row.items() if k != "field"} @@ -131,24 +214,12 @@ async def run_qsv_stats(csv_path: Path, output_path: Path) -> dict[str, dict] | async def run_qsv_frequency(csv_path: Path, output_path: Path) -> dict[str, list[dict]] | None: """Run qsv frequency and return top values per column.""" - proc = await asyncio.create_subprocess_exec( - "qsv", "frequency", str(csv_path), "--limit", "20", - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - stdout, stderr = await proc.communicate() - - if proc.returncode != 0: - print(f" qsv frequency failed: {stderr.decode()[:300]}") + rows = await _run_qsv_csv("frequency", csv_path, output_path, "--limit", "20") + if rows is None: return None - raw = stdout.decode() - output_path.parent.mkdir(parents=True, exist_ok=True) - output_path.write_text(raw) - - reader = csv.DictReader(StringIO(raw)) freq_by_col: dict[str, list[dict]] = {} - for row in reader: + for row in rows: col_name = row.get("field", "") if col_name: freq_by_col.setdefault(col_name, []).append({ @@ -158,6 +229,15 @@ async def run_qsv_frequency(csv_path: Path, output_path: Path) -> dict[str, list return freq_by_col +async def run_qsv_count(csv_path: Path) -> int | None: + """Exact row count of ``csv_path``, or ``None`` if qsv could not count it.""" + try: + return await _qsv().count(csv_path, timeout=_timeout("count")) + except _QSV_FAILURES as e: + _report_failure("count", e) + return None + + def _extract_qsv_fields(qsv_data: dict) -> list[dict]: """Extract field list from qsv describegpt JSON, handling both key casings.""" for key in ("Dictionary", "dictionary"): diff --git a/src/data_concierge/gateway/onboarding_jobs.py b/src/data_concierge/gateway/onboarding_jobs.py index edba054..9ded6da 100644 --- a/src/data_concierge/gateway/onboarding_jobs.py +++ b/src/data_concierge/gateway/onboarding_jobs.py @@ -58,7 +58,6 @@ import json import os import re -import shutil import sys import tempfile import threading @@ -69,6 +68,8 @@ from pathlib import Path from typing import Any +from qsv_client import QsvNotFound, find_qsv + from data_concierge.core.logging import get_logger from data_concierge.data_layer.storage import storage @@ -191,7 +192,9 @@ def runtime_warnings() -> list[str]: "The onboarding scripts are not present in this deployment, so jobs " "cannot be started. The container image must copy scripts/ to /app/scripts." ) - if shutil.which("qsv") is None: + try: + find_qsv() + except QsvNotFound: warnings.append( "qsv is not installed on this host. Downloads will succeed but every " "profiling pass (stats, frequency, describegpt, count) will be skipped, " diff --git a/tests/unit/test_onboarding_jobs.py b/tests/unit/test_onboarding_jobs.py index bf48624..9cdd6d5 100644 --- a/tests/unit/test_onboarding_jobs.py +++ b/tests/unit/test_onboarding_jobs.py @@ -267,9 +267,17 @@ def test_a_job_left_running_by_a_restart_is_reported_as_interrupted( class TestRuntimeWarnings: - def test_missing_qsv_is_reported(self, monkeypatch): - monkeypatch.setattr(oj.shutil, "which", lambda name: None) - assert any("qsv" in w for w in oj.runtime_warnings()) + def test_missing_qsv_is_reported(self, monkeypatch, tmp_path): + monkeypatch.setenv("QSV_BIN", str(tmp_path / "no-such-qsv")) + assert any("qsv is not installed" in w for w in oj.runtime_warnings()) + + def test_qsv_bin_outside_path_counts_as_installed(self, monkeypatch, tmp_path): + qsv = tmp_path / "opt" / "qsv" + qsv.parent.mkdir() + qsv.write_text("") + monkeypatch.setenv("PATH", str(tmp_path / "empty")) + monkeypatch.setenv("QSV_BIN", str(qsv)) + assert not any("qsv is not installed" in w for w in oj.runtime_warnings()) def test_cloud_run_lifetime_is_reported(self, monkeypatch): monkeypatch.setenv("K_SERVICE", "data-concierge") diff --git a/tests/unit/test_qsv_profiling.py b/tests/unit/test_qsv_profiling.py index 9e7bc4d..e2eb5e2 100644 --- a/tests/unit/test_qsv_profiling.py +++ b/tests/unit/test_qsv_profiling.py @@ -5,38 +5,177 @@ import json import os import sys +import time +from pathlib import Path import pytest -from data_concierge.data_layer.qsv_profiling import run_qsv_describegpt +from data_concierge.data_layer import qsv_profiling +from data_concierge.data_layer.qsv_profiling import ( + run_qsv_count, + run_qsv_describegpt, + run_qsv_frequency, + run_qsv_stats, +) KEY = "sk-or-v1-0123456789abcdef" +posix_only = pytest.mark.skipif(sys.platform == "win32", reason="fake qsv is a POSIX script") -@pytest.mark.skipif(sys.platform == "win32", reason="fake qsv is a POSIX shell script") + +def _fake_qsv(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, body: str) -> Path: + """Put a Python script named ``qsv`` first on PATH, ahead of any real qsv.""" + fake = tmp_path / "bin" / "qsv" + fake.parent.mkdir(exist_ok=True) + fake.write_text(f"#!{sys.executable}\nimport json, os, sys, time\n{body}") + fake.chmod(0o755) + monkeypatch.setenv("PATH", f"{fake.parent}:{os.environ['PATH']}") + monkeypatch.delenv("QSV_BIN", raising=False) + monkeypatch.setattr(qsv_profiling, "_describegpt_checked", set()) + return fake + + +@posix_only async def test_describegpt_gets_the_api_key_from_the_environment_not_argv(tmp_path, monkeypatch): # describegpt copies its argv into the descriptions it writes, so the key must not be in it. record = tmp_path / "record.json" - fake = tmp_path / "bin" / "qsv" - fake.parent.mkdir() - fake.write_text( - f"#!{sys.executable}\n" - "import json, os, sys\n" + _fake_qsv( + tmp_path, + monkeypatch, f"json.dump({{'argv': sys.argv[1:], 'key': os.environ.get('QSV_LLM_APIKEY')}}," f" open({str(record)!r}, 'w'))\n" - "print(json.dumps({'description': 'ok'}))\n" + "print(json.dumps({'description': 'ok'}))\n", ) - fake.chmod(0o755) - monkeypatch.setenv("PATH", f"{fake.parent}:{os.environ['PATH']}") monkeypatch.delenv("QSV_LLM_APIKEY", raising=False) out = tmp_path / "qsv_dict.json" data = await run_qsv_describegpt(tmp_path / "data.csv", KEY, out) assert data == {"description": "ok"} - assert json.loads(out.read_text()) == {"description": "ok"} + # qsv_dict.json is published and hashed: keep the indent=2 re-dump, not qsv's raw bytes + assert out.read_text() == json.dumps({"description": "ok"}, indent=2) seen = json.loads(record.read_text()) assert seen["key"] == KEY assert KEY not in " ".join(seen["argv"]) assert "--api-key" not in seen["argv"] assert seen["argv"][:2] == ["describegpt", str(tmp_path / "data.csv")] + + +STATS_CSV = "field,type,cardinality\nname,String,3\nage,Integer,2\n" +FREQ_CSV = "field,value,count,percentage\nname,ann,2,50\nname,bob,1,25\nage,30,4,100\n" + + +@posix_only +async def test_stats_streams_qsv_output_to_disk_unchanged_and_parses_it(tmp_path, monkeypatch): + _fake_qsv(tmp_path, monkeypatch, f"sys.stdout.write({STATS_CSV!r})\n") + out = tmp_path / "res" / "qsv_stats.csv" + + stats = await run_qsv_stats(tmp_path / "data.csv", out) + + assert stats == { + "name": {"type": "String", "cardinality": "3"}, + "age": {"type": "Integer", "cardinality": "2"}, + } + # mirrored to the Fair Store as-is, so the bytes must be exactly qsv's stdout + assert out.read_bytes() == STATS_CSV.encode() + assert not out.with_name(out.name + ".part").exists() + + +@posix_only +async def test_frequency_parses_counts_as_ints(tmp_path, monkeypatch): + _fake_qsv(tmp_path, monkeypatch, f"sys.stdout.write({FREQ_CSV!r})\n") + out = tmp_path / "qsv_frequency.csv" + + freq = await run_qsv_frequency(tmp_path / "data.csv", out) + + assert freq == { + "name": [{"value": "ann", "count": 2}, {"value": "bob", "count": 1}], + "age": [{"value": "30", "count": 4}], + } + assert out.read_text() == FREQ_CSV + + +@posix_only +async def test_failed_run_returns_none_and_keeps_the_previous_output(tmp_path, monkeypatch, capsys): + _fake_qsv( + tmp_path, + monkeypatch, + "sys.stdout.write('field,type\\nhalf')\n" + "sys.stderr.write('csv error: unequal lengths\\n')\n" + "sys.exit(1)\n", + ) + out = tmp_path / "qsv_stats.csv" + out.write_text("previous good output\n") + + assert await run_qsv_stats(tmp_path / "data.csv", out) is None + + assert out.read_text() == "previous good output\n" + assert not out.with_name(out.name + ".part").exists() + assert "qsv stats failed" in capsys.readouterr().out + + +@posix_only +async def test_hung_run_is_killed_at_the_timeout(tmp_path, monkeypatch, capsys): + pid_file = tmp_path / "pid" + _fake_qsv( + tmp_path, + monkeypatch, + f"open({str(pid_file)!r}, 'w').write(str(os.getpid()))\ntime.sleep(60)\n", + ) + monkeypatch.setenv("VERIKAN_QSV_FREQUENCY_TIMEOUT", "1") + out = tmp_path / "qsv_frequency.csv" + + started = time.monotonic() + assert await run_qsv_frequency(tmp_path / "data.csv", out) is None + assert time.monotonic() - started < 10 + + with pytest.raises(ProcessLookupError): + os.kill(int(pid_file.read_text()), 0) + assert not out.exists() + assert not out.with_name(out.name + ".part").exists() + assert "QsvTimeout, timeout" in capsys.readouterr().out + + +@posix_only +async def test_count(tmp_path, monkeypatch): + fake = _fake_qsv(tmp_path, monkeypatch, "print(42)\n") + assert await run_qsv_count(tmp_path / "data.csv") == 42 + + fake.write_text(f"#!{sys.executable}\nimport sys\nsys.exit(1)\n") + assert await run_qsv_count(tmp_path / "data.csv") is None + + +async def test_missing_binary_returns_none_instead_of_raising(tmp_path, monkeypatch, capsys): + monkeypatch.setenv("QSV_BIN", str(tmp_path / "no-such-qsv")) + + assert await run_qsv_count(tmp_path / "data.csv") is None + assert await run_qsv_stats(tmp_path / "data.csv", tmp_path / "s.csv") is None + assert await run_qsv_describegpt(tmp_path / "data.csv", KEY, tmp_path / "d.json") is None + assert "QsvNotFound" in capsys.readouterr().out + + +@posix_only +async def test_describegpt_failure_on_a_binary_without_it_says_so(tmp_path, monkeypatch, capsys): + # qsvlite has no describegpt, and binary discovery falls back to it. + _fake_qsv( + tmp_path, + monkeypatch, + "arg = sys.argv[1]\n" + "if arg == '--version':\n" + " print('qsvlite 24.0.0-standard-16-16; (aarch64-apple-darwin compiled with Rust 1.99)')\n" + "elif arg == '--list':\n" + " print('Installed commands (2):')\n" + " print(' count Count records')\n" + " print(' stats Infer data types')\n" + "else:\n" + " sys.stderr.write('Invalid arguments.\\n')\n" + " sys.exit(1 if arg == '--capabilities' else 2)\n", + ) + out = tmp_path / "qsv_dict.json" + + assert await run_qsv_describegpt(tmp_path / "data.csv", KEY, out) is None + + printed = capsys.readouterr().out + assert "has no describegpt" in printed + assert "qsvlite 24.0.0" in printed + assert not out.exists() diff --git a/uv.lock b/uv.lock index 995364f..074f61a 100644 --- a/uv.lock +++ b/uv.lock @@ -470,6 +470,7 @@ dependencies = [ { name = "python-dotenv" }, { name = "python-multipart" }, { name = "pyyaml" }, + { name = "qsv-client" }, { name = "redis" }, { name = "structlog" }, { name = "tenacity" }, @@ -518,6 +519,7 @@ requires-dist = [ { name = "python-dotenv", specifier = ">=1.0.0" }, { name = "python-multipart", specifier = ">=0.0.6" }, { name = "pyyaml", specifier = ">=6.0.0" }, + { name = "qsv-client", specifier = ">=0.1.0" }, { name = "redis", specifier = ">=5.0.0" }, { name = "respx", marker = "extra == 'dev'", specifier = ">=0.23.1" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.1.0" }, @@ -2184,6 +2186,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6e/97/bc4f0edefb992df4fdebcf9f0cc40f631cd4ed277e1ed59ef2cd99a5c8c5/pyzmq-27.2.0-cp315-cp315t-win_arm64.whl", hash = "sha256:a843094b4d3d633bc3623e47a2ff50742d6af02bc1f7606aa2e67e971e21878d", size = 581985, upload-time = "2026-08-20T19:07:34.19Z" }, ] +[[package]] +name = "qsv-client" +version = "0.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/69/9f/9b7e1dc62f6514e85f9d63ce518210cc31da5ffb499c4c122c599703785f/qsv_client-0.1.0.tar.gz", hash = "sha256:ee00c1b23b1c82d2aabe3c8f9ba1efc005af79a16417b051a5fb13c79e73cc92", size = 74153, upload-time = "2026-10-08T17:45:38.452Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/c5/eb90cab38ba04bb698dfe1963d6ed1eb4b5278a3888520057aedbe8132da/qsv_client-0.1.0-py3-none-any.whl", hash = "sha256:5a2a04a93ee9f689a294fd659c4f2a627cc5462d5a618ab34edfe7cdc3802030", size = 27070, upload-time = "2026-10-08T17:45:37.143Z" }, +] + [[package]] name = "redis" version = "7.4.0"