From 2fc17e0de7bc29ad7e0c10f997d633b7d156e9f4 Mon Sep 17 00:00:00 2001 From: Joel Natividad <1980690+jqnatividad@users.noreply.github.com> Date: Wed, 14 Oct 2026 19:07:11 -0400 Subject: [PATCH] build: require qsv 25.0.0 (MINIMUM_QSV_VERSION, Dockerfile.worker, CI) qsv 25.0.0 is the first release with structured JSON errors (QSV_ERROR_FORMAT=json, qsv#4765) and --capabilities, which QSVCommand reads through qsv-client; on older qsv both fell back to plain text. Bumps MINIMUM_QSV_VERSION, the Dockerfile.worker QSV_VERSION, and QSV_VER in ci.yml, main.yml and test.yml from 20.1.0 to 25.0.0, plus the README install example and CLAUDE.md. The issue #173 date-inference matrix gives the same types under 25.0.0 as under 20.1.0; its guard is re-pinned to 25.0.x. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 4 ++-- .github/workflows/main.yml | 2 +- .github/workflows/test.yml | 2 +- CHANGELOG.md | 1 + CLAUDE.md | 2 +- Dockerfile.worker | 2 +- README.md | 8 ++++---- ckanext/datapusher_plus/config.py | 2 +- tests/test_issue_173_date_format_inference.py | 19 ++++++++++--------- tests/test_qsv_v20_regression.py | 9 ++++++--- 10 files changed, 28 insertions(+), 23 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d742b5c..f778ae2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ on: qsv_version: description: 'qsv version to install' required: false - default: '20.1.0' + default: '25.0.0' type: string polling_timeout_seconds: description: 'Max seconds to wait for DataPusher to process each file' @@ -33,7 +33,7 @@ permissions: env: FILES_DIR: ${{ github.event.inputs.testing_directory || 'quick' }} - QSV_VER: ${{ github.event.inputs.qsv_version || '20.1.0' }} + QSV_VER: ${{ github.event.inputs.qsv_version || '25.0.0' }} # 90s ceiling (45 polls x 2s): a Prefect flow run spawns a fresh interpreter # and bootstraps the full CKAN app context, so per-job latency is ~20-30s -- # well above the old RQ path. Matches main.yml's polling window. diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fb9c24a..1dac8b0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -22,7 +22,7 @@ env: CKAN_SITE_URL: http://localhost:5000 CKAN_SITE_ID: default CKAN_SITE_TITLE: "CKAN Test Instance" - QSV_VER : "20.1.0" + QSV_VER : "25.0.0" jobs: setup: runs-on: ubuntu-latest diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 18b9a14..3a5de56 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,7 +15,7 @@ permissions: contents: read env: - QSV_VER: "20.1.0" + QSV_VER: "25.0.0" jobs: unit-tests: diff --git a/CHANGELOG.md b/CHANGELOG.md index c1d1f72..87254c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Auto-indexing now uses a closed cardinality range `[auto_index_min_threshold, auto_index_threshold]` instead of a single upper bound (issue #142). Defaults bumped: `auto_index_threshold` `3` → `10` and the new `auto_index_min_threshold` defaults to `3`. The new lower bound skips the useless-single-value-column case @EricSoroos flagged in #142 (a 1-value text column previously produced a 10–40MB B-tree the Postgres planner would never choose). The upper bump from 3 → 10 widens the DataTables-SearchBuilder filtering sweet spot to cover typical enum-shaped columns (status = 3–10 values, common enums under ~10). **Operator-facing impact at default settings:** columns with cardinality 1–2 lose their auto-index (intentional — the indexes were dead weight); columns with cardinality 4–10 gain an auto-index. No migration is required; the change re-applies on the next resubmit. To restore pre-#142 behavior (no lower floor) set `ckanext.datapusher_plus.auto_index_min_threshold = 0`. The setting `auto_index_threshold = -1` ("index every column") still works but now hits the `min_threshold = 3` floor unless paired with `auto_index_min_threshold = 0`. * **BREAKING** Bumped `MINIMUM_QSV_VERSION` from `4.0.0` to `20.0.0`. Operators must upgrade their `qsv` binary at the path configured by `ckanext.datapusher_plus.qsv_bin` before deploying this version — DP+ will refuse to start otherwise. See [qsv 20.0.0 release notes](https://github.com/dathere/qsv/releases/tag/20.0.0) and the migration notes below. ([README install snippet](README.md#option-2-install-prebuilt-qsv-binaries) updated accordingly.) * **BREAKING** Bumped `MINIMUM_QSV_VERSION` from `20.0.0` to `20.1.0` (and bumped the qsv version installed by `Dockerfile.worker`, `.github/workflows/ci.yml`, and `.github/workflows/main.yml` to match). Operators must upgrade their `qsv` binary at the path configured by `ckanext.datapusher_plus.qsv_bin` before deploying this version — DP+ will refuse to start otherwise with `JobError: At least qsv version 20.1.0 required. Found 20.0.0.`. qsv 20.1.0 itself introduces no breaking changes against 20.0.0 (per the [20.1.0 release notes](https://github.com/dathere/qsv/releases/tag/20.1.0): "pipelines built on 20.0.0 will upgrade in place"), so the upgrade is binary-swap-only — no data re-ingestion is required. The reason this is still flagged as **BREAKING** is the minimum-version gate, not the qsv behavior. User-visible improvement justifying the floor bump: qsv-dateparser 0.14 → 0.15 in qsv 20.1.0 adds recognition for ISO 8601 `T`-separated datetimes without a timezone suffix (e.g. `2024-10-11T14:30:00`) — qsv 20.0.0 misclassified these as `String` during `qsv stats --infer-dates`, which in DP+ surfaced as a date-typed column being demoted to text on certain CSV shapes (one of the gaps documented in the issue #173 regression test, now closed). The regression-test matrix in `tests/test_issue_173_date_format_inference.py` is updated to the new baseline. +* **BREAKING** Bumped `MINIMUM_QSV_VERSION` from `20.1.0` to `25.0.0` (and bumped the qsv version installed by `Dockerfile.worker`, `.github/workflows/ci.yml`, `.github/workflows/main.yml` and `.github/workflows/test.yml` to match). Operators must upgrade their `qsv` binary at the path configured by `ckanext.datapusher_plus.qsv_bin` before deploying this version — DP+ will refuse to start otherwise with `JobError: At least qsv version 25.0.0 required. Found 20.1.0.`. qsv [25.0.0](https://github.com/dathere/qsv/releases/tag/25.0.0) is the first release with what `QSVCommand` reads through `qsv-client`: structured JSON errors (`QSV_ERROR_FORMAT=json`, [qsv#4765](https://github.com/dathere/qsv/pull/4765)), so a failed qsv step reports its error kind and message instead of raw stderr, and `qsv --capabilities` for the version check. On older qsv, both fell back to parsing plain text. It also redacts API keys from `describegpt`'s attribution ([qsv#4766](https://github.com/dathere/qsv/pull/4766)). Date inference is unchanged: the issue #173 matrix in `tests/test_issue_173_date_format_inference.py` gives the same type for every column under qsv 25.0.0 as under 20.1.0, and is now pinned to 25.0.x. ### qsv 20.0.0 migration notes (operator-facing) The new minimum brings the following user-visible changes from qsv 4.0.0 → 20.0.0 that may affect existing CKAN deployments: diff --git a/CLAUDE.md b/CLAUDE.md index de42a72..c357bfc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -165,7 +165,7 @@ Formula types: ## External Dependencies - **Python 3.10, 3.11, 3.12, 3.13** -- **qsv v20.1.0+** — Must be installed at path specified by `ckanext.datapusher_plus.qsv_bin` (`MINIMUM_QSV_VERSION` is enforced at startup) +- **qsv v25.0.0+** — Must be installed at path specified by `ckanext.datapusher_plus.qsv_bin` (`MINIMUM_QSV_VERSION` is enforced at startup) - **CKAN 2.10+** with ckanext-scheming - **PostgreSQL** datastore - **Prefect 3.7+** — Orchestrates the v3.0 ingestion flow (replaces the v2 RQ-based job runner; `migrate-from-rq` handles the upgrade) diff --git a/Dockerfile.worker b/Dockerfile.worker index a0a2756..755573b 100644 --- a/Dockerfile.worker +++ b/Dockerfile.worker @@ -17,7 +17,7 @@ # with ``no match for platform in manifest``. FROM --platform=linux/amd64 ckan/ckan-dev:2.11 -ARG QSV_VERSION=20.1.0 +ARG QSV_VERSION=25.0.0 ARG QSV_RELEASE=https://github.com/dathere/qsv/releases/download # Root for the apt installs and pip writes. ckan-dev's default user is diff --git a/README.md b/README.md index fc1930f..1c3ec6c 100644 --- a/README.md +++ b/README.md @@ -244,12 +244,12 @@ sudo apt install qsv -y ### Option 2: Install prebuilt qsv binaries [Download the appropriate prebuilt binaries](https://github.com/dathere/qsv/releases/latest) for your platform and copy -it to the appropriate directory. For example you can use the following commands for qsv v20.1.0 on x86_64 Linux (you can update the version `20.1.0` to the latest version available on the [releases page](https://github.com/dathere/qsv/releases)): +it to the appropriate directory. For example you can use the following commands for qsv v25.0.0 on x86_64 Linux (you can update the version `25.0.0` to the latest version available on the [releases page](https://github.com/dathere/qsv/releases)): ```bash -wget https://github.com/dathere/qsv/releases/download/20.1.0/qsv-20.1.0-x86_64-unknown-linux-gnu.zip -unzip qsv-20.1.0-x86_64-unknown-linux-gnu.zip -rm qsv-20.1.0-x86_64-unknown-linux-gnu.zip +wget https://github.com/dathere/qsv/releases/download/25.0.0/qsv-25.0.0-x86_64-unknown-linux-gnu.zip +unzip qsv-25.0.0-x86_64-unknown-linux-gnu.zip +rm qsv-25.0.0-x86_64-unknown-linux-gnu.zip sudo mv qsv* /usr/local/bin ``` diff --git a/ckanext/datapusher_plus/config.py b/ckanext/datapusher_plus/config.py index deec34e..e40bb3a 100644 --- a/ckanext/datapusher_plus/config.py +++ b/ckanext/datapusher_plus/config.py @@ -31,7 +31,7 @@ POSTGRES_BIGINT_MIN = -9223372036854775808 # QSV version requirements -MINIMUM_QSV_VERSION = "20.1.0" +MINIMUM_QSV_VERSION = "25.0.0" # Logging level # TRACE, DEBUG, INFO, WARNING, ERROR, CRITICAL diff --git a/tests/test_issue_173_date_format_inference.py b/tests/test_issue_173_date_format_inference.py index 2453b21..91b110f 100644 --- a/tests/test_issue_173_date_format_inference.py +++ b/tests/test_issue_173_date_format_inference.py @@ -217,7 +217,8 @@ def _qsv_stats_types(csv_path: Path) -> Dict[str, str]: # silently change user-facing behavior. The "gap" rows are NOT bugs in # DP+ — they're qsv inference limitations, called out so the next # maintainer who looks at #173 doesn't have to rediscover them. -# Baseline captured against qsv 20.1.0 — the version pinned in +# Baseline captured against qsv 20.1.0 and re-verified unchanged against +# qsv 25.0.0 — the version now pinned in # ``Dockerfile.worker`` (production worker), CI workflows, and the # ``dpp-test`` container. # @@ -227,9 +228,9 @@ def _qsv_stats_types(csv_path: Path) -> Dict[str, str]: # added that format (see qsv 20.1.0 release notes), so this column is # now correctly inferred as DateTime. # -# When the production qsv pin moves past 20.1.x, update this matrix +# When the production qsv pin moves past 25.0.x, update this matrix # (capture the new baseline via ``qsv stats --infer-dates -# --dates-whitelist all``) AND bump the ``>= (20, 2, 0)`` skip +# --dates-whitelist all``) AND bump the ``>= (25, 1, 0)`` skip # threshold inside ``test_quoted_csv_inference_matrix`` so the test # stays correct. EXPECTED_INFERENCE = { @@ -262,17 +263,17 @@ def test_quoted_csv_inference_matrix(): Either way the test gives the next maintainer a clean diff against a known baseline. """ - # The matrix is pinned to qsv 20.1.x specifically. Skip rather + # The matrix is pinned to qsv 25.0.x specifically. Skip rather # than false-fail when the host's qsv is newer than the - # production-pinned 20.1.x in Dockerfile.worker / CI / dpp-test. - # When the production pin moves past 20.1.x, bump this guard AND + # production-pinned 25.0.x in Dockerfile.worker / CI / dpp-test. + # When the production pin moves past 25.0.x, bump this guard AND # update EXPECTED_INFERENCE in lockstep. - if QSV_VERSION is not None and QSV_VERSION >= (20, 2, 0): + if QSV_VERSION is not None and QSV_VERSION >= (25, 1, 0): pytest.skip( - f"EXPECTED_INFERENCE is pinned to qsv 20.1.x baseline; " + f"EXPECTED_INFERENCE is pinned to qsv 25.0.x baseline; " f"detected qsv {'.'.join(map(str, QSV_VERSION))}. Update the " "matrix and the comment on issue #173 when the production " - "Dockerfile.worker QSV_VERSION pin moves past 20.1.x." + "Dockerfile.worker QSV_VERSION pin moves past 25.0.x." ) types = _qsv_stats_types(QUOTED_FIXTURE) diff --git a/tests/test_qsv_v20_regression.py b/tests/test_qsv_v20_regression.py index df0eab1..937b730 100644 --- a/tests/test_qsv_v20_regression.py +++ b/tests/test_qsv_v20_regression.py @@ -4,7 +4,7 @@ qsv 20.0.0 upgrade regression tests. Exercises the qsv behaviors that changed between the previous -MINIMUM_QSV_VERSION (4.0.0) and the current floor (20.1.0; previously 20.0.0). These tests shell +MINIMUM_QSV_VERSION (4.0.0) and the floor at the time (20.1.0; the floor is now 25.0.0). These tests shell out directly to the qsv binary (resolved from the QSV_BIN env var or `qsv` / `qsvdp` on PATH) rather than through QSVCommand so that they exercise the qsv contract DP+ depends on without requiring CKAN config bootstrap. @@ -454,6 +454,9 @@ def test_minimum_qsv_version_constant_is_pinned(self): # 0.14→0.15 bump that closes the ISO 8601 T-separated-no-tz # inference gap (one of the #173 gaps). See CHANGELOG entry # under [Unreleased] for the operator-facing breaking note. + # Bumped again 20.1.0 → 25.0.0: the first qsv release with the + # structured JSON errors (QSV_ERROR_FORMAT=json) and --capabilities + # that QSVCommand reads through qsv-client. config_py = REPO_ROOT / "ckanext" / "datapusher_plus" / "config.py" text = config_py.read_text(encoding="utf-8") m = re.search( @@ -462,6 +465,6 @@ def test_minimum_qsv_version_constant_is_pinned(self): re.MULTILINE, ) assert m, f"MINIMUM_QSV_VERSION constant not found in {config_py}" - assert m.group(1) == "20.1.0", ( - f"MINIMUM_QSV_VERSION must be '20.1.0' (got {m.group(1)!r})" + assert m.group(1) == "25.0.0", ( + f"MINIMUM_QSV_VERSION must be '25.0.0' (got {m.group(1)!r})" )