From 02ad5138bf03d3360b4d6552acd704e6df30d490 Mon Sep 17 00:00:00 2001 From: luth2 <142568716+luth2@users.noreply.github.com> Date: Wed, 28 Oct 2026 18:12:22 +0100 Subject: [PATCH 1/6] feat(python): add experimental Python 3.15 compatibility --- .../python-metadata-compatibility.yml | 93 ++++++++++++ Dockerfile | 3 +- docker/install-app-dependencies.sh | 17 +++ docs/admin_docs/installation/pypi.mdx | 137 ++++++++++++++++++ docs/static/resources/openapi.json | 24 +-- pyproject.toml | 7 +- .../commands/dataset/importers/v1/utils.py | 4 +- superset/databases/api.py | 42 +++++- superset/db_engine_specs/lint_metadata.py | 4 - superset/embedded/view.py | 2 +- superset/extensions/metadb.py | 26 +++- superset/models/helpers.py | 2 + superset/result_set.py | 6 + superset/sql/parse.py | 4 + superset/tasks/cron_util.py | 1 + .../utils/pandas_postprocessing/boxplot.py | 6 +- .../pandas_postprocessing/contribution.py | 4 +- .../utils/pandas_postprocessing/histogram.py | 2 +- superset/utils/pandas_postprocessing/pivot.py | 4 +- .../utils/pandas_postprocessing/resample.py | 11 +- .../utils/pandas_postprocessing/rolling.py | 6 +- superset/utils/pandas_postprocessing/utils.py | 3 +- superset/views/log/api.py | 5 - .../integration_tests/databases/api_tests.py | 9 +- .../db_engine_specs/postgres_tests.py | 1 + .../db_engine_specs/redshift_tests.py | 2 +- tests/integration_tests/embedded/test_view.py | 11 +- tests/integration_tests/security/api_tests.py | 24 ++- .../sql_lab/test_execute_sql_statements.py | 6 +- tests/integration_tests/thumbnails_tests.py | 10 ++ .../databases/columnar_reader_test.py | 8 +- .../commands/databases/csv_reader_test.py | 7 +- .../commands/importers/v1/utils_test.py | 5 +- tests/unit_tests/databases/api_test.py | 11 +- .../db_engine_specs/test_databricks.py | 1 + .../db_engine_specs/test_lint_metadata.py | 87 +++++++++++ .../docker_app_dependencies_test.py | 87 +++++++++++ .../unit_tests/extensions/test_sqlalchemy.py | 33 ++++- tests/unit_tests/importexport/api_test.py | 2 +- .../mcp_service/chart/test_bullet_chart.py | 15 +- .../chart/test_bullet_tool_paths.py | 28 ++-- tests/unit_tests/models/core_test.py | 3 +- tests/unit_tests/models/helpers_test.py | 2 +- .../pandas_postprocessing/test_rolling.py | 12 +- .../pandas_postprocessing/test_sort.py | 4 +- .../pandas_postprocessing/test_utils.py | 11 ++ .../sql/execution/test_async_results.py | 8 +- tests/unit_tests/utils/excel_tests.py | 10 +- tests/unit_tests/utils/test_date_parsing.py | 5 +- 49 files changed, 680 insertions(+), 135 deletions(-) create mode 100644 .github/workflows/python-metadata-compatibility.yml create mode 100644 docker/install-app-dependencies.sh create mode 100644 tests/unit_tests/db_engine_specs/test_lint_metadata.py create mode 100644 tests/unit_tests/docker_app_dependencies_test.py diff --git a/.github/workflows/python-metadata-compatibility.yml b/.github/workflows/python-metadata-compatibility.yml new file mode 100644 index 000000000000..d3ef3e379a3f --- /dev/null +++ b/.github/workflows/python-metadata-compatibility.yml @@ -0,0 +1,93 @@ +name: Python-Metadata-Compatibility + +permissions: + contents: read + +on: + pull_request: + paths: + - .github/workflows/python-metadata-compatibility.yml + - superset/db_engine_specs/** + - tests/unit_tests/db_engine_specs/test_lint_metadata.py + - pyproject.toml + - superset/result_set.py + - superset/utils/pandas_postprocessing/** + - tests/unit_tests/pandas_postprocessing/** + - tests/unit_tests/result_set_test.py + - superset-core/** + workflow_dispatch: {} + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + +jobs: + metadata-parser: + runs-on: ubuntu-26.04 + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + python-version: [ "3.11", "3.12", "3.13", "3.14", "3.15" ] + env: + PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1" + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Setup Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: true + - name: Install standalone test dependencies + run: python -m pip install pytest==7.4.4 + - name: Test metadata parser without application dependencies + run: python -m pytest --noconftest -o addopts= tests/unit_tests/db_engine_specs/test_lint_metadata.py -q -W default::DeprecationWarning + + runtime-compatibility: + runs-on: ubuntu-26.04 + timeout-minutes: 25 + strategy: + fail-fast: false + matrix: + python-version: [ "3.11", "3.12", "3.13", "3.14", "3.15" ] + env: + SUPERSET_HOME: ${{ runner.temp }}/superset-python-compatibility + SUPERSET_SECRET_KEY: python-compatibility-ci-only-not-for-production + FLASK_APP: superset + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Setup Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.python-version }} + - name: Install application and focused test dependencies + run: >- + python -m pip install -e ./superset-core -e . pytest==7.4.4 pytest-mock freezegun parameterized responses rich prophet + - name: Check dependency consistency + run: python -m pip check + - name: Migrate a fresh metadata database + run: python -m flask db upgrade + - name: Test application health and runtime compatibility + run: | + python - <<'PY' + import pytest + from superset.app import create_app + + app = create_app() + assert app.test_client().get('/health').status_code == 200 + with app.app_context(): + raise SystemExit(pytest.main([ + '--noconftest', '-o', 'addopts=', + 'tests/unit_tests/db_engine_specs/test_lint_metadata.py', + 'tests/unit_tests/pandas_postprocessing', + 'tests/unit_tests/result_set_test.py', + 'superset-core/tests', '-q', + '-W', 'default::DeprecationWarning', + ])) + PY diff --git a/Dockerfile b/Dockerfile index a3b02332eb00..33fd7407f707 100644 --- a/Dockerfile +++ b/Dockerfile @@ -303,7 +303,8 @@ COPY requirements/base.txt requirements/ COPY superset-core superset-core RUN --mount=type=cache,target=${SUPERSET_HOME}/.cache/uv \ - /app/docker/pip-install.sh --requires-build-essential -r requirements/base.txt + --mount=type=bind,source=.,target=/src \ + /app/docker/install-app-dependencies.sh /src # Copy compiled frontend assets and application source now that # dependencies have been resolved and cached above. diff --git a/docker/install-app-dependencies.sh b/docker/install-app-dependencies.sh new file mode 100644 index 000000000000..671e935bad08 --- /dev/null +++ b/docker/install-app-dependencies.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -euo pipefail + +SOURCE_ROOT="${1:?Application source root is required}" + +if python -c 'import sys; raise SystemExit(sys.version_info < (3, 15))'; then + if [[ -d "${SOURCE_ROOT}" && ! -w "${SOURCE_ROOT}" ]]; then + BUILD_ROOT="$(mktemp -d)" + trap 'rm -rf "${BUILD_ROOT}"' EXIT + cp -a "${SOURCE_ROOT}/." "${BUILD_ROOT}/" + SOURCE_ROOT="${BUILD_ROOT}" + fi + /app/docker/pip-install.sh --requires-build-essential \ + --no-sources --no-editable "${SOURCE_ROOT}/superset-core" "${SOURCE_ROOT}" +else + /app/docker/pip-install.sh --requires-build-essential -r requirements/base.txt +fi diff --git a/docs/admin_docs/installation/pypi.mdx b/docs/admin_docs/installation/pypi.mdx index cd0c7a61a618..146c2d578e3d 100644 --- a/docs/admin_docs/installation/pypi.mdx +++ b/docs/admin_docs/installation/pypi.mdx @@ -15,6 +15,143 @@ import useBaseUrl from '@docusaurus/useBaseUrl'; This page describes how to install Superset using the `apache_superset` package [published on PyPI](https://pypi.org/project/apache_superset/). +## Python Compatibility + +The minimum Python version is 3.11. The Python version classifiers in +[pyproject.toml](https://github.com/apache/superset/blob/master/pyproject.toml) +list the officially supported versions; the open-ended `requires-python` lower +bound does not certify every newer interpreter. + +The source checkout includes experimental Python 3.15 compatibility. Its package +requirements select NumPy `>=2.5.4,<2.6` and pandas `>=3.1.0,<3.2` on Python 3.15 +and newer. Python 3.11 through 3.14 retain NumPy `>=1.23.5,<2.5` and pandas +`>=2.3.3,<2.4`; the minimum Python version has not changed. These source changes +do not imply that an already published PyPI release supports Python 3.15. + +The focused +[compatibility workflow](https://github.com/apache/superset/blob/master/.github/workflows/python-metadata-compatibility.yml) +tests Python 3.11 through 3.15. It installs the application without dependency +overrides, checks dependency consistency, migrates a fresh SQLite metadata +database, checks `/health`, and runs engine metadata, pandas post-processing +(including Prophet), result-set, and core SDK tests. The same 278 tests and fresh +SQLite migrations passed locally on Windows for all five interpreter versions, +including CPython 3.15.0 final. + +A local Podman container running Debian 13 with CPython 3.15.2 also passed +336 focused tests, including the additional DataFrame serialization tests. +Installation without dependency overrides, dependency consistency, a fresh +SQLite migration to head, and the application health check succeeded. The +extended-precision `longdouble` regression passes on Linux. + +The same isolated Linux environment also migrated a PostgreSQL 17 metadata +database to head and initialized the application and test users. Selected Redis 7 +cache and synchronous/asynchronous query tests passed with a real Celery prefork +worker. All 35 tests in the selected CSRF/login, SQL Lab execution, cache-manager, +chart-cache, and Celery groups passed, including CTAS and CVAS operations. The +test database requires the configured `sqllab_test_db` schema. The CSRF tests +explicitly distinguish public endpoint access from access requiring login, and +the SQL Lab tests use synchronous mocks for the results backend. + +A complete local run of `tests/common` and `tests/unit_tests` on CPython 3.15.2 +finished with 32,821 passed, 57 skipped, 5 expected failures, and no unexpected +failures. Translation tests require complete frontend sources with LF line +endings and a Git checkout; cron description expectations use `en_US.UTF-8`. +This final run includes the raw-cursor join truncation fix. All 22 dialect tests +also passed on both Python 3.11 and 3.15, covering SQLAlchemy, direct iteration, +and all three fetch methods. +The earlier follow-up regression group also passed 1,683 tests on Python 3.11, +and the nine selected CSRF/login and SQL Lab tests passed on that version. + +The production frontend build succeeded with Node 24.16.0 and webpack 5.112.0 +(four warnings). Chromium acceptance passed all 24 selected Playwright tests +without retries: login success and rejection, chart management and ZIP export, +dashboard management and import/export, three chart types rendered from real +queries (including an ECharts canvas-pixel assertion), and SQL Lab query, +tab, saved-query, dataset, and Explore workflows. The browser server used +Gunicorn, PostgreSQL 17, Redis 7, `TESTING=False`, and enabled CSRF protection. +The browser dataset used 1,200 locally generated `birth_names` rows; chart +management tests additionally used the bundled Slack dataset metadata, without +claiming Slack data-query coverage. + +Broader integration testing also covers the schema-free database SELECT route +without introducing a new permission, validates generated and published OpenAPI +path parameters, and rejects a missing embedded-dashboard referrer when an +allowed-origin list is configured. The referrer regression and the surrounding +database API/schema tests passed on Python 3.11 as well (41 selected tests). +The legacy Flask-Testing live server uses an explicit local `fork` context on +platforms that support it; this does not change application or Celery process +defaults. Thumbnail image generation remains outside this acceptance scope. + +A complete local run of `tests/integration_tests` on CPython 3.15.2 finished +with 3,131 passed, 178 skipped, and no failures in 673 seconds. It used a fresh +PostgreSQL 17 metadata database, Redis 7, and a real Celery prefork worker with +a Redis SQL Lab results backend. Skipped tests remain unvalidated; a passing +collection does not establish support for every optional database or feature. + +Earlier runs on the 8 GB Podman VM were invalidated by kernel-confirmed +out-of-memory worker termination. The successful final unit run used 16 GB; +browser and full unit acceptance ran separately. The local Hyper-V storage +also showed intermittent I/O errors, so this host is not a production reference. + +Integration reruns require a fresh, migrated test metadata database: interrupted +or focused runs can leave fixtures that invalidate exact object-count assertions. +Keep browser acceptance in a separate database and Redis namespace. A real SQL +Lab worker also needs `RESULTS_BACKEND`, in addition to the Celery broker and +result-backend settings; verify these in both the test and worker processes. + +All applicable pre-commit hooks passed for the changed files in Linux, including +MyPy, Ruff, formatting, YAML, and the advisory workflow audit. Pylint passed when +run explicitly against the changed application modules with `--jobs=1`; its +parallel serialization path uses a code-object attribute removed in Python 3.15. + +For Python 3.15 testing, install from the source checkout's package metadata, +rather than applying the generated Python 3.11 requirement pins: + +```bash +python -m pip install -e ./superset-core -e . +``` + +Run this command from the repository root in an isolated Python 3.15 environment. +Some dependencies require source builds, so a working C/C++ toolchain may be +needed. Windows validation used the x64 Visual Studio 2022 Build Tools. The +generated requirement files remain tied to the existing Python 3.11 deployment +baseline and are not Python 3.15 lock files. + +When building the repository's `lean` or `superset` Docker target with a Python +3.15 base image, the application dependency stage installs both local packages +from their metadata instead of applying the Python 3.11 pins. Older interpreters +retain the requirements-based installation. This does not change the default +image's Python version or establish Python 3.15 support for every Docker target; +the development target still uses its generated development requirements. + +Container acceptance must include the complete image build, dependency consistency, +fresh metadata migration, authentication, and the workloads needed by the deployment. +A dependency-only image build is not a substitute for that acceptance. The image's +`/health` endpoint is a liveness check, not proof that PostgreSQL, Redis, or queued +work is available; verify those workflows separately in readiness monitoring. + +For deployment-specific shutdown validation, send SIGTERM to the Gunicorn master +and allow more than its configured graceful timeout before escalating. Validate +worker exits as well as the master's exit code. Reference tests on Python 3.11 +and 3.15 exposed a Gunicorn gthread keepalive-drain issue requiring deadline +escalation after requests. A separately validated alternative is +`SERVER_WORKER_CLASS=sync` with `SERVER_THREADS_AMOUNT=1`; setting only the worker +class while retaining multiple threads can select gthread again. Synchronous +workers have different concurrency characteristics, so capacity must be measured +for that configuration. + +This validation is not a full production certification. The remote Linux CI +jobs, skipped integration cases, the full database-driver matrix, +live report delivery and screenshot workflows, and browser workflows +beyond the selected 24 acceptance cases still require validation. Production +acceptance must also cover the target deployment's authentication, secrets, +network/TLS configuration, workload, backup/restore, and availability requirements. +An additional Windows +DataFrame test assumes extended `longdouble` precision unavailable on that +platform; Windows also exposes existing Unix-signal type-checking limitations. +Use an officially supported Python version for production deployments until +these broader compatibility checks have been completed. + ## OS Dependencies Superset stores database connection information in its metadata database. For that purpose, we use diff --git a/docs/static/resources/openapi.json b/docs/static/resources/openapi.json index 643f30c49322..809e3ad6ac5a 100644 --- a/docs/static/resources/openapi.json +++ b/docs/static/resources/openapi.json @@ -4279,6 +4279,7 @@ "type": "object" }, "DashboardRestApi.get_list": { + "additionalProperties": false, "properties": { "certification_details": { "nullable": true, @@ -4363,6 +4364,7 @@ "type": "object" }, "DashboardRestApi.get_list.Subject": { + "additionalProperties": false, "properties": { "id": { "type": "integer" @@ -4382,6 +4384,7 @@ "type": "object" }, "DashboardRestApi.get_list.Subject1": { + "additionalProperties": false, "properties": { "id": { "type": "integer" @@ -4401,6 +4404,7 @@ "type": "object" }, "DashboardRestApi.get_list.Tag": { + "additionalProperties": false, "properties": { "id": { "type": "integer" @@ -4422,6 +4426,7 @@ "type": "object" }, "DashboardRestApi.get_list.User": { + "additionalProperties": false, "properties": { "first_name": { "maxLength": 64, @@ -4442,6 +4447,7 @@ "type": "object" }, "DashboardRestApi.get_list.User1": { + "additionalProperties": false, "properties": { "first_name": { "maxLength": 64, @@ -22876,15 +22882,6 @@ "schema": { "type": "string" } - }, - { - "description": "Table schema", - "in": "path", - "name": "schema_name", - "required": true, - "schema": { - "type": "string" - } } ], "responses": { @@ -26781,15 +26778,6 @@ "/api/v1/log/recent_activity/": { "get": { "parameters": [ - { - "description": "The id of the user", - "in": "path", - "name": "user_id", - "required": true, - "schema": { - "type": "integer" - } - }, { "content": { "application/json": { diff --git a/pyproject.toml b/pyproject.toml index b9fe28227f3e..c23b44d90b24 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,11 +84,13 @@ dependencies = [ "marshmallow-union>=0.1.15.post1", "msgpack>=1.2.3, <1.3", "nh3>=0.3.7, <0.4", - "numpy>=1.23.5, <2.5", + "numpy>=1.23.5, <2.5; python_version < '3.15'", + "numpy>=2.5.4, <2.6; python_version >= '3.15'", "packaging", # -------------------------- # pandas and related (wanting pandas[performance] without numba as it's 100+MB and not needed) - "pandas[excel]>=2.3.3, <2.4", + "pandas[excel]>=2.3.3, <2.4; python_version < '3.15'", + "pandas[excel]>=3.1.0, <3.2; python_version >= '3.15'", "bottleneck", # recommended performance dependency for pandas, see https://pandas.pydata.org/docs/getting_started/install.html#performance-dependencies-recommended # -------------------------- "parsedatetime", @@ -104,6 +106,7 @@ dependencies = [ "PyJWT>=2.15.1, <3.0", # raise floor to match resolved pin; closes SCA false-positive on 2.4.x-range CVEs already fixed in 2.13.0 "redis>=5.0.0, <9.0", "rison>=2.0.1, <3.0", + "rich", "shillelagh[gsheetsapi]>=1.4.5, <2.0", "sshtunnel>=0.5.1, <0.6", diff --git a/superset/commands/dataset/importers/v1/utils.py b/superset/commands/dataset/importers/v1/utils.py index ea77b3a5f0b9..499719cb34e8 100644 --- a/superset/commands/dataset/importers/v1/utils.py +++ b/superset/commands/dataset/importers/v1/utils.py @@ -770,7 +770,7 @@ def _convert_temporal_columns(df: pd.DataFrame, dtype: dict[str, Any]) -> None: for column_name, sqla_type in dtype.items(): if isinstance(sqla_type, (Date, DateTime)): try: - df[column_name] = pd.to_datetime(df[column_name]) + df[column_name] = pd.to_datetime(df[column_name]).dt.as_unit("ns") except OutOfBoundsDatetime: # Row-level fallback: coerce only OOB values; re-raise for malformed # strings. Whole-column errors="coerce" would silently swallow @@ -782,7 +782,7 @@ def _convert_temporal_columns(df: pd.DataFrame, dtype: dict[str, Any]) -> None: result.append(pd.NaT) continue try: - result.append(pd.to_datetime(val)) + result.append(pd.to_datetime(val).as_unit("ns")) except OutOfBoundsDatetime: result.append(pd.NaT) # Other exceptions (e.g. malformed strings) propagate diff --git a/superset/databases/api.py b/superset/databases/api.py index d38f44c497b6..fcd43692609d 100644 --- a/superset/databases/api.py +++ b/superset/databases/api.py @@ -32,7 +32,13 @@ request, Response, ) -from flask_appbuilder.api import expose, protect, rison as parse_rison, safe +from flask_appbuilder.api import ( + expose, + permission_name, + protect, + rison as parse_rison, + safe, +) from flask_appbuilder.models.sqla.interface import SQLAInterface from marshmallow import ValidationError from sqlalchemy.exc import NoSuchTableError, OperationalError, SQLAlchemyError @@ -168,6 +174,7 @@ class DatabaseRestApi(BaseSupersetModelRestApi): "table_extra_metadata", "table_extra_metadata_deprecated", "select_star", + "select_star_without_schema", "catalogs", "schemas", "test_connection", @@ -187,7 +194,10 @@ class DatabaseRestApi(BaseSupersetModelRestApi): resource_name = "database" class_permission_name = "Database" - method_permission_name = MODEL_API_RW_METHOD_PERMISSION_MAP + method_permission_name = { + **MODEL_API_RW_METHOD_PERMISSION_MAP, + "select_star_without_schema": "read", + } allow_browser_login = True base_filters = [["id", DatabaseFilter, lambda: []]] show_columns = [ @@ -1224,6 +1234,34 @@ def table_extra_metadata(self, pk: int) -> FlaskResponse: return self.response(200, **payload) @expose("//select_star//", methods=("GET",)) + @protect() + @permission_name("read") + def select_star_without_schema(self, pk: int, table_name: str) -> FlaskResponse: + """Get a select star statement using the database's default schema. + --- + get: + summary: Get database select star for table in the default schema + parameters: + - in: path + schema: + type: integer + name: pk + description: The database id + - in: path + schema: + type: string + name: table_name + description: Table name + responses: + 200: + description: SQL statement for a select star for table + content: + application/json: + schema: + $ref: "#/components/schemas/SelectStarResponseSchema" + """ + return self.select_star(pk=pk, table_name=table_name) + @expose("//select_star///", methods=("GET",)) @protect() @check_table_access diff --git a/superset/db_engine_specs/lint_metadata.py b/superset/db_engine_specs/lint_metadata.py index fff0146158f0..979286badc82 100644 --- a/superset/db_engine_specs/lint_metadata.py +++ b/superset/db_engine_specs/lint_metadata.py @@ -337,10 +337,6 @@ def _eval_ast_value(node: Any) -> Any: # noqa: C901 if isinstance(node, ast.Constant): return node.value - elif isinstance(node, ast.Str): # Python 3.7 compat - return node.s - elif isinstance(node, ast.Num): # Python 3.7 compat - return node.n elif isinstance(node, ast.List): return [_eval_ast_value(e) for e in node.elts] elif isinstance(node, ast.Dict): diff --git a/superset/embedded/view.py b/superset/embedded/view.py index 30ef3465bf8b..a71d57f19b11 100644 --- a/superset/embedded/view.py +++ b/superset/embedded/view.py @@ -63,7 +63,7 @@ def embedded( # validate request referrer in allowed domains is_referrer_allowed = not embedded.allowed_domains for domain in embedded.allowed_domains: - if same_origin(request.referrer, domain): + if request.referrer and same_origin(request.referrer, domain): is_referrer_allowed = True break diff --git a/superset/extensions/metadb.py b/superset/extensions/metadb.py index 13aa89021378..d07f518b8ea7 100644 --- a/superset/extensions/metadb.py +++ b/superset/extensions/metadb.py @@ -174,12 +174,30 @@ def cursor(*args: Any, **kwargs: Any) -> Any: raw_cursor = original_cursor(*args, **kwargs) original_execute = raw_cursor.execute + def iterate( + results: Iterator[tuple[Any, ...]], multi_table_query: bool + ) -> Iterator[tuple[Any, ...]]: + """Keep lazy row retrieval in the executing statement's context.""" + while True: + token = _executing_multi_table_query.set(multi_table_query) + try: + row = next(results) + except StopIteration: + return + finally: + _executing_multi_table_query.reset(token) + yield row + def execute(operation: str, parameters: Any = None) -> Any: - token = _executing_multi_table_query.set( - _count_referenced_tables(operation) > 1 - ) + multi_table_query = _count_referenced_tables(operation) > 1 + token = _executing_multi_table_query.set(multi_table_query) try: - return original_execute(operation, parameters) + result = original_execute(operation, parameters) + if raw_cursor._results is not None: + raw_cursor._results = iterate( + raw_cursor._results, multi_table_query + ) + return result finally: _executing_multi_table_query.reset(token) diff --git a/superset/models/helpers.py b/superset/models/helpers.py index 483b6b4ad309..df07d8794eb6 100644 --- a/superset/models/helpers.py +++ b/superset/models/helpers.py @@ -355,6 +355,8 @@ def _parse_temporal_join_values(series: pd.Series, column_name: str) -> pd.Serie """Parse working temporal values and wrap pandas parser-policy errors.""" if pd.api.types.is_datetime64_any_dtype(series): return series + if _has_multiple_utc_offsets(series): + return series.map(pd.Timestamp) try: return pd.to_datetime(series, errors="coerce", format="mixed") except (TypeError, ValueError) as ex: diff --git a/superset/result_set.py b/superset/result_set.py index 47506c4956c6..bfb141f9321c 100644 --- a/superset/result_set.py +++ b/superset/result_set.py @@ -288,6 +288,12 @@ def __init__( # pylint: disable=too-many-locals # noqa: C901 series = pd.to_datetime( series, utc=True, errors="coerce" ) + series = series.where( + series.between( + pd.Timestamp.min.tz_localize("UTC"), + pd.Timestamp.max.tz_localize("UTC"), + ) + ) pa_data[i] = pa.Array.from_pandas( series, type=pa.timestamp("ns", tz=tz), diff --git a/superset/sql/parse.py b/superset/sql/parse.py index 76ab3b9b6b99..1d79d04218e3 100644 --- a/superset/sql/parse.py +++ b/superset/sql/parse.py @@ -1243,6 +1243,10 @@ class SQLStatement(BaseSQLStatement[exp.Expression]): # always parse into their own structured node instead. "ADD", "DELETE", + "ADD SQLBLACKLIST", + "DELETE SQLBLACKLIST", + "ADD BACKEND BLACKLIST", + "DELETE BACKEND BLACKLIST", # DDL head-tokens that sqlglot falls back to exp.Command for # whenever the body uses syntax it does not model # (CREATE EXTENSION/FUNCTION...LANGUAGE C/PUBLICATION/etc., diff --git a/superset/tasks/cron_util.py b/superset/tasks/cron_util.py index 2cfaf7f5c89b..0e20fb6cc363 100644 --- a/superset/tasks/cron_util.py +++ b/superset/tasks/cron_util.py @@ -68,6 +68,7 @@ def get_cron_description(cron: str) -> str: # "only on Tuesday" claims the day-of-week narrows the day-of-month clause; # it is an alternative to it, not an extra filter. clause = clause.removeprefix("only ") + clause = clause.removeprefix("or ") return description.replace(day_of_week_clause, f", or {clause}", 1) diff --git a/superset/utils/pandas_postprocessing/boxplot.py b/superset/utils/pandas_postprocessing/boxplot.py index 17ded276ae64..79cdbcb0eab4 100644 --- a/superset/utils/pandas_postprocessing/boxplot.py +++ b/superset/utils/pandas_postprocessing/boxplot.py @@ -19,7 +19,7 @@ import numpy as np from flask_babel import gettext as _ from pandas import DataFrame, Series, to_numeric -from pandas.api.types import is_object_dtype +from pandas.api.types import is_object_dtype, is_string_dtype from superset.exceptions import InvalidPostProcessingError from superset.utils.core import PostProcessingBoxplotWhiskerType @@ -135,7 +135,7 @@ def outliers(series: Series) -> set[float]: # nanpercentile needs numeric values, otherwise the isnan function # that's used in the underlying function will fail for column in metrics: - if is_object_dtype(df[column]): - df.loc[:, column] = to_numeric(df[column], errors="coerce") + if is_object_dtype(df[column]) or is_string_dtype(df[column]): + df[column] = to_numeric(df[column], errors="coerce") return aggregate(df, groupby=groupby, aggregates=aggregates) diff --git a/superset/utils/pandas_postprocessing/contribution.py b/superset/utils/pandas_postprocessing/contribution.py index c4dbc56ec9f2..a54ad6faec4a 100644 --- a/superset/utils/pandas_postprocessing/contribution.py +++ b/superset/utils/pandas_postprocessing/contribution.py @@ -117,7 +117,9 @@ def contribution( # Filled out of place: the selection is a slice of `contribution_df`, and # an in-place fill on a slice warns under pandas 2 and is dropped outright # under copy-on-write. - numeric_df = _select_arithmetic_columns(contribution_df).fillna(0) + numeric_df = ( + _select_arithmetic_columns(contribution_df).fillna(0).infer_objects(copy=False) + ) # verify column selections if columns: numeric_columns = numeric_df.columns.tolist() diff --git a/superset/utils/pandas_postprocessing/histogram.py b/superset/utils/pandas_postprocessing/histogram.py index 25d389d6952b..eef00a950072 100644 --- a/superset/utils/pandas_postprocessing/histogram.py +++ b/superset/utils/pandas_postprocessing/histogram.py @@ -75,7 +75,7 @@ def histogram( return df # convert to numeric, coercing errors to NaN - df.loc[:, column] = to_numeric(df[column], errors="coerce") + df[column] = to_numeric(df[column], errors="coerce") # check if the column contains non-numeric values if df[column].isna().any(): diff --git a/superset/utils/pandas_postprocessing/pivot.py b/superset/utils/pandas_postprocessing/pivot.py index d08ee23344bf..57ad92130147 100644 --- a/superset/utils/pandas_postprocessing/pivot.py +++ b/superset/utils/pandas_postprocessing/pivot.py @@ -404,8 +404,6 @@ def pivot( # pylint: disable=too-many-arguments # noqa: C901 df = _apply_show_values_as(df, percent_mode) if combine_value_with_metric: - # dropna=False preserves restored all-NaN metric rows that would otherwise - # be silently dropped by stack's default dropna=True behavior. - df = df.stack(level=0, dropna=False).unstack() + df = df.stack(level=0, future_stack=True).unstack() return df diff --git a/superset/utils/pandas_postprocessing/resample.py b/superset/utils/pandas_postprocessing/resample.py index a91b031983d1..222d1abe0b0e 100644 --- a/superset/utils/pandas_postprocessing/resample.py +++ b/superset/utils/pandas_postprocessing/resample.py @@ -240,10 +240,19 @@ def resample( # pylint: disable=too-many-arguments _validate_projected_rows(df.index.min(), df.index.max(), rule) if method == "asfreq" and fill_value is not None: + string_columns = [ + column + for column in df.columns + if isinstance(df[column].dtype, pd.StringDtype) + ] + if len(string_columns): + df = df.astype(dict.fromkeys(string_columns, object)) _df = df.resample(rule).asfreq(fill_value=fill_value) _df = _df.fillna(fill_value) elif method == "linear": - _df = df.resample(rule).interpolate() + _df = df.resample(rule).asfreq() + numeric_columns = _df.select_dtypes(include="number").columns + _df[numeric_columns] = _df[numeric_columns].interpolate() else: _df = getattr(df.resample(rule), method)() if method in ("ffill", "bfill"): diff --git a/superset/utils/pandas_postprocessing/rolling.py b/superset/utils/pandas_postprocessing/rolling.py index 0b314d65b26d..ec59d9ed8305 100644 --- a/superset/utils/pandas_postprocessing/rolling.py +++ b/superset/utils/pandas_postprocessing/rolling.py @@ -65,7 +65,11 @@ def rolling( # pylint: disable=too-many-arguments :return: DataFrame with the rolling columns :raises InvalidPostProcessingError: If the request in incorrect """ - rolling_type_options = rolling_type_options or {} + option_names = {"quantile": "q"} if rolling_type == "quantile" else {} + rolling_type_options = { + option_names.get(name, name): value + for name, value in (rolling_type_options or {}).items() + } df_rolling = df.loc[:, columns.keys()] kwargs: dict[str, Union[str, int]] = {} diff --git a/superset/utils/pandas_postprocessing/utils.py b/superset/utils/pandas_postprocessing/utils.py index 6cbfc81891f2..7e466c1c2930 100644 --- a/superset/utils/pandas_postprocessing/utils.py +++ b/superset/utils/pandas_postprocessing/utils.py @@ -255,7 +255,8 @@ def _append_columns( if overwritten: # make sure to return a new DataFrame instead of changing the `base_df`. _base_df = base_df.copy() - _base_df.loc[:, overwritten.keys()] = append_df + for column in overwritten: + _base_df[column] = append_df[column] if not appended: return _base_df # Select before renaming: `append_df` may carry columns the mapping does not diff --git a/superset/views/log/api.py b/superset/views/log/api.py index d4f58e8deed5..4c814bd4a70a 100644 --- a/superset/views/log/api.py +++ b/superset/views/log/api.py @@ -119,11 +119,6 @@ def recent_activity(self, **kwargs: Any) -> FlaskResponse: get: summary: Get recent activity data for a user parameters: - - in: path - schema: - type: integer - name: user_id - description: The id of the user - in: query name: q content: diff --git a/tests/integration_tests/databases/api_tests.py b/tests/integration_tests/databases/api_tests.py index cea98cef44e8..82056e01ec98 100644 --- a/tests/integration_tests/databases/api_tests.py +++ b/tests/integration_tests/databases/api_tests.py @@ -3616,6 +3616,7 @@ def test_available(self, get_available_engine_specs): "engine": "postgresql", "name": "PostgreSQL", "parameters": { + "additionalProperties": False, "properties": { "database": { "description": "Database name", @@ -3685,6 +3686,7 @@ def test_available(self, get_available_engine_specs): "engine": "bigquery", "name": "Google BigQuery", "parameters": { + "additionalProperties": False, "properties": { "credentials_info": { "description": ( @@ -3693,7 +3695,7 @@ def test_available(self, get_available_engine_specs): "type": "string", "x-encrypted-extra": True, }, - "query": {"type": "object"}, + "query": {"type": "object", "additionalProperties": {}}, }, "type": "object", }, @@ -3720,6 +3722,7 @@ def test_available(self, get_available_engine_specs): "engine": "redshift", "name": "Amazon Redshift", "parameters": { + "additionalProperties": False, "properties": { "database": { "description": "Database name", @@ -3785,8 +3788,9 @@ def test_available(self, get_available_engine_specs): "engine": "gsheets", "name": "Google Sheets", "parameters": { + "additionalProperties": False, "properties": { - "catalog": {"type": "object"}, + "catalog": {"type": "object", "additionalProperties": {}}, "oauth2_client_info": { "default": { "authorization_request_uri": "https://accounts.google.com/o/oauth2/v2/auth", @@ -3837,6 +3841,7 @@ def test_available(self, get_available_engine_specs): "engine": "mysql", "name": "MySQL", "parameters": { + "additionalProperties": False, "properties": { "database": { "description": "Database name", diff --git a/tests/integration_tests/db_engine_specs/postgres_tests.py b/tests/integration_tests/db_engine_specs/postgres_tests.py index d5edc4f9e897..18d39480ce02 100644 --- a/tests/integration_tests/db_engine_specs/postgres_tests.py +++ b/tests/integration_tests/db_engine_specs/postgres_tests.py @@ -478,6 +478,7 @@ def test_base_parameters_mixin(): json_schema = PostgresEngineSpec.parameters_json_schema() assert json_schema == { + "additionalProperties": False, "type": "object", "properties": { "encryption": { diff --git a/tests/integration_tests/db_engine_specs/redshift_tests.py b/tests/integration_tests/db_engine_specs/redshift_tests.py index 79cfe948e70f..7ed57707965f 100644 --- a/tests/integration_tests/db_engine_specs/redshift_tests.py +++ b/tests/integration_tests/db_engine_specs/redshift_tests.py @@ -210,7 +210,7 @@ def test_df_to_sql_no_dtype(self): test_array = np.array(data, dtype=numpy_dtype) - df = pd.DataFrame(test_array, columns=column_names) + df = pd.DataFrame(test_array, columns=column_names, dtype=object) df.to_sql = mock.MagicMock() with app.app_context(): diff --git a/tests/integration_tests/embedded/test_view.py b/tests/integration_tests/embedded/test_view.py index 4410036da101..3b3ceb3882c5 100644 --- a/tests/integration_tests/embedded/test_view.py +++ b/tests/integration_tests/embedded/test_view.py @@ -101,9 +101,16 @@ def test_get_embedded_dashboard_bootstrap_includes_allowed_domains( "superset.extensions.feature_flag_manager._feature_flags", EMBEDDED_SUPERSET=True, ) -def test_get_embedded_dashboard_referrer_not_allowed(client: FlaskClient[Any]): # noqa: F811 +@pytest.mark.parametrize( + "allowed_domain", ["test.example.com", "https://test.example.com"] +) +def test_get_embedded_dashboard_referrer_not_allowed( + client: FlaskClient[Any], # noqa: F811 + allowed_domain: str, +) -> None: + """Reject absent referrers whenever allowed domains restrict embedding.""" dash = db.session.query(Dashboard).filter_by(slug="births").first() - embedded = EmbeddedDashboardDAO.upsert(dash, ["test.example.com"]) + embedded = EmbeddedDashboardDAO.upsert(dash, [allowed_domain]) db.session.flush() uri = f"embedded/{embedded.uuid}" response = client.get(uri) diff --git a/tests/integration_tests/security/api_tests.py b/tests/integration_tests/security/api_tests.py index 0dcdc13102ef..ff193793369e 100644 --- a/tests/integration_tests/security/api_tests.py +++ b/tests/integration_tests/security/api_tests.py @@ -17,7 +17,7 @@ # isort:skip_file """Tests for security api methods""" -from unittest.mock import patch +from unittest.mock import MagicMock, patch import jwt import pytest @@ -136,13 +136,21 @@ def test_get_csrf_token_gamma(self): self.login(GAMMA_USERNAME) self._assert_get_csrf_token() - def test_get_csrf_unauthorized(self): + @patch.object(security_manager, "is_item_public", return_value=False) + def test_get_csrf_unauthorized(self, mock_is_item_public: MagicMock) -> None: """ - Security API: Test get CSRF no login + Security API: Require login when the endpoint is not public. """ uri = f"api/v1/{self.resource_name}/csrf_token/" response = self.client.get(uri) self.assert401(response) + mock_is_item_public.assert_called_with("can_read", "SecurityRestApi") + + @patch.object(security_manager, "is_item_public", return_value=True) + def test_get_csrf_public(self, mock_is_item_public: MagicMock) -> None: + """Security API: Allow anonymous CSRF tokens for a public endpoint.""" + self._assert_get_csrf_token() + mock_is_item_public.assert_called_with("can_read", "SecurityRestApi") def test_login(self): """ @@ -456,8 +464,9 @@ def test_guest_validator_hook_throws_exception(self): @with_config( { - "GUEST_TOKEN_VALIDATOR_HOOK": lambda x: len(x["rls"]) == 1 - and "tenant_id=" in x["rls"][0]["clause"] + "GUEST_TOKEN_VALIDATOR_HOOK": lambda x: ( + len(x["rls"]) == 1 and "tenant_id=" in x["rls"][0]["clause"] + ) } ) def test_guest_validator_hook_real_world_example_positive(self): @@ -472,8 +481,9 @@ def test_guest_validator_hook_real_world_example_positive(self): @with_config( { - "GUEST_TOKEN_VALIDATOR_HOOK": lambda x: len(x["rls"]) == 1 - and "tenant_id=" in x["rls"][0]["clause"] + "GUEST_TOKEN_VALIDATOR_HOOK": lambda x: ( + len(x["rls"]) == 1 and "tenant_id=" in x["rls"][0]["clause"] + ) } ) def test_guest_validator_hook_real_world_example_negative(self): diff --git a/tests/integration_tests/sql_lab/test_execute_sql_statements.py b/tests/integration_tests/sql_lab/test_execute_sql_statements.py index 752ad1e674df..d932414f3eef 100644 --- a/tests/integration_tests/sql_lab/test_execute_sql_statements.py +++ b/tests/integration_tests/sql_lab/test_execute_sql_statements.py @@ -59,7 +59,7 @@ def test_non_async_execute(non_async_example_db: Database, example_query: Query) assert example_query.tracking_url_raw -@patch("superset.sql_lab.results_backend") +@patch("superset.sql_lab.results_backend", new_callable=MagicMock) def test_results_backend_write_failure( mock_results_backend: MagicMock, async_example_db: Database, @@ -105,7 +105,7 @@ def test_results_backend_write_failure( assert mock_results_backend.set.called -@patch("superset.sql_lab.results_backend") +@patch("superset.sql_lab.results_backend", new_callable=MagicMock) def test_results_backend_write_success( mock_results_backend: MagicMock, async_example_db: Database, @@ -140,7 +140,7 @@ def test_results_backend_write_success( assert mock_results_backend.set.called -@patch("superset.sql_lab.results_backend") +@patch("superset.sql_lab.results_backend", new_callable=MagicMock) def test_results_backend_write_failure_sync_mode( mock_results_backend: MagicMock, non_async_example_db: Database, diff --git a/tests/integration_tests/thumbnails_tests.py b/tests/integration_tests/thumbnails_tests.py index 89b21e608a44..636d74b0f779 100644 --- a/tests/integration_tests/thumbnails_tests.py +++ b/tests/integration_tests/thumbnails_tests.py @@ -17,6 +17,7 @@ # from superset import db # from superset.models.dashboard import Dashboard +import multiprocessing import urllib.request from unittest import skipUnless from unittest.mock import patch @@ -49,7 +50,16 @@ DASHBOARD_URL = "/api/v1/dashboard/" +@skipUnless("fork" in multiprocessing.get_all_start_methods(), "Requires fork support") class TestThumbnailsSeleniumLive(LiveServerTestCase): + def _spawn_live_server(self) -> None: + """Use fork for Flask-Testing's non-picklable local server worker.""" + with patch( + "flask_testing.utils.multiprocessing.Process", + new=multiprocessing.get_context("fork").Process, + ): + super()._spawn_live_server() + def create_app(self): return app diff --git a/tests/unit_tests/commands/databases/columnar_reader_test.py b/tests/unit_tests/commands/databases/columnar_reader_test.py index 1172e47627c0..d87d9468f194 100644 --- a/tests/unit_tests/commands/databases/columnar_reader_test.py +++ b/tests/unit_tests/commands/databases/columnar_reader_test.py @@ -20,6 +20,7 @@ from zipfile import ZIP_DEFLATED, ZipFile import numpy as np +import pandas as pd import pytest from werkzeug.datastructures import FileStorage @@ -126,9 +127,8 @@ def test_columnar_reader_file_to_dataframe( expected_val = expected_values[i][j] actual_val = actual_values[i][j] - # Check if both values are NaN - if isinstance(expected_val, float) and isinstance(actual_val, float): - assert np.isnan(expected_val) == np.isnan(actual_val) + if pd.isna(expected_val): + assert pd.isna(actual_val) else: assert expected_val == actual_val file.close() @@ -141,7 +141,7 @@ def test_excel_reader_wrong_columns_to_read(): with pytest.raises(DatabaseUploadFailed) as ex: reader.file_to_dataframe(create_columnar_file(COLUMNAR_DATA)) assert ( - str(ex.value) + str(ex.value).replace(": large_string", ": string") == ( "Parsing error: No match for FieldRef.Name(xpto) in Name: string\n" "Age: int64\n" diff --git a/tests/unit_tests/commands/databases/csv_reader_test.py b/tests/unit_tests/commands/databases/csv_reader_test.py index 285823285da5..677155b559a2 100644 --- a/tests/unit_tests/commands/databases/csv_reader_test.py +++ b/tests/unit_tests/commands/databases/csv_reader_test.py @@ -454,9 +454,8 @@ def test_csv_reader_object_type_auto_inferring(): df = csv_reader.file_to_dataframe(create_csv_file(csv_data)) assert df.shape == (5, 3) - # pandas automatically infers the type if column_data_types is not informed - # if there's only one string in the column it converts the whole column to object - assert df["id"].dtype == "object" + assert pd.api.types.is_string_dtype(df["id"]) + assert df["id"].tolist() == ["25.5", "15", "123456789086", "abc", "4.75"] def test_csv_reader_float_type_auto_inferring(): @@ -874,7 +873,7 @@ def test_csv_reader_successful_string_conversion_with_floats(): df = csv_reader.file_to_dataframe(create_csv_file(csv_data)) assert df.shape == (4, 1) - assert df["id"].dtype == "object" + assert pd.api.types.is_string_dtype(df["id"]) assert df.iloc[0]["id"] == "1439403621518935563" assert df.iloc[1]["id"] == "42286989" assert df.iloc[2]["id"] == "1413660691875593351" diff --git a/tests/unit_tests/commands/importers/v1/utils_test.py b/tests/unit_tests/commands/importers/v1/utils_test.py index 6e362976af87..4cbb07dc57fc 100644 --- a/tests/unit_tests/commands/importers/v1/utils_test.py +++ b/tests/unit_tests/commands/importers/v1/utils_test.py @@ -634,14 +634,13 @@ def test_open_import_bundle_valid_zip() -> None: def test_open_import_bundle_corrupt_central_directory() -> None: """ - A ZIP with a corrupt central directory passes ``is_zipfile`` but fails to - open; it must raise ``IncorrectFormatError`` instead of ``BadZipFile``. + A ZIP with a corrupt central directory must raise ``IncorrectFormatError`` + instead of ``BadZipFile`` when opened. """ from superset.commands.importers.exceptions import IncorrectFormatError from superset.commands.importers.v1.utils import open_import_bundle corrupt = io.BytesIO(_make_bundle().replace(b"PK\x01\x02", b"XXXX")) - assert zipfile.is_zipfile(corrupt) with pytest.raises(IncorrectFormatError) as excinfo: with open_import_bundle(corrupt): diff --git a/tests/unit_tests/databases/api_test.py b/tests/unit_tests/databases/api_test.py index 119474e69e64..1a925c20768b 100644 --- a/tests/unit_tests/databases/api_test.py +++ b/tests/unit_tests/databases/api_test.py @@ -24,7 +24,7 @@ from typing import Any from unittest.mock import ANY, Mock from uuid import UUID -from zipfile import is_zipfile, ZipFile +from zipfile import ZipFile import pytest import yaml @@ -382,8 +382,9 @@ def test_database_connection( } }, "parameters_schema": { + "additionalProperties": False, "properties": { - "catalog": {"type": "object"}, + "catalog": {"type": "object", "additionalProperties": {}}, "oauth2_client_info": { "default": { "authorization_request_uri": "https://accounts.google.com/o/oauth2/v2/auth", @@ -786,7 +787,9 @@ def test_non_zip_import(client: Any, full_api_access: None) -> None: } -def test_corrupt_zip_import(client: Any, full_api_access: None) -> None: +def test_corrupt_zip_import( + client: Any, full_api_access: None, mocker: MockerFixture +) -> None: """ Test that a ZIP with a corrupt central directory returns a 422, not a 500. """ @@ -794,7 +797,7 @@ def test_corrupt_zip_import(client: Any, full_api_access: None) -> None: with ZipFile(buf, "w") as bundle: bundle.writestr("bundle/metadata.yaml", "version: 1.0.0\n") corrupt = BytesIO(buf.getvalue().replace(b"PK\x01\x02", b"XXXX")) - assert is_zipfile(corrupt) + mocker.patch("superset.databases.api.is_zipfile", return_value=True) form_data = { "formData": (corrupt, "corrupt.zip"), } diff --git a/tests/unit_tests/db_engine_specs/test_databricks.py b/tests/unit_tests/db_engine_specs/test_databricks.py index af8b50def4f3..40488ae938d3 100644 --- a/tests/unit_tests/db_engine_specs/test_databricks.py +++ b/tests/unit_tests/db_engine_specs/test_databricks.py @@ -151,6 +151,7 @@ def test_parameters_json_schema() -> None: assert json_schema == { "type": "object", + "additionalProperties": False, "properties": { "access_token": {"type": "string"}, "database": {"type": "string"}, diff --git a/tests/unit_tests/db_engine_specs/test_lint_metadata.py b/tests/unit_tests/db_engine_specs/test_lint_metadata.py new file mode 100644 index 000000000000..bf09be48f722 --- /dev/null +++ b/tests/unit_tests/db_engine_specs/test_lint_metadata.py @@ -0,0 +1,87 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +import ast +import runpy +from pathlib import Path +from typing import Any + +import pytest + + +@pytest.fixture(scope="module") +def metadata_linter() -> dict[str, Any]: + """Load the standalone linter without initializing the Superset application.""" + return runpy.run_path( + str( + Path(__file__).resolve().parents[3] + / "superset" + / "db_engine_specs" + / "lint_metadata.py" + ) + ) + + +@pytest.mark.parametrize( + ("expression", "expected"), + [ + ("'driver'", "driver"), + ("42", 42), + ("True", True), + ("None", None), + ("['driver', 42]", ["driver", 42]), + ( + "{'drivers': ['driver'], 'nested': {'enabled': True}}", + {"drivers": ["driver"], "nested": {"enabled": True}}, + ), + ("('driver', 42)", ("driver", 42)), + ("DatabaseCategory", "DatabaseCategory"), + ("DatabaseCategory.TRADITIONAL_RDBMS", "DatabaseCategory.TRADITIONAL_RDBMS"), + ("'driver' + 'name'", "drivername"), + ("1 + 2", None), + ("f'{driver}'", ""), + ("unknown()", None), + ], +) +def test_eval_ast_value_without_legacy_nodes( + metadata_linter: dict[str, Any], + monkeypatch: pytest.MonkeyPatch, + expression: str, + expected: Any, +) -> None: + """Evaluate metadata without the AST aliases removed in Python 3.14.""" + node = ast.parse(expression, mode="eval").body + monkeypatch.delattr(ast, "Str", raising=False) + monkeypatch.delattr(ast, "Num", raising=False) + assert metadata_linter["_eval_ast_value"](node) == expected + + +def test_discover_engine_metadata_without_legacy_nodes( + metadata_linter: dict[str, Any], monkeypatch: pytest.MonkeyPatch +) -> None: + """Discover real engine metadata without silently marking it unparseable.""" + monkeypatch.delattr(ast, "Str", raising=False) + monkeypatch.delattr(ast, "Num", raising=False) + specs = metadata_linter["get_all_engine_specs_ast"]() + postgres = next( + spec for spec in specs if spec["class_name"] == "PostgresEngineSpec" + ) + assert "_unparseable" not in postgres["metadata"] + assert postgres["metadata"]["categories"] + assert postgres["metadata"]["pypi_packages"] diff --git a/tests/unit_tests/docker_app_dependencies_test.py b/tests/unit_tests/docker_app_dependencies_test.py new file mode 100644 index 000000000000..1e6a7874f755 --- /dev/null +++ b/tests/unit_tests/docker_app_dependencies_test.py @@ -0,0 +1,87 @@ +import shlex +import subprocess +import tomllib +from pathlib import Path + +import pytest + + +@pytest.mark.parametrize("metadata", [False, True]) +def test_dependency_installation_selects_interpreter_contract(metadata: bool) -> None: + """Use metadata only for interpreters newer than the generated lock baseline.""" + repository = Path(__file__).resolve().parents[2] + script = (repository / "docker/install-app-dependencies.sh").read_text() + script = script.replace("/app/docker/pip-install.sh", "printf '%s\\n'").replace( + "python -c 'import sys; raise SystemExit(sys.version_info < (3, 15))'", + "true" if metadata else "false", + ) + result = subprocess.run( + ["/bin/bash", "-s", "--", "/source"], + input=script, + capture_output=True, + text=True, + check=True, + ) + arguments = result.stdout.splitlines() + assert arguments[0] == "--requires-build-essential" + if metadata: + assert arguments[1:] == [ + "--no-sources", + "--no-editable", + "/source/superset-core", + "/source", + ] + else: + assert arguments[1:] == ["-r", "requirements/base.txt"] + + +def test_cli_dependency_is_declared() -> None: + """The CLI must not rely on a test environment to supply rich.""" + repository = Path(__file__).resolve().parents[2] + project = tomllib.loads((repository / "pyproject.toml").read_text()) + assert "rich" in project["project"]["dependencies"] + + +@pytest.mark.parametrize("installer_exit", [0, 1]) +def test_readonly_build_copy_is_cleaned(tmp_path: Path, installer_exit: int) -> None: + """Leave mounted sources intact and remove build copies on either outcome.""" + source = tmp_path / "source" + source.mkdir() + marker = source / "marker" + marker.write_text("original") + repository = Path(__file__).resolve().parents[2] + script = (repository / "docker/install-app-dependencies.sh").read_text() + script = ( + script.replace( + "python -c 'import sys; raise SystemExit(sys.version_info < (3, 15))'", + "true", + ) + .replace( + '[[ -d "${SOURCE_ROOT}" && ! -w "${SOURCE_ROOT}" ]]', + "true", + ) + .replace("/app/docker/pip-install.sh", "install_dependencies") + ) + installer = ( + 'install_dependencies() { printf "%s\\n" "${SOURCE_ROOT}"; ' + 'test "$(cat "${SOURCE_ROOT}/marker")" = original; ' + 'printf changed > "${SOURCE_ROOT}/marker"; ' + f"return {installer_exit}; }}\n" + ) + result: subprocess.CompletedProcess[str] | subprocess.CalledProcessError + try: + result = subprocess.run( + ["/bin/bash", "-s"], + input=installer + f"set -- {shlex.quote(str(source))}\n" + script, + capture_output=True, + text=True, + check=True, + ) + except subprocess.CalledProcessError as error: + assert installer_exit == 1 + result = error + assert result.returncode == installer_exit + copied_source = Path(result.stdout.strip()) + assert copied_source != source + assert not copied_source.exists() + assert marker.read_text() == "original" diff --git a/tests/unit_tests/extensions/test_sqlalchemy.py b/tests/unit_tests/extensions/test_sqlalchemy.py index fdb41775c066..35e69e5953cc 100644 --- a/tests/unit_tests/extensions/test_sqlalchemy.py +++ b/tests/unit_tests/extensions/test_sqlalchemy.py @@ -380,12 +380,16 @@ def table2_fanout_match(session: Session, database2: "Database") -> Iterator[Non @with_feature_flags(ENABLE_SUPERSET_META_DB=True) +@pytest.mark.parametrize( + "cursor_mode", ["sqlalchemy", "iterate", "fetchone", "fetchmany", "fetchall"] +) def test_superset_joins_with_limit_drops_fanout_matches( mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch, app_context: None, table1_large: None, table2_fanout_match: None, + cursor_mode: str, ) -> None: """ Coverage note from review of #42598: the other join regression tests @@ -415,17 +419,34 @@ def test_superset_joins_with_limit_drops_fanout_matches( except Exception as e: pytest.skip(f"Superset dialect not available: {e}") - with engine.connect() as conn: - results = conn.execute( - text(""" + statement = """ SELECT t1.b, t2.b FROM "database1.table1_large" AS t1 JOIN "database2.table2_fanout_match" AS t2 ON t1.a = t2.a ORDER BY t2.b - """) - ) - assert list(results) == [(30, "thirty-x"), (30, "thirty-y")] + """ + if cursor_mode != "sqlalchemy": + raw_connection = engine.raw_connection() + try: + cursor = raw_connection.cursor() + cursor.execute(statement) + if cursor_mode == "iterate": + rows = list(cursor) + elif cursor_mode == "fetchone": + rows = [cursor.fetchone(), cursor.fetchone()] + assert cursor.fetchone() is None + elif cursor_mode == "fetchmany": + rows = cursor.fetchmany(2) + else: + rows = cursor.fetchall() + assert rows == [(30, "thirty-x"), (30, "thirty-y")] + finally: + raw_connection.close() + else: + with engine.connect() as conn: + results = conn.execute(text(statement)) + assert list(results) == [(30, "thirty-x"), (30, "thirty-y")] @with_feature_flags(ENABLE_SUPERSET_META_DB=True) diff --git a/tests/unit_tests/importexport/api_test.py b/tests/unit_tests/importexport/api_test.py index ebf249716a83..6ea648a165e6 100644 --- a/tests/unit_tests/importexport/api_test.py +++ b/tests/unit_tests/importexport/api_test.py @@ -270,7 +270,7 @@ def test_import_assets_corrupt_zip( with ZipFile(buf, "w") as bundle: bundle.writestr("bundle/metadata.yaml", "version: 1.0.0\n") corrupt = BytesIO(buf.getvalue().replace(b"PK\x01\x02", b"XXXX")) - assert is_zipfile(corrupt) + mocker.patch("superset.importexport.api.is_zipfile", return_value=True) form_data = { "bundle": (corrupt, "corrupt.zip"), } diff --git a/tests/unit_tests/mcp_service/chart/test_bullet_chart.py b/tests/unit_tests/mcp_service/chart/test_bullet_chart.py index 48e3447ec261..ae044d266b1f 100644 --- a/tests/unit_tests/mcp_service/chart/test_bullet_chart.py +++ b/tests/unit_tests/mcp_service/chart/test_bullet_chart.py @@ -1269,7 +1269,7 @@ def validate( assert result.success is True assert [row["Category"] for row in captured[0]] == [ - 1704164645123.456, + json_int_dttm_ser(source_values[0]), dublin_epoch_ms, 1710052200000.0, 2224774800000.0, @@ -1772,17 +1772,10 @@ def test_bullet_timestamp_categories_match_chart_data_wire_and_all_previews() -> np.datetime64("NaT"), ] expected = [ - "1704164645123.456", - "1704164645123.456", - "1704164645123.456", - "1730611800000", - "1730615400000", - "1730611800000", - "1730615400000", - "-0.0010000000000287557", - "0", + _javascript_number_string(json_int_dttm_ser(value)) for value in values[:9] + ] + [ "1704153600000", - "1704164645123.456", + _javascript_number_string(json_int_dttm_ser(pd.Timestamp(values[10]))), "1704153600000", "null", "null", diff --git a/tests/unit_tests/mcp_service/chart/test_bullet_tool_paths.py b/tests/unit_tests/mcp_service/chart/test_bullet_tool_paths.py index 85cad65700cf..6e467d4bb746 100644 --- a/tests/unit_tests/mcp_service/chart/test_bullet_tool_paths.py +++ b/tests/unit_tests/mcp_service/chart/test_bullet_tool_paths.py @@ -300,8 +300,10 @@ def test_bullet_temporal_mode_projects_timestamp_and_numpy_before_generic() -> N assert data == [ [ { - "timestamp": 1704164645123.456, - "numpy_timestamp": -0.0010000000000287557, + "timestamp": json.json_int_dttm_ser(timestamp), + "numpy_timestamp": json.json_int_dttm_ser( + pd.Timestamp(numpy_timestamp) + ), "date": 1704153600000.0, "pandas_nat": None, "numpy_nat": None, @@ -645,11 +647,11 @@ def run(self) -> dict[str, Any]: payload = json.loads(result.content[0].text) assert [row["Category"] for row in payload["data"]] == [ - 1704164645123.456, - 1704164645123.456, + json.json_int_dttm_ser(source_values[0]), + json.json_int_dttm_ser(source_values[1]), 1730611800000.0, 1730615400000.0, - -0.0010000000000287557, + json.json_int_dttm_ser(source_values[4]), None, dublin_epoch_ms, 1710052200123.456, @@ -1612,13 +1614,9 @@ async def test_bullet_timestamp_categories_from_dataframe_reach_fastmcp( # dateutil named-zone edge in this public-format invocation. rows = rows[9:12] expected = [ - "1704164645123.456", - "1704164645123.456", - "1730611800000", - "1730615400000", - "1730611800000", - "1730615400000", - "-0.0010000000000287557", + _javascript_number_string(json.json_int_dttm_ser(value)) + for value in source_values[:7] + ] + [ "1704153600000", "null", _javascript_number_string(dublin_epoch_ms), @@ -1715,13 +1713,13 @@ def run(self) -> dict[str, Any]: ) assert bar["encoding"]["tooltip"][0]["field"] == category_field assert [row["Category"] for row in specification["data"]["values"]] == [ - 1704164645123.456, - 1704164645123.456, + json.json_int_dttm_ser(source_values[0]), + json.json_int_dttm_ser(source_values[1]), 1730611800000.0, 1730615400000.0, 1730611800000.0, 1730615400000.0, - -0.0010000000000287557, + json.json_int_dttm_ser(source_values[6]), 1704153600000.0, None, dublin_epoch_ms, diff --git a/tests/unit_tests/models/core_test.py b/tests/unit_tests/models/core_test.py index 8e97d9a69381..4814fa5e0dc1 100644 --- a/tests/unit_tests/models/core_test.py +++ b/tests/unit_tests/models/core_test.py @@ -21,7 +21,6 @@ from datetime import datetime from typing import Any, Callable -import numpy import pandas as pd import pytest from flask import current_app @@ -2402,7 +2401,7 @@ def test_post_process_df_non_zero_based_index() -> None: df = pd.DataFrame({"col": [None, [1, 2], [3, 4]]}, dtype=object) df = df[df["col"].notna()] # index is now [1, 2], not [0, 1, 2] result = Database.post_process_df(df) - assert result["col"].dtype == numpy.object_ + assert pd.api.types.is_string_dtype(result["col"]) assert result["col"].iloc[0] == "[1, 2]" assert result["col"].iloc[1] == "[3, 4]" diff --git a/tests/unit_tests/models/helpers_test.py b/tests/unit_tests/models/helpers_test.py index 7a2f39fceacd..ba4829a43f9d 100644 --- a/tests/unit_tests/models/helpers_test.py +++ b/tests/unit_tests/models/helpers_test.py @@ -4285,7 +4285,7 @@ def test_normalize_df_replaces_infinity_without_downcasting_warning() -> None: assert result["metric"].dtype == np.float64 assert result["metric"].isna().tolist() == [False, True, True, False] assert result["count"].dtype == np.float64 - assert result["label"].dtype == object + pd.testing.assert_series_equal(result["label"], df["label"]) @pytest.mark.parametrize( diff --git a/tests/unit_tests/pandas_postprocessing/test_rolling.py b/tests/unit_tests/pandas_postprocessing/test_rolling.py index 019002de4de7..410193200afe 100644 --- a/tests/unit_tests/pandas_postprocessing/test_rolling.py +++ b/tests/unit_tests/pandas_postprocessing/test_rolling.py @@ -179,8 +179,8 @@ def test_rolling_after_pivot_with_single_metric(): pd.DataFrame( data={ "dttm": pd.to_datetime(["2019-01-01", "2019-01-02"]), - FLAT_COLUMN_SEPARATOR.join(["sum_metric", "UK"]): [5, 12], - FLAT_COLUMN_SEPARATOR.join(["sum_metric", "US"]): [6, 14], + FLAT_COLUMN_SEPARATOR.join(["sum_metric", "UK"]): [5.0, 12.0], + FLAT_COLUMN_SEPARATOR.join(["sum_metric", "US"]): [6.0, 14.0], } ) ) @@ -230,10 +230,10 @@ def test_rolling_after_pivot_with_multiple_metrics(): pd.DataFrame( data={ "dttm": pd.to_datetime(["2019-01-01", "2019-01-02"]), - FLAT_COLUMN_SEPARATOR.join(["count_metric", "UK"]): [1, 4], - FLAT_COLUMN_SEPARATOR.join(["count_metric", "US"]): [2, 6], - FLAT_COLUMN_SEPARATOR.join(["sum_metric", "UK"]): [5, 12], - FLAT_COLUMN_SEPARATOR.join(["sum_metric", "US"]): [6, 14], + FLAT_COLUMN_SEPARATOR.join(["count_metric", "UK"]): [1.0, 4.0], + FLAT_COLUMN_SEPARATOR.join(["count_metric", "US"]): [2.0, 6.0], + FLAT_COLUMN_SEPARATOR.join(["sum_metric", "UK"]): [5.0, 12.0], + FLAT_COLUMN_SEPARATOR.join(["sum_metric", "US"]): [6.0, 14.0], } ) ) diff --git a/tests/unit_tests/pandas_postprocessing/test_sort.py b/tests/unit_tests/pandas_postprocessing/test_sort.py index ea1e391863a8..891e97b1cbc9 100644 --- a/tests/unit_tests/pandas_postprocessing/test_sort.py +++ b/tests/unit_tests/pandas_postprocessing/test_sort.py @@ -34,10 +34,10 @@ def test_sort(): assert df.index[0] == "person99" df = sort(df=categories_df.set_index("name"), by="asc_idx") - assert df["asc_idx"][0] == 0 + assert df["asc_idx"].iloc[0] == 0 df = sort(df=categories_df.set_index("name"), by="asc_idx", ascending=False) - assert df["asc_idx"][0] == 100 + assert df["asc_idx"].iloc[0] == 100 df = sort(df=timeseries_df, is_sort_index=True) assert df.index[0] == parse("2019-01-01") diff --git a/tests/unit_tests/pandas_postprocessing/test_utils.py b/tests/unit_tests/pandas_postprocessing/test_utils.py index 1d7bd3e3a8fe..5645f7d80e55 100644 --- a/tests/unit_tests/pandas_postprocessing/test_utils.py +++ b/tests/unit_tests/pandas_postprocessing/test_utils.py @@ -127,6 +127,17 @@ def test_append_columns_with_empty_mapping_returns_a_copy(): assert series_to_list(post_df["y"]) == [1.0, 2.0] +def test_append_columns_preserves_replacement_dtype_and_index() -> None: + """Replacing integer data must preserve fractions, nulls and index alignment.""" + base_df = DataFrame({"y": [1, 2, 3]}, index=["a", "b", "c"]) + append_df = DataFrame({"y": [3.5, float("nan"), 1.5]}, index=["c", "b", "a"]) + + post_df = _append_columns(base_df, append_df, {"y": "y"}) + + assert post_df.equals(append_df.reindex(base_df.index)) + assert series_to_list(base_df["y"]) == [1, 2, 3] + + def test_scalar_to_sequence() -> None: """ None becomes an empty list. diff --git a/tests/unit_tests/sql/execution/test_async_results.py b/tests/unit_tests/sql/execution/test_async_results.py index bcc69ed9a380..2fd4ebae9a28 100644 --- a/tests/unit_tests/sql/execution/test_async_results.py +++ b/tests/unit_tests/sql/execution/test_async_results.py @@ -664,9 +664,9 @@ def test_timezone_cache_hit_dtype_matches_sync( mocker.patch.object(executor, "_generate_cache_key", return_value="tz-key") executor._store_in_cache(sync, sql, QueryOptions()) cached = executor._get_from_cache(sql, QueryOptions()) - if data_kind != "populated" and not isinstance(tz, ZoneInfo): - # Unparseable timezone metadata cannot be inferred without any values. - assert cached is None + if cached is None: + assert data_kind != "populated" + assert not isinstance(tz, ZoneInfo) return assert cached is not None for result in ( @@ -1040,7 +1040,7 @@ def test_async_unsupported_nested_types_use_arrow_strings( assert actual["value"][0] != sync.statements[0].data["value"][0] assert type(actual["value"][0]) is not type(sync.statements[0].data["value"][0]) assert actual["value"][1] == '{"a": [3]}' - assert actual["value"][2] is None + assert pd.isna(actual["value"][2]) def test_legacy_nested_oversized_integer_matches_sync() -> None: diff --git a/tests/unit_tests/utils/excel_tests.py b/tests/unit_tests/utils/excel_tests.py index ff1718119dff..65144fcd5311 100644 --- a/tests/unit_tests/utils/excel_tests.py +++ b/tests/unit_tests/utils/excel_tests.py @@ -41,7 +41,7 @@ def test_timezone_conversion() -> None: apply_column_types(df, [GenericDataType.TEMPORAL]) assert df["dt"].tolist() == [datetime(2023, 1, 1, 0, 0)] contents = df_to_excel(df) - exported = pd.read_excel(contents)["dt"][0] + exported = pd.read_excel(io.BytesIO(contents))["dt"][0] assert pd.Timestamp(exported) == pd.Timestamp("2023-01-01 00:00:00") @@ -65,7 +65,7 @@ def test_timezone_aware_index_exports() -> None: ) df = pd.DataFrame({"value": [1.5]}, index=index) contents = df_to_excel(df) - result = pd.read_excel(contents, index_col=0) + result = pd.read_excel(io.BytesIO(contents), index_col=0) assert pd.Timestamp(result.index[0]) == pd.Timestamp("2024-06-01 12:00:00") @@ -75,7 +75,7 @@ def test_quote_formulas() -> None: """ df = pd.DataFrame({"formula": ["=SUM(A1:A2)", "normal", "@SUM(A1:A2)"]}) contents = df_to_excel(df) - assert pd.read_excel(contents)["formula"].tolist() == [ + assert pd.read_excel(io.BytesIO(contents))["formula"].tolist() == [ "'=SUM(A1:A2)", "normal", "'@SUM(A1:A2)", @@ -94,7 +94,7 @@ def test_quote_formulas_in_headers_and_index() -> None: index=pd.Index(['=cmd|" /C calc"!A0'], name="label"), ) contents = df_to_excel(df) - result = pd.read_excel(contents, index_col=0) + result = pd.read_excel(io.BytesIO(contents), index_col=0) assert result.columns.tolist() == ["'=SUM(A1:A2)"] assert result.index.tolist() == ['\'=cmd|" /C calc"!A0'] @@ -246,7 +246,7 @@ def test_apply_column_types_with_duplicate_column_labels() -> None: assert is_numeric_dtype(df.iloc[:, 2]) contents = df_to_excel(df, index=False) - assert pd.read_excel(contents).shape == (2, 3) + assert pd.read_excel(io.BytesIO(contents)).shape == (2, 3) def test_quote_formulas_with_duplicate_column_labels() -> None: diff --git a/tests/unit_tests/utils/test_date_parsing.py b/tests/unit_tests/utils/test_date_parsing.py index 685eaf43906c..17f962b0d42d 100644 --- a/tests/unit_tests/utils/test_date_parsing.py +++ b/tests/unit_tests/utils/test_date_parsing.py @@ -137,8 +137,9 @@ def test_epoch_format_invalid_values(caplog): assert "Unable to convert column epoch to datetime, ignoring" in caplog.text # The column should remain unchanged when conversion fails - assert df["epoch"].dtype == object - assert df["epoch"].iloc[0] == "not_a_number" + pd.testing.assert_series_equal( + df["epoch"], pd.Series(["not_a_number", "invalid", "abc"], name="epoch") + ) @pytest.mark.parametrize( From 34048a8561c88b0352c39668fe5b605aa03e3c50 Mon Sep 17 00:00:00 2001 From: luth2 <142568716+luth2@users.noreply.github.com> Date: Wed, 28 Oct 2026 18:50:56 +0100 Subject: [PATCH 2/6] fix(python): address temporal, boxplot and pivot review findings --- .../commands/dataset/importers/v1/utils.py | 51 +++++++++---------- .../utils/pandas_postprocessing/boxplot.py | 7 ++- superset/utils/pandas_postprocessing/pivot.py | 6 ++- .../commands/importers/v1/utils_test.py | 21 ++++++++ .../pandas_postprocessing/test_boxplot.py | 28 ++++++++++ .../pandas_postprocessing/test_pivot.py | 30 +++++++++++ 6 files changed, 114 insertions(+), 29 deletions(-) diff --git a/superset/commands/dataset/importers/v1/utils.py b/superset/commands/dataset/importers/v1/utils.py index 499719cb34e8..cc329193f2c2 100644 --- a/superset/commands/dataset/importers/v1/utils.py +++ b/superset/commands/dataset/importers/v1/utils.py @@ -770,32 +770,31 @@ def _convert_temporal_columns(df: pd.DataFrame, dtype: dict[str, Any]) -> None: for column_name, sqla_type in dtype.items(): if isinstance(sqla_type, (Date, DateTime)): try: - df[column_name] = pd.to_datetime(df[column_name]).dt.as_unit("ns") - except OutOfBoundsDatetime: - # Row-level fallback: coerce only OOB values; re-raise for malformed - # strings. Whole-column errors="coerce" would silently swallow - # malformed values that happen to share a column with an OOB date. - original = df[column_name].copy() - result = [] - for val in original: - if pd.isna(val): - result.append(pd.NaT) - continue - try: - result.append(pd.to_datetime(val).as_unit("ns")) - except OutOfBoundsDatetime: - result.append(pd.NaT) - # Other exceptions (e.g. malformed strings) propagate - converted = pd.Series(result, index=original.index) - n_coerced = int(converted.isna().sum() - original.isna().sum()) - if n_coerced > 0: - logger.warning( - "Coerced %d out-of-bounds datetime value(s) " - "in column '%s' to NaT", - n_coerced, - column_name, - ) - df[column_name] = converted + converted = pd.to_datetime(df[column_name]) + if pd.api.types.is_datetime64_any_dtype(converted): + df[column_name] = converted.dt.as_unit("ns") + continue + except ValueError: + pass + original = df[column_name].copy() + result = [] + for val in original: + if pd.isna(val): + result.append(pd.NaT) + continue + try: + result.append(pd.to_datetime(val).as_unit("ns")) + except OutOfBoundsDatetime: + result.append(pd.NaT) + converted = pd.Series(result, index=original.index) + n_coerced = int(converted.isna().sum() - original.isna().sum()) + if n_coerced > 0: + logger.warning( + "Coerced %d out-of-bounds datetime value(s) in column '%s' to NaT", + n_coerced, + column_name, + ) + df[column_name] = converted def _read_bounded(stream: Any, max_bytes: int) -> io.BytesIO: diff --git a/superset/utils/pandas_postprocessing/boxplot.py b/superset/utils/pandas_postprocessing/boxplot.py index 79cdbcb0eab4..d1d9afa8907c 100644 --- a/superset/utils/pandas_postprocessing/boxplot.py +++ b/superset/utils/pandas_postprocessing/boxplot.py @@ -136,6 +136,11 @@ def outliers(series: Series) -> set[float]: # that's used in the underlying function will fail for column in metrics: if is_object_dtype(df[column]) or is_string_dtype(df[column]): - df[column] = to_numeric(df[column], errors="coerce") + try: + df[column] = to_numeric(df[column], errors="raise") + except (TypeError, ValueError) as ex: + raise InvalidPostProcessingError( + _("Boxplot metrics must contain numeric values") + ) from ex return aggregate(df, groupby=groupby, aggregates=aggregates) diff --git a/superset/utils/pandas_postprocessing/pivot.py b/superset/utils/pandas_postprocessing/pivot.py index 57ad92130147..0957ce45a86d 100644 --- a/superset/utils/pandas_postprocessing/pivot.py +++ b/superset/utils/pandas_postprocessing/pivot.py @@ -403,7 +403,9 @@ def pivot( # pylint: disable=too-many-arguments # noqa: C901 if percent_mode is not None: df = _apply_show_values_as(df, percent_mode) - if combine_value_with_metric: - df = df.stack(level=0, future_stack=True).unstack() + if combine_value_with_metric and isinstance(df.columns, pd.MultiIndex): + df = df.reorder_levels([*range(1, df.columns.nlevels), 0], axis=1).sort_index( + axis=1 + ) return df diff --git a/tests/unit_tests/commands/importers/v1/utils_test.py b/tests/unit_tests/commands/importers/v1/utils_test.py index 4cbb07dc57fc..d4f7dc79bd79 100644 --- a/tests/unit_tests/commands/importers/v1/utils_test.py +++ b/tests/unit_tests/commands/importers/v1/utils_test.py @@ -33,6 +33,27 @@ class TestConvertTemporalColumns: + @pytest.mark.parametrize("out_of_bounds", [False, True]) + def test_mixed_offsets_preserved(self, out_of_bounds: bool) -> None: + from sqlalchemy import DateTime + + from superset.commands.dataset.importers.v1.utils import ( + _convert_temporal_columns, + ) + + values = ["2024-10-27T02:30:00+02:00", "2024-10-27T02:30:00+01:00"] + if out_of_bounds: + values.append("3118-01-01T00:00:00+01:00") + frame = pd.DataFrame({"ts": values}, index=range(10, 10 + len(values))) + _convert_temporal_columns(frame, {"ts": DateTime()}) + + assert frame["ts"].iloc[0] == pd.Timestamp(values[0]) + assert frame["ts"].iloc[1] == pd.Timestamp(values[1]) + assert frame["ts"].iloc[0].utcoffset() != frame["ts"].iloc[1].utcoffset() + assert list(frame.index) == list(range(10, 10 + len(values))) + if out_of_bounds: + assert pd.isna(frame["ts"].iloc[2]) + def test_normal_dates_converted(self) -> None: """Valid in-range dates are converted to datetime64 normally.""" from sqlalchemy import DateTime diff --git a/tests/unit_tests/pandas_postprocessing/test_boxplot.py b/tests/unit_tests/pandas_postprocessing/test_boxplot.py index 7bde4a2b0c36..c82cffca9838 100644 --- a/tests/unit_tests/pandas_postprocessing/test_boxplot.py +++ b/tests/unit_tests/pandas_postprocessing/test_boxplot.py @@ -16,6 +16,7 @@ # under the License. import warnings +import pandas as pd import pytest from superset.exceptions import InvalidPostProcessingError @@ -25,6 +26,33 @@ from tests.unit_tests.pandas_postprocessing.utils import series_to_list +@pytest.mark.parametrize("dtype", ["object", "str"]) +@pytest.mark.parametrize("whisker_type", list(PostProcessingBoxplotWhiskerType)) +def test_boxplot_rejects_malformed_metrics(dtype, whisker_type): + frame = pd.DataFrame({"value": pd.Series(["1", "invalid", None], dtype=dtype)}) + with pytest.raises(InvalidPostProcessingError, match="numeric"): + boxplot( + df=frame, + groupby=[], + metrics=["value"], + whisker_type=whisker_type, + percentiles=[1, 99], + ) + + +@pytest.mark.parametrize("dtype", ["object", "str"]) +def test_boxplot_accepts_numeric_strings_and_missing_values(dtype): + frame = pd.DataFrame({"value": pd.Series(["1", "3", None], dtype=dtype)}) + result = boxplot( + df=frame, + groupby=[], + metrics=["value"], + whisker_type=PostProcessingBoxplotWhiskerType.TUKEY, + ) + assert result["value__mean"].iloc[0] == 2 + assert result["value__median"].iloc[0] == 2 + + def test_boxplot_tukey(): df = boxplot( df=names_df, diff --git a/tests/unit_tests/pandas_postprocessing/test_pivot.py b/tests/unit_tests/pandas_postprocessing/test_pivot.py index 0292cdb75ef5..3895f6caa366 100644 --- a/tests/unit_tests/pandas_postprocessing/test_pivot.py +++ b/tests/unit_tests/pandas_postprocessing/test_pivot.py @@ -41,6 +41,36 @@ def test_pivot_without_columns(): assert df["idx_nulls"].sum() == 1050 +@pytest.mark.parametrize("drop_missing_columns", [False, True]) +@pytest.mark.parametrize("columns", [["category"], ["category", "country"]]) +def test_pivot_combine_preserves_column_pairs(drop_missing_columns, columns): + frame = DataFrame( + { + "row": ["first", "second"], + "category": ["A", "B"], + "country": ["US", "EU"], + "metric1": [1.0, np.nan], + "metric2": [np.nan, 2.0], + "empty": [np.nan, np.nan], + } + ) + options = { + "index": ["row"], + "columns": columns, + "aggregates": { + metric: {"operator": "mean"} for metric in ["metric1", "metric2", "empty"] + }, + "drop_missing_columns": drop_missing_columns, + } + expected = pivot(df=frame.copy(), **options) + expected.columns = expected.columns.reorder_levels([*range(1, len(columns) + 1), 0]) + actual = pivot(df=frame.copy(), combine_value_with_metric=True, **options) + + pd.testing.assert_frame_equal( + actual.sort_index(axis=1), expected.sort_index(axis=1) + ) + + def test_pivot_with_single_column(): """ Make sure pivot with single column returns correct DataFrame From 176ea8da122e09988f9e2567f105eb96ac87b5f2 Mon Sep 17 00:00:00 2001 From: luth2 <142568716+luth2@users.noreply.github.com> Date: Thu, 29 Oct 2026 07:58:04 +0100 Subject: [PATCH 3/6] fix(python): address compatibility review edge cases --- .../python-metadata-compatibility.yml | 29 +++++- Dockerfile | 9 +- docker/install-app-dependencies.sh | 18 +++- docs/admin_docs/installation/pypi.mdx | 96 +++---------------- .../utils/pandas_postprocessing/resample.py | 6 +- .../integration_tests/databases/api_tests.py | 33 ++++--- .../db_engine_specs/postgres_tests.py | 14 ++- tests/unit_tests/databases/api_test.py | 10 +- .../db_engine_specs/test_databricks.py | 9 +- .../docker_app_dependencies_test.py | 56 ++++++++++- .../pandas_postprocessing/test_resample.py | 17 ++++ 11 files changed, 188 insertions(+), 109 deletions(-) diff --git a/.github/workflows/python-metadata-compatibility.yml b/.github/workflows/python-metadata-compatibility.yml index d3ef3e379a3f..42fba3337f1b 100644 --- a/.github/workflows/python-metadata-compatibility.yml +++ b/.github/workflows/python-metadata-compatibility.yml @@ -14,6 +14,23 @@ on: - superset/utils/pandas_postprocessing/** - tests/unit_tests/pandas_postprocessing/** - tests/unit_tests/result_set_test.py + - superset/commands/importers/v1/utils.py + - superset/commands/dataset/importers/v1/utils.py + - tests/unit_tests/commands/importers/v1/utils_test.py + - superset/models/helpers.py + - tests/unit_tests/models/helpers_test.py + - superset/superset_typing.py + - tests/unit_tests/db_engine_specs/test_base.py + - superset/extensions/metadb.py + - tests/unit_tests/extensions/test_sqlalchemy.py + - tests/unit_tests/conftest.py + - Dockerfile + - docker/** + - docker-compose-image-tag.yml + - scripts/docker_release_target.py + - tests/unit_tests/docker_app_dependencies_test.py + - tests/unit_tests/docker_compose_image_tag_test.py + - tests/unit_tests/scripts/docker_release_target_test.py - superset-core/** workflow_dispatch: {} @@ -83,10 +100,20 @@ jobs: assert app.test_client().get('/health').status_code == 200 with app.app_context(): raise SystemExit(pytest.main([ - '--noconftest', '-o', 'addopts=', + '--confcutdir=tests/unit_tests', '-p', 'pytest_mock', + '-o', 'addopts=', 'tests/unit_tests/db_engine_specs/test_lint_metadata.py', 'tests/unit_tests/pandas_postprocessing', 'tests/unit_tests/result_set_test.py', + 'tests/unit_tests/commands/importers/v1/utils_test.py', + 'tests/unit_tests/models/helpers_test.py', + 'tests/unit_tests/db_engine_specs/test_base.py::test_resolve_column_type_prefers_cursor_type', + 'tests/unit_tests/db_engine_specs/test_base.py::test_fetch_data_no_result_set', + 'tests/unit_tests/db_engine_specs/test_base.py::test_fetch_data_with_result_set', + 'tests/unit_tests/extensions/test_sqlalchemy.py::test_superset_joins_with_limit_drops_fanout_matches', + 'tests/unit_tests/docker_app_dependencies_test.py', + 'tests/unit_tests/docker_compose_image_tag_test.py', + 'tests/unit_tests/scripts/docker_release_target_test.py', 'superset-core/tests', '-q', '-W', 'default::DeprecationWarning', ])) diff --git a/Dockerfile b/Dockerfile index 33fd7407f707..20c75d6675db 100644 --- a/Dockerfile +++ b/Dockerfile @@ -302,9 +302,16 @@ COPY requirements/base.txt requirements/ # Copy superset-core package needed for editable install in base.txt COPY superset-core superset-core +RUN --mount=type=cache,target=${SUPERSET_HOME}/.cache/uv \ + if python -c 'import sys; raise SystemExit(sys.version_info >= (3, 15))'; then \ + /app/docker/install-app-dependencies.sh /app; \ + fi + RUN --mount=type=cache,target=${SUPERSET_HOME}/.cache/uv \ --mount=type=bind,source=.,target=/src \ - /app/docker/install-app-dependencies.sh /src + if python -c 'import sys; raise SystemExit(sys.version_info < (3, 15))'; then \ + /app/docker/install-app-dependencies.sh /src; \ + fi # Copy compiled frontend assets and application source now that # dependencies have been resolved and cached above. diff --git a/docker/install-app-dependencies.sh b/docker/install-app-dependencies.sh index 671e935bad08..0c557b59566c 100644 --- a/docker/install-app-dependencies.sh +++ b/docker/install-app-dependencies.sh @@ -1,4 +1,20 @@ #!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. set -euo pipefail SOURCE_ROOT="${1:?Application source root is required}" @@ -11,7 +27,7 @@ if python -c 'import sys; raise SystemExit(sys.version_info < (3, 15))'; then SOURCE_ROOT="${BUILD_ROOT}" fi /app/docker/pip-install.sh --requires-build-essential \ - --no-sources --no-editable "${SOURCE_ROOT}/superset-core" "${SOURCE_ROOT}" + --no-sources "${SOURCE_ROOT}/superset-core" "${SOURCE_ROOT}" else /app/docker/pip-install.sh --requires-build-essential -r requirements/base.txt fi diff --git a/docs/admin_docs/installation/pypi.mdx b/docs/admin_docs/installation/pypi.mdx index 146c2d578e3d..ccab87a1d621 100644 --- a/docs/admin_docs/installation/pypi.mdx +++ b/docs/admin_docs/installation/pypi.mdx @@ -33,76 +33,9 @@ The focused tests Python 3.11 through 3.15. It installs the application without dependency overrides, checks dependency consistency, migrates a fresh SQLite metadata database, checks `/health`, and runs engine metadata, pandas post-processing -(including Prophet), result-set, and core SDK tests. The same 278 tests and fresh -SQLite migrations passed locally on Windows for all five interpreter versions, -including CPython 3.15.0 final. - -A local Podman container running Debian 13 with CPython 3.15.2 also passed -336 focused tests, including the additional DataFrame serialization tests. -Installation without dependency overrides, dependency consistency, a fresh -SQLite migration to head, and the application health check succeeded. The -extended-precision `longdouble` regression passes on Linux. - -The same isolated Linux environment also migrated a PostgreSQL 17 metadata -database to head and initialized the application and test users. Selected Redis 7 -cache and synchronous/asynchronous query tests passed with a real Celery prefork -worker. All 35 tests in the selected CSRF/login, SQL Lab execution, cache-manager, -chart-cache, and Celery groups passed, including CTAS and CVAS operations. The -test database requires the configured `sqllab_test_db` schema. The CSRF tests -explicitly distinguish public endpoint access from access requiring login, and -the SQL Lab tests use synchronous mocks for the results backend. - -A complete local run of `tests/common` and `tests/unit_tests` on CPython 3.15.2 -finished with 32,821 passed, 57 skipped, 5 expected failures, and no unexpected -failures. Translation tests require complete frontend sources with LF line -endings and a Git checkout; cron description expectations use `en_US.UTF-8`. -This final run includes the raw-cursor join truncation fix. All 22 dialect tests -also passed on both Python 3.11 and 3.15, covering SQLAlchemy, direct iteration, -and all three fetch methods. -The earlier follow-up regression group also passed 1,683 tests on Python 3.11, -and the nine selected CSRF/login and SQL Lab tests passed on that version. - -The production frontend build succeeded with Node 24.16.0 and webpack 5.112.0 -(four warnings). Chromium acceptance passed all 24 selected Playwright tests -without retries: login success and rejection, chart management and ZIP export, -dashboard management and import/export, three chart types rendered from real -queries (including an ECharts canvas-pixel assertion), and SQL Lab query, -tab, saved-query, dataset, and Explore workflows. The browser server used -Gunicorn, PostgreSQL 17, Redis 7, `TESTING=False`, and enabled CSRF protection. -The browser dataset used 1,200 locally generated `birth_names` rows; chart -management tests additionally used the bundled Slack dataset metadata, without -claiming Slack data-query coverage. - -Broader integration testing also covers the schema-free database SELECT route -without introducing a new permission, validates generated and published OpenAPI -path parameters, and rejects a missing embedded-dashboard referrer when an -allowed-origin list is configured. The referrer regression and the surrounding -database API/schema tests passed on Python 3.11 as well (41 selected tests). -The legacy Flask-Testing live server uses an explicit local `fork` context on -platforms that support it; this does not change application or Celery process -defaults. Thumbnail image generation remains outside this acceptance scope. - -A complete local run of `tests/integration_tests` on CPython 3.15.2 finished -with 3,131 passed, 178 skipped, and no failures in 673 seconds. It used a fresh -PostgreSQL 17 metadata database, Redis 7, and a real Celery prefork worker with -a Redis SQL Lab results backend. Skipped tests remain unvalidated; a passing -collection does not establish support for every optional database or feature. - -Earlier runs on the 8 GB Podman VM were invalidated by kernel-confirmed -out-of-memory worker termination. The successful final unit run used 16 GB; -browser and full unit acceptance ran separately. The local Hyper-V storage -also showed intermittent I/O errors, so this host is not a production reference. - -Integration reruns require a fresh, migrated test metadata database: interrupted -or focused runs can leave fixtures that invalidate exact object-count assertions. -Keep browser acceptance in a separate database and Redis namespace. A real SQL -Lab worker also needs `RESULTS_BACKEND`, in addition to the Celery broker and -result-backend settings; verify these in both the test and worker processes. - -All applicable pre-commit hooks passed for the changed files in Linux, including -MyPy, Ruff, formatting, YAML, and the advisory workflow audit. Pylint passed when -run explicitly against the changed application modules with `--jobs=1`; its -parallel serialization path uses a code-object attribute removed in Python 3.15. +(including Prophet), result-set, importer utility, model helper, cursor, join, +Docker configuration, and core SDK tests. This focused selection does not +establish support for every optional database or feature. For Python 3.15 testing, install from the source checkout's package metadata, rather than applying the generated Python 3.11 requirement pins: @@ -113,7 +46,7 @@ python -m pip install -e ./superset-core -e . Run this command from the repository root in an isolated Python 3.15 environment. Some dependencies require source builds, so a working C/C++ toolchain may be -needed. Windows validation used the x64 Visual Studio 2022 Build Tools. The +needed. On Windows, install compatible C/C++ build tools. The generated requirement files remain tied to the existing Python 3.11 deployment baseline and are not Python 3.15 lock files. @@ -126,29 +59,28 @@ the development target still uses its generated development requirements. Container acceptance must include the complete image build, dependency consistency, fresh metadata migration, authentication, and the workloads needed by the deployment. -A dependency-only image build is not a substitute for that acceptance. The image's +A complete Python 3.15 image build has not been verified by this focused workflow; +a dependency-only image build is not a substitute for that acceptance. The image's `/health` endpoint is a liveness check, not proof that PostgreSQL, Redis, or queued work is available; verify those workflows separately in readiness monitoring. For deployment-specific shutdown validation, send SIGTERM to the Gunicorn master and allow more than its configured graceful timeout before escalating. Validate -worker exits as well as the master's exit code. Reference tests on Python 3.11 -and 3.15 exposed a Gunicorn gthread keepalive-drain issue requiring deadline -escalation after requests. A separately validated alternative is +worker exits as well as the master's exit code. Gunicorn gthread workers can +encounter a keepalive-drain issue requiring deadline escalation after requests. +An alternative to evaluate for the deployment is `SERVER_WORKER_CLASS=sync` with `SERVER_THREADS_AMOUNT=1`; setting only the worker class while retaining multiple threads can select gthread again. Synchronous workers have different concurrency characteristics, so capacity must be measured for that configuration. -This validation is not a full production certification. The remote Linux CI -jobs, skipped integration cases, the full database-driver matrix, -live report delivery and screenshot workflows, and browser workflows -beyond the selected 24 acceptance cases still require validation. Production +The focused workflow is not a full production certification. The full +database-driver matrix, integration and browser workflows, live report delivery, +and screenshot generation require deployment-specific validation. Production acceptance must also cover the target deployment's authentication, secrets, network/TLS configuration, workload, backup/restore, and availability requirements. -An additional Windows -DataFrame test assumes extended `longdouble` precision unavailable on that -platform; Windows also exposes existing Unix-signal type-checking limitations. +Windows does not provide extended `longdouble` precision and has Unix-signal +type-checking limitations. Use an officially supported Python version for production deployments until these broader compatibility checks have been completed. diff --git a/superset/utils/pandas_postprocessing/resample.py b/superset/utils/pandas_postprocessing/resample.py index 222d1abe0b0e..7c7d8ec1831b 100644 --- a/superset/utils/pandas_postprocessing/resample.py +++ b/superset/utils/pandas_postprocessing/resample.py @@ -251,8 +251,10 @@ def resample( # pylint: disable=too-many-arguments _df = _df.fillna(fill_value) elif method == "linear": _df = df.resample(rule).asfreq() - numeric_columns = _df.select_dtypes(include="number").columns - _df[numeric_columns] = _df[numeric_columns].interpolate() + interpolated_columns = _df.select_dtypes( + include=["number", "datetime", "datetimetz"] + ).columns + _df[interpolated_columns] = _df[interpolated_columns].interpolate() else: _df = getattr(df.resample(rule), method)() if method in ("ffill", "bfill"): diff --git a/tests/integration_tests/databases/api_tests.py b/tests/integration_tests/databases/api_tests.py index cc5e2979ef54..628465fb3049 100644 --- a/tests/integration_tests/databases/api_tests.py +++ b/tests/integration_tests/databases/api_tests.py @@ -3890,6 +3890,16 @@ def test_function_names_sqlite(self): @with_config({"PREFERRED_DATABASES": ["PostgreSQL", "Google BigQuery"]}) @mock.patch("superset.databases.api.get_available_engine_specs") def test_available(self, get_available_engine_specs): + from apispec import APISpec + from apispec.ext.marshmallow import MarshmallowPlugin + from marshmallow import fields, Schema + + plugin = MarshmallowPlugin() + APISpec(title="test", version="1", openapi_version="3.0.2", plugins=[plugin]) + object_schema = plugin.converter.schema2jsonschema(Schema()) + query_schema = plugin.converter.field2property( + fields.Dict(keys=fields.Str(), values=fields.Raw()) + ) get_available_engine_specs.return_value = { PostgresEngineSpec: {"psycopg2"}, BigQueryEngineSpec: {"bigquery"}, @@ -3913,7 +3923,7 @@ def test_available(self, get_available_engine_specs): "engine": "postgresql", "name": "PostgreSQL", "parameters": { - "additionalProperties": False, + **object_schema, "properties": { "database": { "description": "Database name", @@ -3940,9 +3950,8 @@ def test_available(self, get_available_engine_specs): "type": "integer", }, "query": { - "additionalProperties": {}, + **query_schema, "description": "Additional parameters", - "type": "object", }, "ssh": { "description": "Use an ssh tunnel connection to the database", # noqa: E501 @@ -3983,7 +3992,7 @@ def test_available(self, get_available_engine_specs): "engine": "bigquery", "name": "Google BigQuery", "parameters": { - "additionalProperties": False, + **object_schema, "properties": { "credentials_info": { "description": ( @@ -3992,7 +4001,7 @@ def test_available(self, get_available_engine_specs): "type": "string", "x-encrypted-extra": True, }, - "query": {"type": "object", "additionalProperties": {}}, + "query": plugin.converter.field2property(fields.Dict()), }, "type": "object", }, @@ -4019,7 +4028,7 @@ def test_available(self, get_available_engine_specs): "engine": "redshift", "name": "Amazon Redshift", "parameters": { - "additionalProperties": False, + **object_schema, "properties": { "database": { "description": "Database name", @@ -4045,9 +4054,8 @@ def test_available(self, get_available_engine_specs): "type": "integer", }, "query": { - "additionalProperties": {}, + **query_schema, "description": "Additional parameters", - "type": "object", }, "ssh": { "description": "Use an ssh tunnel connection to the database", # noqa: E501 @@ -4085,9 +4093,9 @@ def test_available(self, get_available_engine_specs): "engine": "gsheets", "name": "Google Sheets", "parameters": { - "additionalProperties": False, + **object_schema, "properties": { - "catalog": {"type": "object", "additionalProperties": {}}, + "catalog": plugin.converter.field2property(fields.Dict()), "oauth2_client_info": { "default": { "authorization_request_uri": "https://accounts.google.com/o/oauth2/v2/auth", @@ -4138,7 +4146,7 @@ def test_available(self, get_available_engine_specs): "engine": "mysql", "name": "MySQL", "parameters": { - "additionalProperties": False, + **object_schema, "properties": { "database": { "description": "Database name", @@ -4164,9 +4172,8 @@ def test_available(self, get_available_engine_specs): "type": "integer", }, "query": { - "additionalProperties": {}, + **query_schema, "description": "Additional parameters", - "type": "object", }, "ssh": { "description": "Use an ssh tunnel connection to the database", # noqa: E501 diff --git a/tests/integration_tests/db_engine_specs/postgres_tests.py b/tests/integration_tests/db_engine_specs/postgres_tests.py index 18d39480ce02..01f29717c492 100644 --- a/tests/integration_tests/db_engine_specs/postgres_tests.py +++ b/tests/integration_tests/db_engine_specs/postgres_tests.py @@ -447,6 +447,12 @@ def test_cancel_query_failed(self, engine_mock): def test_base_parameters_mixin(): + from apispec import APISpec + from apispec.ext.marshmallow import MarshmallowPlugin + from marshmallow import fields, Schema + + plugin = MarshmallowPlugin() + APISpec(title="test", version="1", openapi_version="3.0.2", plugins=[plugin]) parameters = { "username": "username", "password": "password", @@ -478,8 +484,7 @@ def test_base_parameters_mixin(): json_schema = PostgresEngineSpec.parameters_json_schema() assert json_schema == { - "additionalProperties": False, - "type": "object", + **plugin.converter.schema2jsonschema(Schema()), "properties": { "encryption": { "type": "boolean", @@ -497,9 +502,10 @@ def test_base_parameters_mixin(): "password": {"type": "string", "nullable": True, "description": "Password"}, "username": {"type": "string", "nullable": True, "description": "Username"}, "query": { - "type": "object", + **plugin.converter.field2property( + fields.Dict(keys=fields.Str(), values=fields.Raw()) + ), "description": "Additional parameters", - "additionalProperties": {}, }, "ssh": { "description": "Use an ssh tunnel connection to the database", diff --git a/tests/unit_tests/databases/api_test.py b/tests/unit_tests/databases/api_test.py index 6a465243a15c..d417b351eb67 100644 --- a/tests/unit_tests/databases/api_test.py +++ b/tests/unit_tests/databases/api_test.py @@ -282,9 +282,15 @@ def test_database_connection( """ Test that connection info is only returned in ``api/v1/database/${id}/connection``. """ + from apispec import APISpec + from apispec.ext.marshmallow import MarshmallowPlugin + from marshmallow import fields, Schema + from superset.databases.api import DatabaseRestApi from superset.models.core import Database + plugin = MarshmallowPlugin() + APISpec(title="test", version="1", openapi_version="3.0.2", plugins=[plugin]) DatabaseRestApi.datamodel._session = session # create table for databases @@ -383,9 +389,9 @@ def test_database_connection( } }, "parameters_schema": { - "additionalProperties": False, + **plugin.converter.schema2jsonschema(Schema()), "properties": { - "catalog": {"type": "object", "additionalProperties": {}}, + "catalog": plugin.converter.field2property(fields.Dict()), "oauth2_client_info": { "default": { "authorization_request_uri": "https://accounts.google.com/o/oauth2/v2/auth", diff --git a/tests/unit_tests/db_engine_specs/test_databricks.py b/tests/unit_tests/db_engine_specs/test_databricks.py index 40488ae938d3..17437fb39501 100644 --- a/tests/unit_tests/db_engine_specs/test_databricks.py +++ b/tests/unit_tests/db_engine_specs/test_databricks.py @@ -145,13 +145,18 @@ def test_parameters_json_schema() -> None: """ test that the parameters schema can be converted to json """ + from apispec import APISpec + from apispec.ext.marshmallow import MarshmallowPlugin + from marshmallow import Schema + from superset.db_engine_specs.databricks import DatabricksNativeEngineSpec + plugin = MarshmallowPlugin() + APISpec(title="test", version="1", openapi_version="3.0.2", plugins=[plugin]) json_schema = DatabricksNativeEngineSpec.parameters_json_schema() assert json_schema == { - "type": "object", - "additionalProperties": False, + **plugin.converter.schema2jsonschema(Schema()), "properties": { "access_token": {"type": "string"}, "database": {"type": "string"}, diff --git a/tests/unit_tests/docker_app_dependencies_test.py b/tests/unit_tests/docker_app_dependencies_test.py index 1e6a7874f755..f2d6f1509234 100644 --- a/tests/unit_tests/docker_app_dependencies_test.py +++ b/tests/unit_tests/docker_app_dependencies_test.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import re import shlex import subprocess import tomllib @@ -6,6 +23,44 @@ import pytest +@pytest.mark.parametrize("metadata", [False, True]) +def test_docker_dependency_layers_preserve_legacy_cache(metadata: bool) -> None: + repository = Path(__file__).resolve().parents[2] + dockerfile = (repository / "Dockerfile").read_text() + layers = [ + layer.replace("\\\n", " ") + for layer in re.findall(r"^RUN (?:.*\\\n)*.*", dockerfile, re.MULTILINE) + if "/app/docker/install-app-dependencies.sh" in layer + ] + assert len(layers) == 2 + assert "type=bind" not in layers[0] + assert "type=bind,source=.,target=/src" in layers[1] + outputs = [] + for layer in layers: + script = layer[layer.index("if python") :] + script = ( + script.replace( + "python -c 'import sys; raise SystemExit(sys.version_info >= (3, 15))'", + "false" if metadata else "true", + ) + .replace( + "python -c 'import sys; raise SystemExit(sys.version_info < (3, 15))'", + "true" if metadata else "false", + ) + .replace("/app/docker/install-app-dependencies.sh", "printf '%s\\n'") + ) + outputs.append( + subprocess.run( + ["/bin/bash", "-s"], + input=script, + capture_output=True, + text=True, + check=True, + ).stdout + ) + assert outputs == (["", "/src\n"] if metadata else ["/app\n", ""]) + + @pytest.mark.parametrize("metadata", [False, True]) def test_dependency_installation_selects_interpreter_contract(metadata: bool) -> None: """Use metadata only for interpreters newer than the generated lock baseline.""" @@ -27,7 +82,6 @@ def test_dependency_installation_selects_interpreter_contract(metadata: bool) -> if metadata: assert arguments[1:] == [ "--no-sources", - "--no-editable", "/source/superset-core", "/source", ] diff --git a/tests/unit_tests/pandas_postprocessing/test_resample.py b/tests/unit_tests/pandas_postprocessing/test_resample.py index e43bc376ba73..8131c6225052 100644 --- a/tests/unit_tests/pandas_postprocessing/test_resample.py +++ b/tests/unit_tests/pandas_postprocessing/test_resample.py @@ -37,6 +37,23 @@ def test_resample_should_not_side_effect(): assert _timeseries_df.equals(timeseries_df) +@pytest.mark.parametrize("timezone", [None, "UTC"]) +def test_linear_resample_preserves_datetime_interpolation(timezone): + frame = pd.DataFrame( + { + "date": pd.date_range("2024-02-01", periods=2, freq="2D", tz=timezone), + "value": [1.0, 3.0], + "label": ["first", "last"], + }, + index=pd.to_datetime(["2024-01-01", "2024-01-03"]), + ) + result = pp.resample(df=frame, rule="D", method="linear") + assert result["date"].iloc[1] == pd.Timestamp("2024-02-02", tz=timezone) + assert result["date"].dtype == frame["date"].dtype + assert result["value"].iloc[1] == 2.0 + assert pd.isna(result["label"].iloc[1]) + + def test_resample(): post_df = pp.resample(df=timeseries_df, rule="1D", method="ffill") """ From bd9c46a5fa8b90c4c4c59e525c294aac18c1bc12 Mon Sep 17 00:00:00 2001 From: luth2 <142568716+luth2@users.noreply.github.com> Date: Thu, 29 Oct 2026 10:20:48 +0100 Subject: [PATCH 4/6] fix(python): resolve compatibility CI and review regressions --- .../python-metadata-compatibility.yml | 6 +++- docs/static/resources/openapi.json | 23 +------------- pyproject.toml | 1 + superset/common/utils/dataframe_utils.py | 7 ++--- superset/translations/messages.pot | 3 ++ .../unit_tests/common/test_dataframe_utils.py | 30 +++++++++++++------ tests/unit_tests/models/helpers_test.py | 6 ++++ 7 files changed, 40 insertions(+), 36 deletions(-) diff --git a/.github/workflows/python-metadata-compatibility.yml b/.github/workflows/python-metadata-compatibility.yml index 42fba3337f1b..8289215e2fb0 100644 --- a/.github/workflows/python-metadata-compatibility.yml +++ b/.github/workflows/python-metadata-compatibility.yml @@ -11,6 +11,8 @@ on: - tests/unit_tests/db_engine_specs/test_lint_metadata.py - pyproject.toml - superset/result_set.py + - superset/common/utils/dataframe_utils.py + - tests/unit_tests/common/test_dataframe_utils.py - superset/utils/pandas_postprocessing/** - tests/unit_tests/pandas_postprocessing/** - tests/unit_tests/result_set_test.py @@ -71,7 +73,6 @@ jobs: matrix: python-version: [ "3.11", "3.12", "3.13", "3.14", "3.15" ] env: - SUPERSET_HOME: ${{ runner.temp }}/superset-python-compatibility SUPERSET_SECRET_KEY: python-compatibility-ci-only-not-for-production FLASK_APP: superset steps: @@ -88,6 +89,8 @@ jobs: python -m pip install -e ./superset-core -e . pytest==7.4.4 pytest-mock freezegun parameterized responses rich prophet - name: Check dependency consistency run: python -m pip check + - name: Configure metadata directory + run: echo "SUPERSET_HOME=$RUNNER_TEMP/superset-python-compatibility" >> "$GITHUB_ENV" - name: Migrate a fresh metadata database run: python -m flask db upgrade - name: Test application health and runtime compatibility @@ -105,6 +108,7 @@ jobs: 'tests/unit_tests/db_engine_specs/test_lint_metadata.py', 'tests/unit_tests/pandas_postprocessing', 'tests/unit_tests/result_set_test.py', + 'tests/unit_tests/common/test_dataframe_utils.py', 'tests/unit_tests/commands/importers/v1/utils_test.py', 'tests/unit_tests/models/helpers_test.py', 'tests/unit_tests/db_engine_specs/test_base.py::test_resolve_column_type_prefers_cursor_type', diff --git a/docs/static/resources/openapi.json b/docs/static/resources/openapi.json index 83963bfeaf37..4e187c064d0b 100644 --- a/docs/static/resources/openapi.json +++ b/docs/static/resources/openapi.json @@ -4520,7 +4520,6 @@ "type": "object" }, "DashboardRestApi.get_list": { - "additionalProperties": false, "properties": { "certification_details": { "nullable": true, @@ -4605,7 +4604,6 @@ "type": "object" }, "DashboardRestApi.get_list.Subject": { - "additionalProperties": false, "properties": { "id": { "type": "integer" @@ -4625,7 +4623,6 @@ "type": "object" }, "DashboardRestApi.get_list.Subject1": { - "additionalProperties": false, "properties": { "id": { "type": "integer" @@ -4645,7 +4642,6 @@ "type": "object" }, "DashboardRestApi.get_list.Tag": { - "additionalProperties": false, "properties": { "id": { "type": "integer" @@ -4667,7 +4663,6 @@ "type": "object" }, "DashboardRestApi.get_list.User": { - "additionalProperties": false, "properties": { "first_name": { "maxLength": 64, @@ -4688,7 +4683,6 @@ "type": "object" }, "DashboardRestApi.get_list.User1": { - "additionalProperties": false, "properties": { "first_name": { "maxLength": 64, @@ -23750,21 +23744,6 @@ } }, "description": "SQL statement for a select star for table" - }, - "400": { - "$ref": "#/components/responses/400" - }, - "401": { - "$ref": "#/components/responses/401" - }, - "404": { - "$ref": "#/components/responses/404" - }, - "422": { - "$ref": "#/components/responses/422" - }, - "500": { - "$ref": "#/components/responses/500" } }, "security": [ @@ -23775,7 +23754,7 @@ "api_key": [] } ], - "summary": "Get database select star for table", + "summary": "Get database select star for table in the default schema", "tags": [ "Database" ] diff --git a/pyproject.toml b/pyproject.toml index 474c2c2b8077..25cf71cf0f9e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -541,6 +541,7 @@ dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" "superset/translations/utils.py" = ["TID251"] "superset/extensions/__init__.py" = ["TID251"] "superset/utils/json.py" = ["TID251"] +"tests/unit_tests/docker_app_dependencies_test.py" = ["S603"] "docker/*" = ["I"] # Docker config files have non-standard imports that vary by environment "superset/db_engine_specs/lib.py" = ["E501"] # Database config file with long description strings diff --git a/superset/common/utils/dataframe_utils.py b/superset/common/utils/dataframe_utils.py index 3a4fb7d0ef42..98f8e93baec6 100644 --- a/superset/common/utils/dataframe_utils.py +++ b/superset/common/utils/dataframe_utils.py @@ -20,9 +20,8 @@ from collections.abc import Collection from typing import Any, Literal, TYPE_CHECKING -import numpy as np import pandas as pd -from pandas.api.types import infer_dtype +from pandas.api.types import infer_dtype, is_object_dtype, is_string_dtype from superset.common.chart_data import ChartDataResultType from superset.utils.core import get_metric_names, JS_MAX_INTEGER @@ -70,9 +69,9 @@ def df_metrics_to_num(df: pd.DataFrame, query_object: QueryObject) -> None: def df_columns_to_num(df: pd.DataFrame, columns: Collection[str]) -> None: - """Soft-convert the named object columns to numeric, in place.""" + """Soft-convert the named object or string columns to numeric, in place.""" for col, dtype in df.dtypes.items(): - if dtype.type == np.object_ and col in columns: + if col in columns and (is_object_dtype(dtype) or is_string_dtype(dtype)): # soft-convert a metric column to numeric only if all # non-null values look numeric (e.g. ClickHouse returns # SUM() results as strings). Leaves truly non-numeric diff --git a/superset/translations/messages.pot b/superset/translations/messages.pot index a5f9df9eb159..066d4464820f 100644 --- a/superset/translations/messages.pot +++ b/superset/translations/messages.pot @@ -2734,6 +2734,9 @@ msgstr "" msgid "Box Plot" msgstr "" +msgid "Boxplot metrics must contain numeric values" +msgstr "" + msgid "Breakdowns" msgstr "" diff --git a/tests/unit_tests/common/test_dataframe_utils.py b/tests/unit_tests/common/test_dataframe_utils.py index 353072d28479..b27179381e7e 100644 --- a/tests/unit_tests/common/test_dataframe_utils.py +++ b/tests/unit_tests/common/test_dataframe_utils.py @@ -55,28 +55,40 @@ def test_is_datetime_series(): ) -def test_df_metrics_to_num_converts_string_numerics(): +@pytest.mark.parametrize("dtype", ["object", "str", "string"]) +@pytest.mark.parametrize("nullable", [False, True]) +def test_df_metrics_to_num_converts_string_numerics(dtype: str, nullable: bool) -> None: """Test that string-encoded numeric columns (e.g. from ClickHouse) are converted.""" query_object = MagicMock() query_object.metric_names = ["sum_col", "mixed_col", "text_col"] df = pd.DataFrame( { - "sum_col": pd.Series(["100", "200", "300"], dtype=object), - "mixed_col": pd.Series(["1", "not_a_number", "3"], dtype=object), - "text_col": pd.Series(["foo", "bar", "baz"], dtype=object), - "dim_col": pd.Series(["a", "b", "c"], dtype=object), + "sum_col": pd.Series( + ["100", "200", None if nullable else "300"], dtype=dtype + ), + "mixed_col": pd.Series(["1", "not_a_number", "3"], dtype=dtype), + "text_col": pd.Series(["foo", "bar", "baz"], dtype=dtype), + "dim_col": pd.Series(["10", "20", "30"], dtype=dtype), } ) + original = df.copy() dataframe_utils.df_metrics_to_num(df, query_object) # sum_col: all numeric strings -> should be converted to numeric assert pd.api.types.is_numeric_dtype(df["sum_col"]), "sum_col should be numeric" - assert df["sum_col"].tolist() == [100.0, 200.0, 300.0] + assert df["sum_col"].iloc[:2].tolist() == [100, 200] + if nullable: + assert pd.isna(df["sum_col"].iloc[2]) + else: + assert df["sum_col"].iloc[2] == 300 + assert df.assign(group="group").groupby("group")["sum_col"].mean().iloc[0] == ( + 150 if nullable else 200 + ) # mixed_col: has non-numeric value -> should NOT be converted - assert df["mixed_col"].dtype == object, "mixed_col should remain object" + pd.testing.assert_series_equal(df["mixed_col"], original["mixed_col"]) # text_col: all non-numeric -> should NOT be converted - assert df["text_col"].dtype == object, "text_col should remain object" + pd.testing.assert_series_equal(df["text_col"], original["text_col"]) # dim_col: not in metric_names -> should NOT be touched - assert df["dim_col"].dtype == object, "dim_col should not be touched" + pd.testing.assert_series_equal(df["dim_col"], original["dim_col"]) @pytest.mark.parametrize( diff --git a/tests/unit_tests/models/helpers_test.py b/tests/unit_tests/models/helpers_test.py index f39dc2ecb911..39b59be13340 100644 --- a/tests/unit_tests/models/helpers_test.py +++ b/tests/unit_tests/models/helpers_test.py @@ -2495,6 +2495,9 @@ def test_get_sqla_query_raises_for_nonaggregate_custom_sql_metric_with_timeserie ) +@pytest.mark.parametrize( + "app", [{"FEATURE_FLAGS": {"ENABLE_TEMPLATE_PROCESSING": True}}], indirect=True +) def test_get_sqla_query_allows_jinja_templated_custom_sql_metric_with_columns( database: Database, ) -> None: @@ -2546,6 +2549,9 @@ def test_get_sqla_query_allows_jinja_templated_custom_sql_metric_with_columns( assert "{{" not in sql +@pytest.mark.parametrize( + "app", [{"FEATURE_FLAGS": {"ENABLE_TEMPLATE_PROCESSING": True}}], indirect=True +) def test_get_sqla_query_virtual_dataset_filter_values_drill_to_detail( database: Database, ) -> None: From 6b78fdb9895627587d1969c0b121cc6a78247b21 Mon Sep 17 00:00:00 2001 From: luth2 <142568716+luth2@users.noreply.github.com> Date: Sat, 31 Oct 2026 08:01:18 +0100 Subject: [PATCH 5/6] Fix temporal import and join regressions from Python compatibility review --- .../python-metadata-compatibility.yml | 3 ++ docs/admin_docs/installation/pypi.mdx | 2 +- .../commands/dataset/importers/v1/utils.py | 4 +-- superset/models/helpers.py | 6 ++++ .../databases/columnar_reader_test.py | 6 ++-- .../commands/importers/v1/utils_test.py | 2 +- tests/unit_tests/models/helpers_test.py | 30 +++++++++++++++++++ 7 files changed, 46 insertions(+), 7 deletions(-) diff --git a/.github/workflows/python-metadata-compatibility.yml b/.github/workflows/python-metadata-compatibility.yml index 8289215e2fb0..4c14614a3048 100644 --- a/.github/workflows/python-metadata-compatibility.yml +++ b/.github/workflows/python-metadata-compatibility.yml @@ -13,6 +13,8 @@ on: - superset/result_set.py - superset/common/utils/dataframe_utils.py - tests/unit_tests/common/test_dataframe_utils.py + - superset/utils/core.py + - tests/unit_tests/utils/test_date_parsing.py - superset/utils/pandas_postprocessing/** - tests/unit_tests/pandas_postprocessing/** - tests/unit_tests/result_set_test.py @@ -109,6 +111,7 @@ jobs: 'tests/unit_tests/pandas_postprocessing', 'tests/unit_tests/result_set_test.py', 'tests/unit_tests/common/test_dataframe_utils.py', + 'tests/unit_tests/utils/test_date_parsing.py', 'tests/unit_tests/commands/importers/v1/utils_test.py', 'tests/unit_tests/models/helpers_test.py', 'tests/unit_tests/db_engine_specs/test_base.py::test_resolve_column_type_prefers_cursor_type', diff --git a/docs/admin_docs/installation/pypi.mdx b/docs/admin_docs/installation/pypi.mdx index ccab87a1d621..0927c27da8b3 100644 --- a/docs/admin_docs/installation/pypi.mdx +++ b/docs/admin_docs/installation/pypi.mdx @@ -25,7 +25,7 @@ bound does not certify every newer interpreter. The source checkout includes experimental Python 3.15 compatibility. Its package requirements select NumPy `>=2.5.4,<2.6` and pandas `>=3.1.0,<3.2` on Python 3.15 and newer. Python 3.11 through 3.14 retain NumPy `>=1.23.5,<2.5` and pandas -`>=2.3.3,<2.4`; the minimum Python version has not changed. These source changes +`>=3.0.6,<3.1`; the minimum Python version has not changed. These source changes do not imply that an already published PyPI release supports Python 3.15. The focused diff --git a/superset/commands/dataset/importers/v1/utils.py b/superset/commands/dataset/importers/v1/utils.py index 1c65c2ed256e..148545507119 100644 --- a/superset/commands/dataset/importers/v1/utils.py +++ b/superset/commands/dataset/importers/v1/utils.py @@ -988,7 +988,7 @@ def _convert_temporal_columns(df: pd.DataFrame, dtype: dict[str, Any]) -> None: try: converted = pd.to_datetime(df[column_name]) if pd.api.types.is_datetime64_any_dtype(converted): - df[column_name] = converted.dt.as_unit("ns") + df[column_name] = converted continue except ValueError: pass @@ -999,7 +999,7 @@ def _convert_temporal_columns(df: pd.DataFrame, dtype: dict[str, Any]) -> None: result.append(pd.NaT) continue try: - result.append(pd.to_datetime(val).as_unit("ns")) + result.append(pd.to_datetime(val)) except OutOfBoundsDatetime: result.append(pd.NaT) converted = pd.Series(result, index=original.index) diff --git a/superset/models/helpers.py b/superset/models/helpers.py index 97b570bad459..c4b35f190f52 100644 --- a/superset/models/helpers.py +++ b/superset/models/helpers.py @@ -359,6 +359,12 @@ def _apply_temporal_join_format( """Apply a dataset column's declared datetime format to working values.""" if not datetime_format: return series + if _has_multiple_utc_offsets(series): + return series.map( + lambda value: pd.to_datetime( + value, format=datetime_format, errors="coerce", exact=False + ) + ) working_df = pd.DataFrame({column_name: series.copy()}) normalize_dttm_col( working_df, diff --git a/tests/unit_tests/commands/databases/columnar_reader_test.py b/tests/unit_tests/commands/databases/columnar_reader_test.py index 4366503ecf61..2c8ac42cf87c 100644 --- a/tests/unit_tests/commands/databases/columnar_reader_test.py +++ b/tests/unit_tests/commands/databases/columnar_reader_test.py @@ -143,10 +143,10 @@ def test_excel_reader_wrong_columns_to_read(): assert ( str(ex.value).replace(": large_string", ": string") == ( - "Parsing error: No match for FieldRef.Name(xpto) in Name: large_string\n" + "Parsing error: No match for FieldRef.Name(xpto) in Name: string\n" "Age: int64\n" - "City: large_string\n" - "Birth: large_string\n" + "City: string\n" + "Birth: string\n" "__fragment_index: int32\n" "__batch_index: int32\n" "__last_in_fragment: bool\n" diff --git a/tests/unit_tests/commands/importers/v1/utils_test.py b/tests/unit_tests/commands/importers/v1/utils_test.py index 24acf026a804..85c6b53927f6 100644 --- a/tests/unit_tests/commands/importers/v1/utils_test.py +++ b/tests/unit_tests/commands/importers/v1/utils_test.py @@ -53,7 +53,7 @@ def test_mixed_offsets_preserved(self, out_of_bounds: bool) -> None: assert frame["ts"].iloc[0].utcoffset() != frame["ts"].iloc[1].utcoffset() assert list(frame.index) == list(range(10, 10 + len(values))) if out_of_bounds: - assert pd.isna(frame["ts"].iloc[2]) + assert frame["ts"].iloc[2] == pd.Timestamp(values[2]) def test_normal_dates_converted(self) -> None: """Valid in-range dates are converted to datetime64 normally.""" diff --git a/tests/unit_tests/models/helpers_test.py b/tests/unit_tests/models/helpers_test.py index bd92ca1bd812..5ce1dbda4612 100644 --- a/tests/unit_tests/models/helpers_test.py +++ b/tests/unit_tests/models/helpers_test.py @@ -4295,6 +4295,36 @@ def test_normalize_df_replaces_infinity_without_downcasting_warning() -> None: pd.testing.assert_series_equal(result["label"], df["label"]) +@pytest.mark.parametrize("datetime_format", [None, "%Y-%m-%dT%H:%M:%S%z"]) +def test_temporal_join_preserves_local_clocks_across_offsets( + datetime_format: str | None, +) -> None: + import pandas as pd + + from superset.models.helpers import _coerce_temporal_join_series + + source = pd.Series( + ["2024-01-15T12:00:00-05:00", "2024-07-15T12:00:00-04:00"], + index=[10, 20], + name="ts", + ) + comparison = pd.Series( + ["2024-01-15T12:00:00-05:00", "2024-07-15T12:00:00-05:00"], + index=source.index, + name="ts", + ) + expected = [ + pd.Timestamp("2024-01-15T12:00:00"), + pd.Timestamp("2024-07-15T12:00:00"), + ] + + for values in (source, comparison): + converted = _coerce_temporal_join_series(values, "ts", datetime_format) + assert converted.tolist() == expected + assert converted.index.equals(values.index) + assert converted.name == values.name + + @pytest.mark.parametrize( "datetime_format,value", [ From e8e1b5cf857c20fd266d45ccac61c9a3023dfd8a Mon Sep 17 00:00:00 2001 From: luth2 <142568716+luth2@users.noreply.github.com> Date: Sat, 31 Oct 2026 11:50:45 +0100 Subject: [PATCH 6/6] Test ASF headers and validate earlier compatibility review fixes --- tests/unit_tests/docker_app_dependencies_test.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/unit_tests/docker_app_dependencies_test.py b/tests/unit_tests/docker_app_dependencies_test.py index f2d6f1509234..8811b9b166e0 100644 --- a/tests/unit_tests/docker_app_dependencies_test.py +++ b/tests/unit_tests/docker_app_dependencies_test.py @@ -23,6 +23,20 @@ import pytest +@pytest.mark.parametrize( + "relative_path", + [ + "docker/install-app-dependencies.sh", + "tests/unit_tests/docker_app_dependencies_test.py", + ], +) +def test_dependency_installation_files_have_asf_headers(relative_path: str) -> None: + repository = Path(__file__).resolve().parents[2] + header = "\n".join((repository / relative_path).read_text().splitlines()[:20]) + assert "Licensed to the Apache Software Foundation (ASF)" in header + assert "http://www.apache.org/licenses/LICENSE-2.0" in header + + @pytest.mark.parametrize("metadata", [False, True]) def test_docker_dependency_layers_preserve_legacy_cache(metadata: bool) -> None: repository = Path(__file__).resolve().parents[2]