diff --git a/.github/workflows/python-metadata-compatibility.yml b/.github/workflows/python-metadata-compatibility.yml new file mode 100644 index 000000000000..4c14614a3048 --- /dev/null +++ b/.github/workflows/python-metadata-compatibility.yml @@ -0,0 +1,127 @@ +name: Python-Metadata-Compatibility + +permissions: + contents: read + +on: + pull_request: + paths: + - .github/workflows/python-metadata-compatibility.yml + - superset/db_engine_specs/** + - tests/unit_tests/db_engine_specs/test_lint_metadata.py + - pyproject.toml + - superset/result_set.py + - superset/common/utils/dataframe_utils.py + - tests/unit_tests/common/test_dataframe_utils.py + - superset/utils/core.py + - tests/unit_tests/utils/test_date_parsing.py + - superset/utils/pandas_postprocessing/** + - tests/unit_tests/pandas_postprocessing/** + - tests/unit_tests/result_set_test.py + - superset/commands/importers/v1/utils.py + - superset/commands/dataset/importers/v1/utils.py + - tests/unit_tests/commands/importers/v1/utils_test.py + - superset/models/helpers.py + - tests/unit_tests/models/helpers_test.py + - superset/superset_typing.py + - tests/unit_tests/db_engine_specs/test_base.py + - superset/extensions/metadb.py + - tests/unit_tests/extensions/test_sqlalchemy.py + - tests/unit_tests/conftest.py + - Dockerfile + - docker/** + - docker-compose-image-tag.yml + - scripts/docker_release_target.py + - tests/unit_tests/docker_app_dependencies_test.py + - tests/unit_tests/docker_compose_image_tag_test.py + - tests/unit_tests/scripts/docker_release_target_test.py + - superset-core/** + workflow_dispatch: {} + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + +jobs: + metadata-parser: + runs-on: ubuntu-26.04 + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + python-version: [ "3.11", "3.12", "3.13", "3.14", "3.15" ] + env: + PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1" + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Setup Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: true + - name: Install standalone test dependencies + run: python -m pip install pytest==7.4.4 + - name: Test metadata parser without application dependencies + run: python -m pytest --noconftest -o addopts= tests/unit_tests/db_engine_specs/test_lint_metadata.py -q -W default::DeprecationWarning + + runtime-compatibility: + runs-on: ubuntu-26.04 + timeout-minutes: 25 + strategy: + fail-fast: false + matrix: + python-version: [ "3.11", "3.12", "3.13", "3.14", "3.15" ] + env: + SUPERSET_SECRET_KEY: python-compatibility-ci-only-not-for-production + FLASK_APP: superset + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Setup Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.python-version }} + - name: Install application and focused test dependencies + run: >- + python -m pip install -e ./superset-core -e . pytest==7.4.4 pytest-mock freezegun parameterized responses rich prophet + - name: Check dependency consistency + run: python -m pip check + - name: Configure metadata directory + run: echo "SUPERSET_HOME=$RUNNER_TEMP/superset-python-compatibility" >> "$GITHUB_ENV" + - name: Migrate a fresh metadata database + run: python -m flask db upgrade + - name: Test application health and runtime compatibility + run: | + python - <<'PY' + import pytest + from superset.app import create_app + + app = create_app() + assert app.test_client().get('/health').status_code == 200 + with app.app_context(): + raise SystemExit(pytest.main([ + '--confcutdir=tests/unit_tests', '-p', 'pytest_mock', + '-o', 'addopts=', + 'tests/unit_tests/db_engine_specs/test_lint_metadata.py', + 'tests/unit_tests/pandas_postprocessing', + 'tests/unit_tests/result_set_test.py', + 'tests/unit_tests/common/test_dataframe_utils.py', + 'tests/unit_tests/utils/test_date_parsing.py', + 'tests/unit_tests/commands/importers/v1/utils_test.py', + 'tests/unit_tests/models/helpers_test.py', + 'tests/unit_tests/db_engine_specs/test_base.py::test_resolve_column_type_prefers_cursor_type', + 'tests/unit_tests/db_engine_specs/test_base.py::test_fetch_data_no_result_set', + 'tests/unit_tests/db_engine_specs/test_base.py::test_fetch_data_with_result_set', + 'tests/unit_tests/extensions/test_sqlalchemy.py::test_superset_joins_with_limit_drops_fanout_matches', + 'tests/unit_tests/docker_app_dependencies_test.py', + 'tests/unit_tests/docker_compose_image_tag_test.py', + 'tests/unit_tests/scripts/docker_release_target_test.py', + 'superset-core/tests', '-q', + '-W', 'default::DeprecationWarning', + ])) + PY diff --git a/Dockerfile b/Dockerfile index a3b02332eb00..20c75d6675db 100644 --- a/Dockerfile +++ b/Dockerfile @@ -303,7 +303,15 @@ COPY requirements/base.txt requirements/ COPY superset-core superset-core RUN --mount=type=cache,target=${SUPERSET_HOME}/.cache/uv \ - /app/docker/pip-install.sh --requires-build-essential -r requirements/base.txt + if python -c 'import sys; raise SystemExit(sys.version_info >= (3, 15))'; then \ + /app/docker/install-app-dependencies.sh /app; \ + fi + +RUN --mount=type=cache,target=${SUPERSET_HOME}/.cache/uv \ + --mount=type=bind,source=.,target=/src \ + if python -c 'import sys; raise SystemExit(sys.version_info < (3, 15))'; then \ + /app/docker/install-app-dependencies.sh /src; \ + fi # Copy compiled frontend assets and application source now that # dependencies have been resolved and cached above. diff --git a/docker/install-app-dependencies.sh b/docker/install-app-dependencies.sh new file mode 100644 index 000000000000..0c557b59566c --- /dev/null +++ b/docker/install-app-dependencies.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +set -euo pipefail + +SOURCE_ROOT="${1:?Application source root is required}" + +if python -c 'import sys; raise SystemExit(sys.version_info < (3, 15))'; then + if [[ -d "${SOURCE_ROOT}" && ! -w "${SOURCE_ROOT}" ]]; then + BUILD_ROOT="$(mktemp -d)" + trap 'rm -rf "${BUILD_ROOT}"' EXIT + cp -a "${SOURCE_ROOT}/." "${BUILD_ROOT}/" + SOURCE_ROOT="${BUILD_ROOT}" + fi + /app/docker/pip-install.sh --requires-build-essential \ + --no-sources "${SOURCE_ROOT}/superset-core" "${SOURCE_ROOT}" +else + /app/docker/pip-install.sh --requires-build-essential -r requirements/base.txt +fi diff --git a/docs/admin_docs/installation/pypi.mdx b/docs/admin_docs/installation/pypi.mdx index cd0c7a61a618..0927c27da8b3 100644 --- a/docs/admin_docs/installation/pypi.mdx +++ b/docs/admin_docs/installation/pypi.mdx @@ -15,6 +15,75 @@ import useBaseUrl from '@docusaurus/useBaseUrl'; This page describes how to install Superset using the `apache_superset` package [published on PyPI](https://pypi.org/project/apache_superset/). +## Python Compatibility + +The minimum Python version is 3.11. The Python version classifiers in +[pyproject.toml](https://github.com/apache/superset/blob/master/pyproject.toml) +list the officially supported versions; the open-ended `requires-python` lower +bound does not certify every newer interpreter. + +The source checkout includes experimental Python 3.15 compatibility. Its package +requirements select NumPy `>=2.5.4,<2.6` and pandas `>=3.1.0,<3.2` on Python 3.15 +and newer. Python 3.11 through 3.14 retain NumPy `>=1.23.5,<2.5` and pandas +`>=3.0.6,<3.1`; the minimum Python version has not changed. These source changes +do not imply that an already published PyPI release supports Python 3.15. + +The focused +[compatibility workflow](https://github.com/apache/superset/blob/master/.github/workflows/python-metadata-compatibility.yml) +tests Python 3.11 through 3.15. It installs the application without dependency +overrides, checks dependency consistency, migrates a fresh SQLite metadata +database, checks `/health`, and runs engine metadata, pandas post-processing +(including Prophet), result-set, importer utility, model helper, cursor, join, +Docker configuration, and core SDK tests. This focused selection does not +establish support for every optional database or feature. + +For Python 3.15 testing, install from the source checkout's package metadata, +rather than applying the generated Python 3.11 requirement pins: + +```bash +python -m pip install -e ./superset-core -e . +``` + +Run this command from the repository root in an isolated Python 3.15 environment. +Some dependencies require source builds, so a working C/C++ toolchain may be +needed. On Windows, install compatible C/C++ build tools. The +generated requirement files remain tied to the existing Python 3.11 deployment +baseline and are not Python 3.15 lock files. + +When building the repository's `lean` or `superset` Docker target with a Python +3.15 base image, the application dependency stage installs both local packages +from their metadata instead of applying the Python 3.11 pins. Older interpreters +retain the requirements-based installation. This does not change the default +image's Python version or establish Python 3.15 support for every Docker target; +the development target still uses its generated development requirements. + +Container acceptance must include the complete image build, dependency consistency, +fresh metadata migration, authentication, and the workloads needed by the deployment. +A complete Python 3.15 image build has not been verified by this focused workflow; +a dependency-only image build is not a substitute for that acceptance. The image's +`/health` endpoint is a liveness check, not proof that PostgreSQL, Redis, or queued +work is available; verify those workflows separately in readiness monitoring. + +For deployment-specific shutdown validation, send SIGTERM to the Gunicorn master +and allow more than its configured graceful timeout before escalating. Validate +worker exits as well as the master's exit code. Gunicorn gthread workers can +encounter a keepalive-drain issue requiring deadline escalation after requests. +An alternative to evaluate for the deployment is +`SERVER_WORKER_CLASS=sync` with `SERVER_THREADS_AMOUNT=1`; setting only the worker +class while retaining multiple threads can select gthread again. Synchronous +workers have different concurrency characteristics, so capacity must be measured +for that configuration. + +The focused workflow is not a full production certification. The full +database-driver matrix, integration and browser workflows, live report delivery, +and screenshot generation require deployment-specific validation. Production +acceptance must also cover the target deployment's authentication, secrets, +network/TLS configuration, workload, backup/restore, and availability requirements. +Windows does not provide extended `longdouble` precision and has Unix-signal +type-checking limitations. +Use an officially supported Python version for production deployments until +these broader compatibility checks have been completed. + ## OS Dependencies Superset stores database connection information in its metadata database. For that purpose, we use diff --git a/docs/static/resources/openapi.json b/docs/static/resources/openapi.json index 07af26d4a7c7..efe84047e8b2 100644 --- a/docs/static/resources/openapi.json +++ b/docs/static/resources/openapi.json @@ -23732,15 +23732,6 @@ "schema": { "type": "string" } - }, - { - "description": "Table schema", - "in": "path", - "name": "schema_name", - "required": true, - "schema": { - "type": "string" - } } ], "responses": { @@ -23778,7 +23769,7 @@ "api_key": [] } ], - "summary": "Get database select star for table", + "summary": "Get database select star for table in the default schema", "tags": [ "Database" ] @@ -27758,15 +27749,6 @@ "/api/v1/log/recent_activity/": { "get": { "parameters": [ - { - "description": "The id of the user", - "in": "path", - "name": "user_id", - "required": true, - "schema": { - "type": "integer" - } - }, { "content": { "application/json": { diff --git a/pyproject.toml b/pyproject.toml index 2e348ba0057f..459f41bc4389 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,11 +84,13 @@ dependencies = [ "marshmallow-union>=0.1.15.post1", "msgpack>=1.2.3, <1.3", "nh3>=0.3.7, <0.4", - "numpy>=1.23.5, <2.5", + "numpy>=1.23.5, <2.5; python_version < '3.15'", + "numpy>=2.5.4, <2.6; python_version >= '3.15'", "packaging", # -------------------------- # pandas and related (wanting pandas[performance] without numba as it's 100+MB and not needed) - "pandas[excel]>=3.0.6, <3.1", + "pandas[excel]>=3.0.6, <3.1; python_version < '3.15'", + "pandas[excel]>=3.1.0, <3.2; python_version >= '3.15'", "bottleneck", # recommended performance dependency for pandas, see https://pandas.pydata.org/docs/getting_started/install.html#performance-dependencies-recommended # -------------------------- "parsedatetime", @@ -104,6 +106,7 @@ dependencies = [ "PyJWT>=2.15.1, <3.0", # raise floor to match resolved pin; closes SCA false-positive on 2.4.x-range CVEs already fixed in 2.13.0 "redis>=5.0.0, <9.0", "rison>=2.0.1, <3.0", + "rich", "shillelagh[gsheetsapi]>=1.4.5, <2.0", "sshtunnel>=0.5.1, <0.6", @@ -538,6 +541,7 @@ dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" "superset/translations/utils.py" = ["TID251"] "superset/extensions/__init__.py" = ["TID251"] "superset/utils/json.py" = ["TID251"] +"tests/unit_tests/docker_app_dependencies_test.py" = ["S603"] "docker/*" = ["I"] # Docker config files have non-standard imports that vary by environment "superset/db_engine_specs/lib.py" = ["E501"] # Database config file with long description strings diff --git a/superset/commands/dataset/importers/v1/utils.py b/superset/commands/dataset/importers/v1/utils.py index efa26c6d9e95..148545507119 100644 --- a/superset/commands/dataset/importers/v1/utils.py +++ b/superset/commands/dataset/importers/v1/utils.py @@ -986,32 +986,31 @@ def _convert_temporal_columns(df: pd.DataFrame, dtype: dict[str, Any]) -> None: for column_name, sqla_type in dtype.items(): if isinstance(sqla_type, (Date, DateTime)): try: - df[column_name] = pd.to_datetime(df[column_name]) - except OutOfBoundsDatetime: - # Row-level fallback: coerce only OOB values; re-raise for malformed - # strings. Whole-column errors="coerce" would silently swallow - # malformed values that happen to share a column with an OOB date. - original = df[column_name].copy() - result = [] - for val in original: - if pd.isna(val): - result.append(pd.NaT) - continue - try: - result.append(pd.to_datetime(val)) - except OutOfBoundsDatetime: - result.append(pd.NaT) - # Other exceptions (e.g. malformed strings) propagate - converted = pd.Series(result, index=original.index) - n_coerced = int(converted.isna().sum() - original.isna().sum()) - if n_coerced > 0: - logger.warning( - "Coerced %d out-of-bounds datetime value(s) " - "in column '%s' to NaT", - n_coerced, - column_name, - ) - df[column_name] = converted + converted = pd.to_datetime(df[column_name]) + if pd.api.types.is_datetime64_any_dtype(converted): + df[column_name] = converted + continue + except ValueError: + pass + original = df[column_name].copy() + result = [] + for val in original: + if pd.isna(val): + result.append(pd.NaT) + continue + try: + result.append(pd.to_datetime(val)) + except OutOfBoundsDatetime: + result.append(pd.NaT) + converted = pd.Series(result, index=original.index) + n_coerced = int(converted.isna().sum() - original.isna().sum()) + if n_coerced > 0: + logger.warning( + "Coerced %d out-of-bounds datetime value(s) in column '%s' to NaT", + n_coerced, + column_name, + ) + df[column_name] = converted def _read_bounded(stream: Any, max_bytes: int) -> io.BytesIO: diff --git a/superset/common/utils/dataframe_utils.py b/superset/common/utils/dataframe_utils.py index 3a4fb7d0ef42..98f8e93baec6 100644 --- a/superset/common/utils/dataframe_utils.py +++ b/superset/common/utils/dataframe_utils.py @@ -20,9 +20,8 @@ from collections.abc import Collection from typing import Any, Literal, TYPE_CHECKING -import numpy as np import pandas as pd -from pandas.api.types import infer_dtype +from pandas.api.types import infer_dtype, is_object_dtype, is_string_dtype from superset.common.chart_data import ChartDataResultType from superset.utils.core import get_metric_names, JS_MAX_INTEGER @@ -70,9 +69,9 @@ def df_metrics_to_num(df: pd.DataFrame, query_object: QueryObject) -> None: def df_columns_to_num(df: pd.DataFrame, columns: Collection[str]) -> None: - """Soft-convert the named object columns to numeric, in place.""" + """Soft-convert the named object or string columns to numeric, in place.""" for col, dtype in df.dtypes.items(): - if dtype.type == np.object_ and col in columns: + if col in columns and (is_object_dtype(dtype) or is_string_dtype(dtype)): # soft-convert a metric column to numeric only if all # non-null values look numeric (e.g. ClickHouse returns # SUM() results as strings). Leaves truly non-numeric diff --git a/superset/databases/api.py b/superset/databases/api.py index 1fd874d0861f..3442c1433e01 100644 --- a/superset/databases/api.py +++ b/superset/databases/api.py @@ -32,7 +32,13 @@ request, Response, ) -from flask_appbuilder.api import expose, protect, rison as parse_rison, safe +from flask_appbuilder.api import ( + expose, + permission_name, + protect, + rison as parse_rison, + safe, +) from flask_appbuilder.api.schemas import get_list_schema from flask_appbuilder.models.sqla.interface import SQLAInterface from marshmallow import ValidationError @@ -179,6 +185,7 @@ class DatabaseRestApi(BaseSupersetModelRestApi): "table_extra_metadata", "table_extra_metadata_deprecated", "select_star", + "select_star_without_schema", "catalogs", "schemas", "test_connection", @@ -198,7 +205,10 @@ class DatabaseRestApi(BaseSupersetModelRestApi): resource_name = "database" class_permission_name = "Database" - method_permission_name = MODEL_API_RW_METHOD_PERMISSION_MAP + method_permission_name = { + **MODEL_API_RW_METHOD_PERMISSION_MAP, + "select_star_without_schema": "read", + } allow_browser_login = True base_filters = [["id", DatabaseFilter, lambda: []]] show_columns = [ @@ -1240,6 +1250,44 @@ def table_extra_metadata(self, pk: int) -> FlaskResponse: return self.response(200, **payload) @expose("//select_star//", methods=("GET",)) + @protect() + @permission_name("read") + def select_star_without_schema(self, pk: int, table_name: str) -> FlaskResponse: + """Get a select star statement using the database's default schema. + --- + get: + summary: Get database select star for table in the default schema + parameters: + - in: path + schema: + type: integer + name: pk + description: The database id + - in: path + schema: + type: string + name: table_name + description: Table name + responses: + 200: + description: SQL statement for a select star for table + content: + application/json: + schema: + $ref: "#/components/schemas/SelectStarResponseSchema" + 400: + $ref: '#/components/responses/400' + 401: + $ref: '#/components/responses/401' + 404: + $ref: '#/components/responses/404' + 422: + $ref: '#/components/responses/422' + 500: + $ref: '#/components/responses/500' + """ + return self.select_star(pk=pk, table_name=table_name) + @expose("//select_star///", methods=("GET",)) @protect() @check_table_access @@ -2255,9 +2303,9 @@ def get_list( # pylint: disable=arguments-differ @safe @statsd_metrics @event_logger.log_this_with_context( - action=lambda self, - *args, - **kwargs: f"{self.__class__.__name__}.delete_oauth2_tokens", + action=lambda self, *args, **kwargs: ( + f"{self.__class__.__name__}.delete_oauth2_tokens" + ), log_to_statsd=False, ) @requires_json @@ -2312,9 +2360,9 @@ def delete_oauth2_tokens(self, pk: int) -> FlaskResponse: @safe @statsd_metrics @event_logger.log_this_with_context( - action=lambda self, - *args, - **kwargs: f"{self.__class__.__name__}.delete_all_oauth2_tokens", + action=lambda self, *args, **kwargs: ( + f"{self.__class__.__name__}.delete_all_oauth2_tokens" + ), log_to_statsd=False, ) def delete_all_oauth2_tokens(self, pk: int) -> FlaskResponse: diff --git a/superset/db_engine_specs/lint_metadata.py b/superset/db_engine_specs/lint_metadata.py index fff0146158f0..979286badc82 100644 --- a/superset/db_engine_specs/lint_metadata.py +++ b/superset/db_engine_specs/lint_metadata.py @@ -337,10 +337,6 @@ def _eval_ast_value(node: Any) -> Any: # noqa: C901 if isinstance(node, ast.Constant): return node.value - elif isinstance(node, ast.Str): # Python 3.7 compat - return node.s - elif isinstance(node, ast.Num): # Python 3.7 compat - return node.n elif isinstance(node, ast.List): return [_eval_ast_value(e) for e in node.elts] elif isinstance(node, ast.Dict): diff --git a/superset/embedded/view.py b/superset/embedded/view.py index 30ef3465bf8b..a71d57f19b11 100644 --- a/superset/embedded/view.py +++ b/superset/embedded/view.py @@ -63,7 +63,7 @@ def embedded( # validate request referrer in allowed domains is_referrer_allowed = not embedded.allowed_domains for domain in embedded.allowed_domains: - if same_origin(request.referrer, domain): + if request.referrer and same_origin(request.referrer, domain): is_referrer_allowed = True break diff --git a/superset/extensions/metadb.py b/superset/extensions/metadb.py index 13aa89021378..d07f518b8ea7 100644 --- a/superset/extensions/metadb.py +++ b/superset/extensions/metadb.py @@ -174,12 +174,30 @@ def cursor(*args: Any, **kwargs: Any) -> Any: raw_cursor = original_cursor(*args, **kwargs) original_execute = raw_cursor.execute + def iterate( + results: Iterator[tuple[Any, ...]], multi_table_query: bool + ) -> Iterator[tuple[Any, ...]]: + """Keep lazy row retrieval in the executing statement's context.""" + while True: + token = _executing_multi_table_query.set(multi_table_query) + try: + row = next(results) + except StopIteration: + return + finally: + _executing_multi_table_query.reset(token) + yield row + def execute(operation: str, parameters: Any = None) -> Any: - token = _executing_multi_table_query.set( - _count_referenced_tables(operation) > 1 - ) + multi_table_query = _count_referenced_tables(operation) > 1 + token = _executing_multi_table_query.set(multi_table_query) try: - return original_execute(operation, parameters) + result = original_execute(operation, parameters) + if raw_cursor._results is not None: + raw_cursor._results = iterate( + raw_cursor._results, multi_table_query + ) + return result finally: _executing_multi_table_query.reset(token) diff --git a/superset/models/helpers.py b/superset/models/helpers.py index 5016871dc9ea..df414fdb13f3 100644 --- a/superset/models/helpers.py +++ b/superset/models/helpers.py @@ -359,6 +359,12 @@ def _apply_temporal_join_format( """Apply a dataset column's declared datetime format to working values.""" if not datetime_format: return series + if _has_multiple_utc_offsets(series): + return series.map( + lambda value: pd.to_datetime( + value, format=datetime_format, errors="coerce", exact=False + ) + ) working_df = pd.DataFrame({column_name: series.copy()}) normalize_dttm_col( working_df, @@ -376,6 +382,8 @@ def _parse_temporal_join_values(series: pd.Series, column_name: str) -> pd.Serie """Parse working temporal values and wrap pandas parser-policy errors.""" if pd.api.types.is_datetime64_any_dtype(series): return series + if _has_multiple_utc_offsets(series): + return series.map(pd.Timestamp) try: return pd.to_datetime(series, errors="coerce", format="mixed") except ValueError as ex: diff --git a/superset/sql/parse.py b/superset/sql/parse.py index 3627d6362451..d15e87c9106c 100644 --- a/superset/sql/parse.py +++ b/superset/sql/parse.py @@ -1274,6 +1274,10 @@ class SQLStatement(BaseSQLStatement[exp.Expression]): # always parse into their own structured node instead. "ADD", "DELETE", + "ADD SQLBLACKLIST", + "DELETE SQLBLACKLIST", + "ADD BACKEND BLACKLIST", + "DELETE BACKEND BLACKLIST", # DDL head-tokens that sqlglot falls back to exp.Command for # whenever the body uses syntax it does not model # (CREATE EXTENSION/FUNCTION...LANGUAGE C/PUBLICATION/etc., diff --git a/superset/tasks/cron_util.py b/superset/tasks/cron_util.py index 2cfaf7f5c89b..0e20fb6cc363 100644 --- a/superset/tasks/cron_util.py +++ b/superset/tasks/cron_util.py @@ -68,6 +68,7 @@ def get_cron_description(cron: str) -> str: # "only on Tuesday" claims the day-of-week narrows the day-of-month clause; # it is an alternative to it, not an extra filter. clause = clause.removeprefix("only ") + clause = clause.removeprefix("or ") return description.replace(day_of_week_clause, f", or {clause}", 1) diff --git a/superset/translations/messages.pot b/superset/translations/messages.pot index 349600d099c6..27eb08b9fed6 100644 --- a/superset/translations/messages.pot +++ b/superset/translations/messages.pot @@ -2740,6 +2740,9 @@ msgstr "" msgid "Box Plot" msgstr "" +msgid "Boxplot metrics must contain numeric values" +msgstr "" + msgid "Breakdowns" msgstr "" diff --git a/superset/utils/core.py b/superset/utils/core.py index 08fac34b6a5c..2ae136d96060 100644 --- a/superset/utils/core.py +++ b/superset/utils/core.py @@ -2131,13 +2131,24 @@ def _process_datetime_column( # Parse with or without format (suppress warning if no format) if format_to_use: - converted = pd.to_datetime( - df[col.col_label], - utc=False, - format=format_to_use, - errors="coerce", - exact=False, - ) + try: + converted = pd.to_datetime( + df[col.col_label], + utc=False, + format=format_to_use, + errors="coerce", + exact=False, + ) + except ValueError as ex: + if "Mixed timezones" not in str(ex): + raise + converted = pd.to_datetime( + df[col.col_label], + utc=True, + format=format_to_use, + errors="coerce", + exact=False, + ) # A format that coerces every non-null value to NaT is a mismatch # (e.g. an epoch-millis column that inherited a '%Y' string format # when used as a chart's granularity). Assigning it would silently @@ -2154,13 +2165,25 @@ def _process_datetime_column( else: with warnings.catch_warnings(): warnings.filterwarnings("ignore", message=".*Could not infer format.*") - df[col.col_label] = pd.to_datetime( - df[col.col_label], - utc=False, - format=None, - errors="coerce", - exact=False, - ) + try: + converted = pd.to_datetime( + df[col.col_label], + utc=False, + format=None, + errors="coerce", + exact=False, + ) + except ValueError as ex: + if "Mixed timezones" not in str(ex): + raise + converted = pd.to_datetime( + df[col.col_label], + utc=True, + format=None, + errors="coerce", + exact=False, + ) + df[col.col_label] = converted def normalize_dttm_col( diff --git a/superset/utils/pandas_postprocessing/boxplot.py b/superset/utils/pandas_postprocessing/boxplot.py index 231cea18c1ff..779eb3fce564 100644 --- a/superset/utils/pandas_postprocessing/boxplot.py +++ b/superset/utils/pandas_postprocessing/boxplot.py @@ -137,6 +137,11 @@ def outliers(series: Series) -> set[float]: # include both legacy object dtype and pandas 3.0's default StringDtype. for column in metrics: if not is_numeric_dtype(df[column]): - df[column] = to_numeric(df[column], errors="coerce") + try: + df[column] = to_numeric(df[column], errors="raise") + except (TypeError, ValueError) as ex: + raise InvalidPostProcessingError( + _("Boxplot metrics must contain numeric values") + ) from ex return aggregate(df, groupby=groupby, aggregates=aggregates) diff --git a/superset/utils/pandas_postprocessing/pivot.py b/superset/utils/pandas_postprocessing/pivot.py index d86bdfcc7830..0957ce45a86d 100644 --- a/superset/utils/pandas_postprocessing/pivot.py +++ b/superset/utils/pandas_postprocessing/pivot.py @@ -403,10 +403,9 @@ def pivot( # pylint: disable=too-many-arguments # noqa: C901 if percent_mode is not None: df = _apply_show_values_as(df, percent_mode) - if combine_value_with_metric: - # stack() no longer introduces rows of NA values to drop (pandas >=2.1's - # "future" stack implementation, the only one since pandas 3.0), so the - # restored all-NaN metric rows are preserved without a dropna= argument. - df = df.stack(level=0).unstack() + if combine_value_with_metric and isinstance(df.columns, pd.MultiIndex): + df = df.reorder_levels([*range(1, df.columns.nlevels), 0], axis=1).sort_index( + axis=1 + ) return df diff --git a/superset/utils/pandas_postprocessing/resample.py b/superset/utils/pandas_postprocessing/resample.py index d4cea40e212e..d8d47426592e 100644 --- a/superset/utils/pandas_postprocessing/resample.py +++ b/superset/utils/pandas_postprocessing/resample.py @@ -14,6 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +import re from datetime import datetime, tzinfo from typing import Any, Optional, Union @@ -225,6 +226,11 @@ def resample( # pylint: disable=too-many-arguments _("Resample method should be in ") + ", ".join(RESAMPLE_METHOD) + "." ) + if legacy_rule := re.fullmatch(r"(\d*)(AS|A|T|H|M)(-[A-Z]+)?", rule): + count, alias, anchor = legacy_rule.groups() + aliases = {"T": "min", "H": "h", "M": "ME", "AS": "YS", "A": "YE"} + rule = f"{count}{aliases[alias]}{anchor or ''}" + tz = df.index.tz df = _pad_to_time_range( df, @@ -240,20 +246,15 @@ def resample( # pylint: disable=too-many-arguments _validate_projected_rows(df.index.min(), df.index.max(), rule) if method == "asfreq" and fill_value is not None: - # fill_value only makes sense for numeric (metric) columns; a non-numeric - # (dimension/label) column has no meaningful "zero" and, since pandas 3.0 - # enforces per-column dtype purity, writing a number into a string column - # now raises instead of silently coercing. _df = df.resample(rule).asfreq() - numeric_cols = _df.select_dtypes(include="number").columns - _df[numeric_cols] = _df[numeric_cols].fillna(fill_value) + numeric_columns = _df.select_dtypes(include="number").columns + _df[numeric_columns] = _df[numeric_columns].fillna(fill_value) elif method == "linear": - # interpolate() only supports numeric dtypes; leave non-numeric columns - # as the NaN that resample's upsampling introduced for them. - resampled = df.resample(rule).asfreq() - _df = resampled.copy() - numeric_cols = resampled.select_dtypes(include="number").columns - _df[numeric_cols] = resampled[numeric_cols].interpolate() + _df = df.resample(rule).asfreq() + interpolated_columns = _df.select_dtypes( + include=["number", "datetime", "datetimetz"] + ).columns + _df[interpolated_columns] = _df[interpolated_columns].interpolate() else: _df = getattr(df.resample(rule), method)() if method in ("ffill", "bfill"): diff --git a/superset/utils/pandas_postprocessing/rolling.py b/superset/utils/pandas_postprocessing/rolling.py index 71823d0d534a..20c28107b55b 100644 --- a/superset/utils/pandas_postprocessing/rolling.py +++ b/superset/utils/pandas_postprocessing/rolling.py @@ -81,7 +81,11 @@ def rolling( # pylint: disable=too-many-arguments :return: DataFrame with the rolling columns :raises InvalidPostProcessingError: If the request in incorrect """ - rolling_type_options = rolling_type_options or {} + option_names = {"quantile": "q"} if rolling_type == "quantile" else {} + rolling_type_options = { + option_names.get(name, name): value + for name, value in (rolling_type_options or {}).items() + } df_rolling = df.loc[:, columns.keys()] kwargs: dict[str, Union[str, int]] = {} diff --git a/superset/utils/pandas_postprocessing/utils.py b/superset/utils/pandas_postprocessing/utils.py index 4d75add92a04..15c9a433f9db 100644 --- a/superset/utils/pandas_postprocessing/utils.py +++ b/superset/utils/pandas_postprocessing/utils.py @@ -255,18 +255,8 @@ def _append_columns( if overwritten: # make sure to return a new DataFrame instead of changing the `base_df`. _base_df = base_df.copy() - # One column (list-wrapped, so a pivoted MultiIndex label's sub-columns - # resolve correctly) at a time rather than one bulk `.loc` set: pandas 3 - # still lets `.loc` downcast a lossless float result (e.g. a rolling sum - # with no NaN) back to the base column's int64, but raises on a value - # that truly needs a wider dtype (e.g. NaN into int64) instead of - # silently upcasting the way pandas <3 did -- so that case falls back to - # plain column replacement, which allows the dtype to change. for key in overwritten: - try: - _base_df.loc[:, [key]] = append_df[[key]] - except TypeError: - _base_df[key] = append_df[key] + _base_df[key] = append_df[key] if not appended: return _base_df # Select before renaming: `append_df` may carry columns the mapping does not diff --git a/superset/views/log/api.py b/superset/views/log/api.py index d4f58e8deed5..4c814bd4a70a 100644 --- a/superset/views/log/api.py +++ b/superset/views/log/api.py @@ -119,11 +119,6 @@ def recent_activity(self, **kwargs: Any) -> FlaskResponse: get: summary: Get recent activity data for a user parameters: - - in: path - schema: - type: integer - name: user_id - description: The id of the user - in: query name: q content: diff --git a/tests/integration_tests/databases/api_tests.py b/tests/integration_tests/databases/api_tests.py index 7edb992217a1..628465fb3049 100644 --- a/tests/integration_tests/databases/api_tests.py +++ b/tests/integration_tests/databases/api_tests.py @@ -3890,6 +3890,16 @@ def test_function_names_sqlite(self): @with_config({"PREFERRED_DATABASES": ["PostgreSQL", "Google BigQuery"]}) @mock.patch("superset.databases.api.get_available_engine_specs") def test_available(self, get_available_engine_specs): + from apispec import APISpec + from apispec.ext.marshmallow import MarshmallowPlugin + from marshmallow import fields, Schema + + plugin = MarshmallowPlugin() + APISpec(title="test", version="1", openapi_version="3.0.2", plugins=[plugin]) + object_schema = plugin.converter.schema2jsonschema(Schema()) + query_schema = plugin.converter.field2property( + fields.Dict(keys=fields.Str(), values=fields.Raw()) + ) get_available_engine_specs.return_value = { PostgresEngineSpec: {"psycopg2"}, BigQueryEngineSpec: {"bigquery"}, @@ -3913,6 +3923,7 @@ def test_available(self, get_available_engine_specs): "engine": "postgresql", "name": "PostgreSQL", "parameters": { + **object_schema, "properties": { "database": { "description": "Database name", @@ -3939,9 +3950,8 @@ def test_available(self, get_available_engine_specs): "type": "integer", }, "query": { - "additionalProperties": {}, + **query_schema, "description": "Additional parameters", - "type": "object", }, "ssh": { "description": "Use an ssh tunnel connection to the database", # noqa: E501 @@ -3982,6 +3992,7 @@ def test_available(self, get_available_engine_specs): "engine": "bigquery", "name": "Google BigQuery", "parameters": { + **object_schema, "properties": { "credentials_info": { "description": ( @@ -3990,7 +4001,7 @@ def test_available(self, get_available_engine_specs): "type": "string", "x-encrypted-extra": True, }, - "query": {"type": "object"}, + "query": plugin.converter.field2property(fields.Dict()), }, "type": "object", }, @@ -4017,6 +4028,7 @@ def test_available(self, get_available_engine_specs): "engine": "redshift", "name": "Amazon Redshift", "parameters": { + **object_schema, "properties": { "database": { "description": "Database name", @@ -4042,9 +4054,8 @@ def test_available(self, get_available_engine_specs): "type": "integer", }, "query": { - "additionalProperties": {}, + **query_schema, "description": "Additional parameters", - "type": "object", }, "ssh": { "description": "Use an ssh tunnel connection to the database", # noqa: E501 @@ -4082,8 +4093,9 @@ def test_available(self, get_available_engine_specs): "engine": "gsheets", "name": "Google Sheets", "parameters": { + **object_schema, "properties": { - "catalog": {"type": "object"}, + "catalog": plugin.converter.field2property(fields.Dict()), "oauth2_client_info": { "default": { "authorization_request_uri": "https://accounts.google.com/o/oauth2/v2/auth", @@ -4134,6 +4146,7 @@ def test_available(self, get_available_engine_specs): "engine": "mysql", "name": "MySQL", "parameters": { + **object_schema, "properties": { "database": { "description": "Database name", @@ -4159,9 +4172,8 @@ def test_available(self, get_available_engine_specs): "type": "integer", }, "query": { - "additionalProperties": {}, + **query_schema, "description": "Additional parameters", - "type": "object", }, "ssh": { "description": "Use an ssh tunnel connection to the database", # noqa: E501 diff --git a/tests/integration_tests/db_engine_specs/postgres_tests.py b/tests/integration_tests/db_engine_specs/postgres_tests.py index d5edc4f9e897..01f29717c492 100644 --- a/tests/integration_tests/db_engine_specs/postgres_tests.py +++ b/tests/integration_tests/db_engine_specs/postgres_tests.py @@ -447,6 +447,12 @@ def test_cancel_query_failed(self, engine_mock): def test_base_parameters_mixin(): + from apispec import APISpec + from apispec.ext.marshmallow import MarshmallowPlugin + from marshmallow import fields, Schema + + plugin = MarshmallowPlugin() + APISpec(title="test", version="1", openapi_version="3.0.2", plugins=[plugin]) parameters = { "username": "username", "password": "password", @@ -478,7 +484,7 @@ def test_base_parameters_mixin(): json_schema = PostgresEngineSpec.parameters_json_schema() assert json_schema == { - "type": "object", + **plugin.converter.schema2jsonschema(Schema()), "properties": { "encryption": { "type": "boolean", @@ -496,9 +502,10 @@ def test_base_parameters_mixin(): "password": {"type": "string", "nullable": True, "description": "Password"}, "username": {"type": "string", "nullable": True, "description": "Username"}, "query": { - "type": "object", + **plugin.converter.field2property( + fields.Dict(keys=fields.Str(), values=fields.Raw()) + ), "description": "Additional parameters", - "additionalProperties": {}, }, "ssh": { "description": "Use an ssh tunnel connection to the database", diff --git a/tests/integration_tests/embedded/test_view.py b/tests/integration_tests/embedded/test_view.py index 4410036da101..3b3ceb3882c5 100644 --- a/tests/integration_tests/embedded/test_view.py +++ b/tests/integration_tests/embedded/test_view.py @@ -101,9 +101,16 @@ def test_get_embedded_dashboard_bootstrap_includes_allowed_domains( "superset.extensions.feature_flag_manager._feature_flags", EMBEDDED_SUPERSET=True, ) -def test_get_embedded_dashboard_referrer_not_allowed(client: FlaskClient[Any]): # noqa: F811 +@pytest.mark.parametrize( + "allowed_domain", ["test.example.com", "https://test.example.com"] +) +def test_get_embedded_dashboard_referrer_not_allowed( + client: FlaskClient[Any], # noqa: F811 + allowed_domain: str, +) -> None: + """Reject absent referrers whenever allowed domains restrict embedding.""" dash = db.session.query(Dashboard).filter_by(slug="births").first() - embedded = EmbeddedDashboardDAO.upsert(dash, ["test.example.com"]) + embedded = EmbeddedDashboardDAO.upsert(dash, [allowed_domain]) db.session.flush() uri = f"embedded/{embedded.uuid}" response = client.get(uri) diff --git a/tests/integration_tests/security/api_tests.py b/tests/integration_tests/security/api_tests.py index 0dcdc13102ef..ff193793369e 100644 --- a/tests/integration_tests/security/api_tests.py +++ b/tests/integration_tests/security/api_tests.py @@ -17,7 +17,7 @@ # isort:skip_file """Tests for security api methods""" -from unittest.mock import patch +from unittest.mock import MagicMock, patch import jwt import pytest @@ -136,13 +136,21 @@ def test_get_csrf_token_gamma(self): self.login(GAMMA_USERNAME) self._assert_get_csrf_token() - def test_get_csrf_unauthorized(self): + @patch.object(security_manager, "is_item_public", return_value=False) + def test_get_csrf_unauthorized(self, mock_is_item_public: MagicMock) -> None: """ - Security API: Test get CSRF no login + Security API: Require login when the endpoint is not public. """ uri = f"api/v1/{self.resource_name}/csrf_token/" response = self.client.get(uri) self.assert401(response) + mock_is_item_public.assert_called_with("can_read", "SecurityRestApi") + + @patch.object(security_manager, "is_item_public", return_value=True) + def test_get_csrf_public(self, mock_is_item_public: MagicMock) -> None: + """Security API: Allow anonymous CSRF tokens for a public endpoint.""" + self._assert_get_csrf_token() + mock_is_item_public.assert_called_with("can_read", "SecurityRestApi") def test_login(self): """ @@ -456,8 +464,9 @@ def test_guest_validator_hook_throws_exception(self): @with_config( { - "GUEST_TOKEN_VALIDATOR_HOOK": lambda x: len(x["rls"]) == 1 - and "tenant_id=" in x["rls"][0]["clause"] + "GUEST_TOKEN_VALIDATOR_HOOK": lambda x: ( + len(x["rls"]) == 1 and "tenant_id=" in x["rls"][0]["clause"] + ) } ) def test_guest_validator_hook_real_world_example_positive(self): @@ -472,8 +481,9 @@ def test_guest_validator_hook_real_world_example_positive(self): @with_config( { - "GUEST_TOKEN_VALIDATOR_HOOK": lambda x: len(x["rls"]) == 1 - and "tenant_id=" in x["rls"][0]["clause"] + "GUEST_TOKEN_VALIDATOR_HOOK": lambda x: ( + len(x["rls"]) == 1 and "tenant_id=" in x["rls"][0]["clause"] + ) } ) def test_guest_validator_hook_real_world_example_negative(self): diff --git a/tests/integration_tests/sql_lab/test_execute_sql_statements.py b/tests/integration_tests/sql_lab/test_execute_sql_statements.py index 752ad1e674df..d932414f3eef 100644 --- a/tests/integration_tests/sql_lab/test_execute_sql_statements.py +++ b/tests/integration_tests/sql_lab/test_execute_sql_statements.py @@ -59,7 +59,7 @@ def test_non_async_execute(non_async_example_db: Database, example_query: Query) assert example_query.tracking_url_raw -@patch("superset.sql_lab.results_backend") +@patch("superset.sql_lab.results_backend", new_callable=MagicMock) def test_results_backend_write_failure( mock_results_backend: MagicMock, async_example_db: Database, @@ -105,7 +105,7 @@ def test_results_backend_write_failure( assert mock_results_backend.set.called -@patch("superset.sql_lab.results_backend") +@patch("superset.sql_lab.results_backend", new_callable=MagicMock) def test_results_backend_write_success( mock_results_backend: MagicMock, async_example_db: Database, @@ -140,7 +140,7 @@ def test_results_backend_write_success( assert mock_results_backend.set.called -@patch("superset.sql_lab.results_backend") +@patch("superset.sql_lab.results_backend", new_callable=MagicMock) def test_results_backend_write_failure_sync_mode( mock_results_backend: MagicMock, non_async_example_db: Database, diff --git a/tests/integration_tests/thumbnails_tests.py b/tests/integration_tests/thumbnails_tests.py index 89b21e608a44..636d74b0f779 100644 --- a/tests/integration_tests/thumbnails_tests.py +++ b/tests/integration_tests/thumbnails_tests.py @@ -17,6 +17,7 @@ # from superset import db # from superset.models.dashboard import Dashboard +import multiprocessing import urllib.request from unittest import skipUnless from unittest.mock import patch @@ -49,7 +50,16 @@ DASHBOARD_URL = "/api/v1/dashboard/" +@skipUnless("fork" in multiprocessing.get_all_start_methods(), "Requires fork support") class TestThumbnailsSeleniumLive(LiveServerTestCase): + def _spawn_live_server(self) -> None: + """Use fork for Flask-Testing's non-picklable local server worker.""" + with patch( + "flask_testing.utils.multiprocessing.Process", + new=multiprocessing.get_context("fork").Process, + ): + super()._spawn_live_server() + def create_app(self): return app diff --git a/tests/unit_tests/commands/databases/columnar_reader_test.py b/tests/unit_tests/commands/databases/columnar_reader_test.py index b3f0ccb9c899..2c8ac42cf87c 100644 --- a/tests/unit_tests/commands/databases/columnar_reader_test.py +++ b/tests/unit_tests/commands/databases/columnar_reader_test.py @@ -20,6 +20,7 @@ from zipfile import ZIP_DEFLATED, ZipFile import numpy as np +import pandas as pd import pytest from werkzeug.datastructures import FileStorage @@ -126,9 +127,8 @@ def test_columnar_reader_file_to_dataframe( expected_val = expected_values[i][j] actual_val = actual_values[i][j] - # Check if both values are NaN - if isinstance(expected_val, float) and isinstance(actual_val, float): - assert np.isnan(expected_val) == np.isnan(actual_val) + if pd.isna(expected_val): + assert pd.isna(actual_val) else: assert expected_val == actual_val file.close() @@ -141,12 +141,12 @@ def test_excel_reader_wrong_columns_to_read(): with pytest.raises(DatabaseUploadFailed) as ex: reader.file_to_dataframe(create_columnar_file(COLUMNAR_DATA)) assert ( - str(ex.value) + str(ex.value).replace(": large_string", ": string") == ( - "Parsing error: No match for FieldRef.Name(xpto) in Name: large_string\n" + "Parsing error: No match for FieldRef.Name(xpto) in Name: string\n" "Age: int64\n" - "City: large_string\n" - "Birth: large_string\n" + "City: string\n" + "Birth: string\n" "__fragment_index: int32\n" "__batch_index: int32\n" "__last_in_fragment: bool\n" diff --git a/tests/unit_tests/commands/databases/csv_reader_test.py b/tests/unit_tests/commands/databases/csv_reader_test.py index 29eb5cf31e58..00f1aa5c9548 100644 --- a/tests/unit_tests/commands/databases/csv_reader_test.py +++ b/tests/unit_tests/commands/databases/csv_reader_test.py @@ -457,6 +457,7 @@ def test_csv_reader_object_type_auto_inferring(): # pandas automatically infers the type if column_data_types is not informed # if there's only one string in the column it converts the whole column to string assert df["id"].dtype == "str" + assert df["id"].tolist() == ["25.5", "15", "123456789086", "abc", "4.75"] def test_csv_reader_float_type_auto_inferring(): diff --git a/tests/unit_tests/commands/importers/v1/utils_test.py b/tests/unit_tests/commands/importers/v1/utils_test.py index 445062bc9fb0..85c6b53927f6 100644 --- a/tests/unit_tests/commands/importers/v1/utils_test.py +++ b/tests/unit_tests/commands/importers/v1/utils_test.py @@ -34,6 +34,27 @@ class TestConvertTemporalColumns: + @pytest.mark.parametrize("out_of_bounds", [False, True]) + def test_mixed_offsets_preserved(self, out_of_bounds: bool) -> None: + from sqlalchemy import DateTime + + from superset.commands.dataset.importers.v1.utils import ( + _convert_temporal_columns, + ) + + values = ["2024-10-27T02:30:00+02:00", "2024-10-27T02:30:00+01:00"] + if out_of_bounds: + values.append("3118-01-01T00:00:00+01:00") + frame = pd.DataFrame({"ts": values}, index=range(10, 10 + len(values))) + _convert_temporal_columns(frame, {"ts": DateTime()}) + + assert frame["ts"].iloc[0] == pd.Timestamp(values[0]) + assert frame["ts"].iloc[1] == pd.Timestamp(values[1]) + assert frame["ts"].iloc[0].utcoffset() != frame["ts"].iloc[1].utcoffset() + assert list(frame.index) == list(range(10, 10 + len(values))) + if out_of_bounds: + assert frame["ts"].iloc[2] == pd.Timestamp(values[2]) + def test_normal_dates_converted(self) -> None: """Valid in-range dates are converted to datetime64 normally.""" from sqlalchemy import DateTime @@ -679,14 +700,13 @@ def test_open_import_bundle_valid_zip() -> None: def test_open_import_bundle_corrupt_central_directory() -> None: """ - A ZIP with a corrupt central directory passes ``is_zipfile`` but fails to - open; it must raise ``IncorrectFormatError`` instead of ``BadZipFile``. + A ZIP with a corrupt central directory must raise ``IncorrectFormatError`` + instead of ``BadZipFile`` when opened. """ from superset.commands.importers.exceptions import IncorrectFormatError from superset.commands.importers.v1.utils import open_import_bundle corrupt = io.BytesIO(_make_bundle().replace(b"PK\x01\x02", b"XXXX")) - assert zipfile.is_zipfile(corrupt) with pytest.raises(IncorrectFormatError) as excinfo: with open_import_bundle(corrupt): diff --git a/tests/unit_tests/common/test_dataframe_utils.py b/tests/unit_tests/common/test_dataframe_utils.py index 353072d28479..b27179381e7e 100644 --- a/tests/unit_tests/common/test_dataframe_utils.py +++ b/tests/unit_tests/common/test_dataframe_utils.py @@ -55,28 +55,40 @@ def test_is_datetime_series(): ) -def test_df_metrics_to_num_converts_string_numerics(): +@pytest.mark.parametrize("dtype", ["object", "str", "string"]) +@pytest.mark.parametrize("nullable", [False, True]) +def test_df_metrics_to_num_converts_string_numerics(dtype: str, nullable: bool) -> None: """Test that string-encoded numeric columns (e.g. from ClickHouse) are converted.""" query_object = MagicMock() query_object.metric_names = ["sum_col", "mixed_col", "text_col"] df = pd.DataFrame( { - "sum_col": pd.Series(["100", "200", "300"], dtype=object), - "mixed_col": pd.Series(["1", "not_a_number", "3"], dtype=object), - "text_col": pd.Series(["foo", "bar", "baz"], dtype=object), - "dim_col": pd.Series(["a", "b", "c"], dtype=object), + "sum_col": pd.Series( + ["100", "200", None if nullable else "300"], dtype=dtype + ), + "mixed_col": pd.Series(["1", "not_a_number", "3"], dtype=dtype), + "text_col": pd.Series(["foo", "bar", "baz"], dtype=dtype), + "dim_col": pd.Series(["10", "20", "30"], dtype=dtype), } ) + original = df.copy() dataframe_utils.df_metrics_to_num(df, query_object) # sum_col: all numeric strings -> should be converted to numeric assert pd.api.types.is_numeric_dtype(df["sum_col"]), "sum_col should be numeric" - assert df["sum_col"].tolist() == [100.0, 200.0, 300.0] + assert df["sum_col"].iloc[:2].tolist() == [100, 200] + if nullable: + assert pd.isna(df["sum_col"].iloc[2]) + else: + assert df["sum_col"].iloc[2] == 300 + assert df.assign(group="group").groupby("group")["sum_col"].mean().iloc[0] == ( + 150 if nullable else 200 + ) # mixed_col: has non-numeric value -> should NOT be converted - assert df["mixed_col"].dtype == object, "mixed_col should remain object" + pd.testing.assert_series_equal(df["mixed_col"], original["mixed_col"]) # text_col: all non-numeric -> should NOT be converted - assert df["text_col"].dtype == object, "text_col should remain object" + pd.testing.assert_series_equal(df["text_col"], original["text_col"]) # dim_col: not in metric_names -> should NOT be touched - assert df["dim_col"].dtype == object, "dim_col should not be touched" + pd.testing.assert_series_equal(df["dim_col"], original["dim_col"]) @pytest.mark.parametrize( diff --git a/tests/unit_tests/databases/api_test.py b/tests/unit_tests/databases/api_test.py index b5e7b488cd66..d417b351eb67 100644 --- a/tests/unit_tests/databases/api_test.py +++ b/tests/unit_tests/databases/api_test.py @@ -24,7 +24,7 @@ from typing import Any from unittest.mock import ANY, Mock from uuid import UUID -from zipfile import is_zipfile, ZipFile +from zipfile import ZipFile import pytest import yaml @@ -282,9 +282,15 @@ def test_database_connection( """ Test that connection info is only returned in ``api/v1/database/${id}/connection``. """ + from apispec import APISpec + from apispec.ext.marshmallow import MarshmallowPlugin + from marshmallow import fields, Schema + from superset.databases.api import DatabaseRestApi from superset.models.core import Database + plugin = MarshmallowPlugin() + APISpec(title="test", version="1", openapi_version="3.0.2", plugins=[plugin]) DatabaseRestApi.datamodel._session = session # create table for databases @@ -383,8 +389,9 @@ def test_database_connection( } }, "parameters_schema": { + **plugin.converter.schema2jsonschema(Schema()), "properties": { - "catalog": {"type": "object"}, + "catalog": plugin.converter.field2property(fields.Dict()), "oauth2_client_info": { "default": { "authorization_request_uri": "https://accounts.google.com/o/oauth2/v2/auth", @@ -788,7 +795,9 @@ def test_non_zip_import(client: Any, full_api_access: None) -> None: } -def test_corrupt_zip_import(client: Any, full_api_access: None) -> None: +def test_corrupt_zip_import( + client: Any, full_api_access: None, mocker: MockerFixture +) -> None: """ Test that a ZIP with a corrupt central directory returns a 422, not a 500. """ @@ -796,7 +805,7 @@ def test_corrupt_zip_import(client: Any, full_api_access: None) -> None: with ZipFile(buf, "w") as bundle: bundle.writestr("bundle/metadata.yaml", "version: 1.0.0\n") corrupt = BytesIO(buf.getvalue().replace(b"PK\x01\x02", b"XXXX")) - assert is_zipfile(corrupt) + mocker.patch("superset.databases.api.is_zipfile", return_value=True) form_data = { "formData": (corrupt, "corrupt.zip"), } diff --git a/tests/unit_tests/db_engine_specs/test_databricks.py b/tests/unit_tests/db_engine_specs/test_databricks.py index af8b50def4f3..17437fb39501 100644 --- a/tests/unit_tests/db_engine_specs/test_databricks.py +++ b/tests/unit_tests/db_engine_specs/test_databricks.py @@ -145,12 +145,18 @@ def test_parameters_json_schema() -> None: """ test that the parameters schema can be converted to json """ + from apispec import APISpec + from apispec.ext.marshmallow import MarshmallowPlugin + from marshmallow import Schema + from superset.db_engine_specs.databricks import DatabricksNativeEngineSpec + plugin = MarshmallowPlugin() + APISpec(title="test", version="1", openapi_version="3.0.2", plugins=[plugin]) json_schema = DatabricksNativeEngineSpec.parameters_json_schema() assert json_schema == { - "type": "object", + **plugin.converter.schema2jsonschema(Schema()), "properties": { "access_token": {"type": "string"}, "database": {"type": "string"}, diff --git a/tests/unit_tests/db_engine_specs/test_lint_metadata.py b/tests/unit_tests/db_engine_specs/test_lint_metadata.py new file mode 100644 index 000000000000..bf09be48f722 --- /dev/null +++ b/tests/unit_tests/db_engine_specs/test_lint_metadata.py @@ -0,0 +1,87 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +import ast +import runpy +from pathlib import Path +from typing import Any + +import pytest + + +@pytest.fixture(scope="module") +def metadata_linter() -> dict[str, Any]: + """Load the standalone linter without initializing the Superset application.""" + return runpy.run_path( + str( + Path(__file__).resolve().parents[3] + / "superset" + / "db_engine_specs" + / "lint_metadata.py" + ) + ) + + +@pytest.mark.parametrize( + ("expression", "expected"), + [ + ("'driver'", "driver"), + ("42", 42), + ("True", True), + ("None", None), + ("['driver', 42]", ["driver", 42]), + ( + "{'drivers': ['driver'], 'nested': {'enabled': True}}", + {"drivers": ["driver"], "nested": {"enabled": True}}, + ), + ("('driver', 42)", ("driver", 42)), + ("DatabaseCategory", "DatabaseCategory"), + ("DatabaseCategory.TRADITIONAL_RDBMS", "DatabaseCategory.TRADITIONAL_RDBMS"), + ("'driver' + 'name'", "drivername"), + ("1 + 2", None), + ("f'{driver}'", ""), + ("unknown()", None), + ], +) +def test_eval_ast_value_without_legacy_nodes( + metadata_linter: dict[str, Any], + monkeypatch: pytest.MonkeyPatch, + expression: str, + expected: Any, +) -> None: + """Evaluate metadata without the AST aliases removed in Python 3.14.""" + node = ast.parse(expression, mode="eval").body + monkeypatch.delattr(ast, "Str", raising=False) + monkeypatch.delattr(ast, "Num", raising=False) + assert metadata_linter["_eval_ast_value"](node) == expected + + +def test_discover_engine_metadata_without_legacy_nodes( + metadata_linter: dict[str, Any], monkeypatch: pytest.MonkeyPatch +) -> None: + """Discover real engine metadata without silently marking it unparseable.""" + monkeypatch.delattr(ast, "Str", raising=False) + monkeypatch.delattr(ast, "Num", raising=False) + specs = metadata_linter["get_all_engine_specs_ast"]() + postgres = next( + spec for spec in specs if spec["class_name"] == "PostgresEngineSpec" + ) + assert "_unparseable" not in postgres["metadata"] + assert postgres["metadata"]["categories"] + assert postgres["metadata"]["pypi_packages"] diff --git a/tests/unit_tests/docker_app_dependencies_test.py b/tests/unit_tests/docker_app_dependencies_test.py new file mode 100644 index 000000000000..8811b9b166e0 --- /dev/null +++ b/tests/unit_tests/docker_app_dependencies_test.py @@ -0,0 +1,155 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import re +import shlex +import subprocess +import tomllib +from pathlib import Path + +import pytest + + +@pytest.mark.parametrize( + "relative_path", + [ + "docker/install-app-dependencies.sh", + "tests/unit_tests/docker_app_dependencies_test.py", + ], +) +def test_dependency_installation_files_have_asf_headers(relative_path: str) -> None: + repository = Path(__file__).resolve().parents[2] + header = "\n".join((repository / relative_path).read_text().splitlines()[:20]) + assert "Licensed to the Apache Software Foundation (ASF)" in header + assert "http://www.apache.org/licenses/LICENSE-2.0" in header + + +@pytest.mark.parametrize("metadata", [False, True]) +def test_docker_dependency_layers_preserve_legacy_cache(metadata: bool) -> None: + repository = Path(__file__).resolve().parents[2] + dockerfile = (repository / "Dockerfile").read_text() + layers = [ + layer.replace("\\\n", " ") + for layer in re.findall(r"^RUN (?:.*\\\n)*.*", dockerfile, re.MULTILINE) + if "/app/docker/install-app-dependencies.sh" in layer + ] + assert len(layers) == 2 + assert "type=bind" not in layers[0] + assert "type=bind,source=.,target=/src" in layers[1] + outputs = [] + for layer in layers: + script = layer[layer.index("if python") :] + script = ( + script.replace( + "python -c 'import sys; raise SystemExit(sys.version_info >= (3, 15))'", + "false" if metadata else "true", + ) + .replace( + "python -c 'import sys; raise SystemExit(sys.version_info < (3, 15))'", + "true" if metadata else "false", + ) + .replace("/app/docker/install-app-dependencies.sh", "printf '%s\\n'") + ) + outputs.append( + subprocess.run( + ["/bin/bash", "-s"], + input=script, + capture_output=True, + text=True, + check=True, + ).stdout + ) + assert outputs == (["", "/src\n"] if metadata else ["/app\n", ""]) + + +@pytest.mark.parametrize("metadata", [False, True]) +def test_dependency_installation_selects_interpreter_contract(metadata: bool) -> None: + """Use metadata only for interpreters newer than the generated lock baseline.""" + repository = Path(__file__).resolve().parents[2] + script = (repository / "docker/install-app-dependencies.sh").read_text() + script = script.replace("/app/docker/pip-install.sh", "printf '%s\\n'").replace( + "python -c 'import sys; raise SystemExit(sys.version_info < (3, 15))'", + "true" if metadata else "false", + ) + result = subprocess.run( + ["/bin/bash", "-s", "--", "/source"], + input=script, + capture_output=True, + text=True, + check=True, + ) + arguments = result.stdout.splitlines() + assert arguments[0] == "--requires-build-essential" + if metadata: + assert arguments[1:] == [ + "--no-sources", + "/source/superset-core", + "/source", + ] + else: + assert arguments[1:] == ["-r", "requirements/base.txt"] + + +def test_cli_dependency_is_declared() -> None: + """The CLI must not rely on a test environment to supply rich.""" + repository = Path(__file__).resolve().parents[2] + project = tomllib.loads((repository / "pyproject.toml").read_text()) + assert "rich" in project["project"]["dependencies"] + + +@pytest.mark.parametrize("installer_exit", [0, 1]) +def test_readonly_build_copy_is_cleaned(tmp_path: Path, installer_exit: int) -> None: + """Leave mounted sources intact and remove build copies on either outcome.""" + source = tmp_path / "source" + source.mkdir() + marker = source / "marker" + marker.write_text("original") + repository = Path(__file__).resolve().parents[2] + script = (repository / "docker/install-app-dependencies.sh").read_text() + script = ( + script.replace( + "python -c 'import sys; raise SystemExit(sys.version_info < (3, 15))'", + "true", + ) + .replace( + '[[ -d "${SOURCE_ROOT}" && ! -w "${SOURCE_ROOT}" ]]', + "true", + ) + .replace("/app/docker/pip-install.sh", "install_dependencies") + ) + installer = ( + 'install_dependencies() { printf "%s\\n" "${SOURCE_ROOT}"; ' + 'test "$(cat "${SOURCE_ROOT}/marker")" = original; ' + 'printf changed > "${SOURCE_ROOT}/marker"; ' + f"return {installer_exit}; }}\n" + ) + result: subprocess.CompletedProcess[str] | subprocess.CalledProcessError + try: + result = subprocess.run( + ["/bin/bash", "-s"], + input=installer + f"set -- {shlex.quote(str(source))}\n" + script, + capture_output=True, + text=True, + check=True, + ) + except subprocess.CalledProcessError as error: + assert installer_exit == 1 + result = error + assert result.returncode == installer_exit + copied_source = Path(result.stdout.strip()) + assert copied_source != source + assert not copied_source.exists() + assert marker.read_text() == "original" diff --git a/tests/unit_tests/extensions/test_sqlalchemy.py b/tests/unit_tests/extensions/test_sqlalchemy.py index fdb41775c066..35e69e5953cc 100644 --- a/tests/unit_tests/extensions/test_sqlalchemy.py +++ b/tests/unit_tests/extensions/test_sqlalchemy.py @@ -380,12 +380,16 @@ def table2_fanout_match(session: Session, database2: "Database") -> Iterator[Non @with_feature_flags(ENABLE_SUPERSET_META_DB=True) +@pytest.mark.parametrize( + "cursor_mode", ["sqlalchemy", "iterate", "fetchone", "fetchmany", "fetchall"] +) def test_superset_joins_with_limit_drops_fanout_matches( mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch, app_context: None, table1_large: None, table2_fanout_match: None, + cursor_mode: str, ) -> None: """ Coverage note from review of #42598: the other join regression tests @@ -415,17 +419,34 @@ def test_superset_joins_with_limit_drops_fanout_matches( except Exception as e: pytest.skip(f"Superset dialect not available: {e}") - with engine.connect() as conn: - results = conn.execute( - text(""" + statement = """ SELECT t1.b, t2.b FROM "database1.table1_large" AS t1 JOIN "database2.table2_fanout_match" AS t2 ON t1.a = t2.a ORDER BY t2.b - """) - ) - assert list(results) == [(30, "thirty-x"), (30, "thirty-y")] + """ + if cursor_mode != "sqlalchemy": + raw_connection = engine.raw_connection() + try: + cursor = raw_connection.cursor() + cursor.execute(statement) + if cursor_mode == "iterate": + rows = list(cursor) + elif cursor_mode == "fetchone": + rows = [cursor.fetchone(), cursor.fetchone()] + assert cursor.fetchone() is None + elif cursor_mode == "fetchmany": + rows = cursor.fetchmany(2) + else: + rows = cursor.fetchall() + assert rows == [(30, "thirty-x"), (30, "thirty-y")] + finally: + raw_connection.close() + else: + with engine.connect() as conn: + results = conn.execute(text(statement)) + assert list(results) == [(30, "thirty-x"), (30, "thirty-y")] @with_feature_flags(ENABLE_SUPERSET_META_DB=True) diff --git a/tests/unit_tests/importexport/api_test.py b/tests/unit_tests/importexport/api_test.py index ebf249716a83..6ea648a165e6 100644 --- a/tests/unit_tests/importexport/api_test.py +++ b/tests/unit_tests/importexport/api_test.py @@ -270,7 +270,7 @@ def test_import_assets_corrupt_zip( with ZipFile(buf, "w") as bundle: bundle.writestr("bundle/metadata.yaml", "version: 1.0.0\n") corrupt = BytesIO(buf.getvalue().replace(b"PK\x01\x02", b"XXXX")) - assert is_zipfile(corrupt) + mocker.patch("superset.importexport.api.is_zipfile", return_value=True) form_data = { "bundle": (corrupt, "corrupt.zip"), } diff --git a/tests/unit_tests/mcp_service/chart/test_bullet_chart.py b/tests/unit_tests/mcp_service/chart/test_bullet_chart.py index 48e3447ec261..ae044d266b1f 100644 --- a/tests/unit_tests/mcp_service/chart/test_bullet_chart.py +++ b/tests/unit_tests/mcp_service/chart/test_bullet_chart.py @@ -1269,7 +1269,7 @@ def validate( assert result.success is True assert [row["Category"] for row in captured[0]] == [ - 1704164645123.456, + json_int_dttm_ser(source_values[0]), dublin_epoch_ms, 1710052200000.0, 2224774800000.0, @@ -1772,17 +1772,10 @@ def test_bullet_timestamp_categories_match_chart_data_wire_and_all_previews() -> np.datetime64("NaT"), ] expected = [ - "1704164645123.456", - "1704164645123.456", - "1704164645123.456", - "1730611800000", - "1730615400000", - "1730611800000", - "1730615400000", - "-0.0010000000000287557", - "0", + _javascript_number_string(json_int_dttm_ser(value)) for value in values[:9] + ] + [ "1704153600000", - "1704164645123.456", + _javascript_number_string(json_int_dttm_ser(pd.Timestamp(values[10]))), "1704153600000", "null", "null", diff --git a/tests/unit_tests/mcp_service/chart/test_bullet_tool_paths.py b/tests/unit_tests/mcp_service/chart/test_bullet_tool_paths.py index 85cad65700cf..6e467d4bb746 100644 --- a/tests/unit_tests/mcp_service/chart/test_bullet_tool_paths.py +++ b/tests/unit_tests/mcp_service/chart/test_bullet_tool_paths.py @@ -300,8 +300,10 @@ def test_bullet_temporal_mode_projects_timestamp_and_numpy_before_generic() -> N assert data == [ [ { - "timestamp": 1704164645123.456, - "numpy_timestamp": -0.0010000000000287557, + "timestamp": json.json_int_dttm_ser(timestamp), + "numpy_timestamp": json.json_int_dttm_ser( + pd.Timestamp(numpy_timestamp) + ), "date": 1704153600000.0, "pandas_nat": None, "numpy_nat": None, @@ -645,11 +647,11 @@ def run(self) -> dict[str, Any]: payload = json.loads(result.content[0].text) assert [row["Category"] for row in payload["data"]] == [ - 1704164645123.456, - 1704164645123.456, + json.json_int_dttm_ser(source_values[0]), + json.json_int_dttm_ser(source_values[1]), 1730611800000.0, 1730615400000.0, - -0.0010000000000287557, + json.json_int_dttm_ser(source_values[4]), None, dublin_epoch_ms, 1710052200123.456, @@ -1612,13 +1614,9 @@ async def test_bullet_timestamp_categories_from_dataframe_reach_fastmcp( # dateutil named-zone edge in this public-format invocation. rows = rows[9:12] expected = [ - "1704164645123.456", - "1704164645123.456", - "1730611800000", - "1730615400000", - "1730611800000", - "1730615400000", - "-0.0010000000000287557", + _javascript_number_string(json.json_int_dttm_ser(value)) + for value in source_values[:7] + ] + [ "1704153600000", "null", _javascript_number_string(dublin_epoch_ms), @@ -1715,13 +1713,13 @@ def run(self) -> dict[str, Any]: ) assert bar["encoding"]["tooltip"][0]["field"] == category_field assert [row["Category"] for row in specification["data"]["values"]] == [ - 1704164645123.456, - 1704164645123.456, + json.json_int_dttm_ser(source_values[0]), + json.json_int_dttm_ser(source_values[1]), 1730611800000.0, 1730615400000.0, 1730611800000.0, 1730615400000.0, - -0.0010000000000287557, + json.json_int_dttm_ser(source_values[6]), 1704153600000.0, None, dublin_epoch_ms, diff --git a/tests/unit_tests/models/helpers_test.py b/tests/unit_tests/models/helpers_test.py index 4ef8a2ff4f13..5ce1dbda4612 100644 --- a/tests/unit_tests/models/helpers_test.py +++ b/tests/unit_tests/models/helpers_test.py @@ -2495,6 +2495,9 @@ def test_get_sqla_query_raises_for_nonaggregate_custom_sql_metric_with_timeserie ) +@pytest.mark.parametrize( + "app", [{"FEATURE_FLAGS": {"ENABLE_TEMPLATE_PROCESSING": True}}], indirect=True +) def test_get_sqla_query_allows_jinja_templated_custom_sql_metric_with_columns( database: Database, ) -> None: @@ -2546,6 +2549,9 @@ def test_get_sqla_query_allows_jinja_templated_custom_sql_metric_with_columns( assert "{{" not in sql +@pytest.mark.parametrize( + "app", [{"FEATURE_FLAGS": {"ENABLE_TEMPLATE_PROCESSING": True}}], indirect=True +) def test_get_sqla_query_virtual_dataset_filter_values_drill_to_detail( database: Database, ) -> None: @@ -4286,6 +4292,37 @@ def test_normalize_df_replaces_infinity_without_downcasting_warning() -> None: assert result["metric"].isna().tolist() == [False, True, True, False] assert result["count"].dtype == np.float64 assert result["label"].dtype == "str" + pd.testing.assert_series_equal(result["label"], df["label"]) + + +@pytest.mark.parametrize("datetime_format", [None, "%Y-%m-%dT%H:%M:%S%z"]) +def test_temporal_join_preserves_local_clocks_across_offsets( + datetime_format: str | None, +) -> None: + import pandas as pd + + from superset.models.helpers import _coerce_temporal_join_series + + source = pd.Series( + ["2024-01-15T12:00:00-05:00", "2024-07-15T12:00:00-04:00"], + index=[10, 20], + name="ts", + ) + comparison = pd.Series( + ["2024-01-15T12:00:00-05:00", "2024-07-15T12:00:00-05:00"], + index=source.index, + name="ts", + ) + expected = [ + pd.Timestamp("2024-01-15T12:00:00"), + pd.Timestamp("2024-07-15T12:00:00"), + ] + + for values in (source, comparison): + converted = _coerce_temporal_join_series(values, "ts", datetime_format) + assert converted.tolist() == expected + assert converted.index.equals(values.index) + assert converted.name == values.name @pytest.mark.parametrize( diff --git a/tests/unit_tests/openapi/test_static_spec.py b/tests/unit_tests/openapi/test_static_spec.py index b27df6519904..011f5b59fe93 100644 --- a/tests/unit_tests/openapi/test_static_spec.py +++ b/tests/unit_tests/openapi/test_static_spec.py @@ -20,6 +20,19 @@ from superset.utils import json +def test_select_star_without_schema_documents_errors() -> None: + """The schema-free wrapper retains the delegated handler's error contract.""" + spec_path = ( + Path(__file__).parents[3] / "docs" / "static" / "resources" / "openapi.json" + ) + spec = json.loads(spec_path.read_text(encoding="utf-8")) + responses = spec["paths"]["/api/v1/database/{pk}/select_star/{table_name}/"]["get"][ + "responses" + ] + for status in ("400", "401", "404", "422", "500"): + assert responses[status] == {"$ref": f"#/components/responses/{status}"} + + def test_documented_dashboard_list_schema_matches_public_contract() -> None: """The published dashboard list contract must use generated relationships.""" spec_path = ( diff --git a/tests/unit_tests/pandas_postprocessing/test_boxplot.py b/tests/unit_tests/pandas_postprocessing/test_boxplot.py index 7bde4a2b0c36..c82cffca9838 100644 --- a/tests/unit_tests/pandas_postprocessing/test_boxplot.py +++ b/tests/unit_tests/pandas_postprocessing/test_boxplot.py @@ -16,6 +16,7 @@ # under the License. import warnings +import pandas as pd import pytest from superset.exceptions import InvalidPostProcessingError @@ -25,6 +26,33 @@ from tests.unit_tests.pandas_postprocessing.utils import series_to_list +@pytest.mark.parametrize("dtype", ["object", "str"]) +@pytest.mark.parametrize("whisker_type", list(PostProcessingBoxplotWhiskerType)) +def test_boxplot_rejects_malformed_metrics(dtype, whisker_type): + frame = pd.DataFrame({"value": pd.Series(["1", "invalid", None], dtype=dtype)}) + with pytest.raises(InvalidPostProcessingError, match="numeric"): + boxplot( + df=frame, + groupby=[], + metrics=["value"], + whisker_type=whisker_type, + percentiles=[1, 99], + ) + + +@pytest.mark.parametrize("dtype", ["object", "str"]) +def test_boxplot_accepts_numeric_strings_and_missing_values(dtype): + frame = pd.DataFrame({"value": pd.Series(["1", "3", None], dtype=dtype)}) + result = boxplot( + df=frame, + groupby=[], + metrics=["value"], + whisker_type=PostProcessingBoxplotWhiskerType.TUKEY, + ) + assert result["value__mean"].iloc[0] == 2 + assert result["value__median"].iloc[0] == 2 + + def test_boxplot_tukey(): df = boxplot( df=names_df, diff --git a/tests/unit_tests/pandas_postprocessing/test_histogram.py b/tests/unit_tests/pandas_postprocessing/test_histogram.py index 474c412f0326..34b3ad1adae3 100644 --- a/tests/unit_tests/pandas_postprocessing/test_histogram.py +++ b/tests/unit_tests/pandas_postprocessing/test_histogram.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. import pytest -from pandas import DataFrame +from pandas import DataFrame, Series from superset.exceptions import InvalidPostProcessingError from superset.utils.pandas_postprocessing import histogram @@ -31,6 +31,13 @@ bins = 5 +@pytest.mark.parametrize("dtype", ["string", "str"]) +def test_histogram_string_dtype(dtype: str) -> None: + frame = DataFrame({"value": Series(["1", "2", "3", "4"], dtype=dtype)}) + result = histogram(frame, "value", [], bins=2) + assert result.values.tolist() == [[2, 2]] + + def test_histogram_no_groupby(): data_with_no_groupings = DataFrame( {"a": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], "b": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]} diff --git a/tests/unit_tests/pandas_postprocessing/test_pivot.py b/tests/unit_tests/pandas_postprocessing/test_pivot.py index 0292cdb75ef5..3895f6caa366 100644 --- a/tests/unit_tests/pandas_postprocessing/test_pivot.py +++ b/tests/unit_tests/pandas_postprocessing/test_pivot.py @@ -41,6 +41,36 @@ def test_pivot_without_columns(): assert df["idx_nulls"].sum() == 1050 +@pytest.mark.parametrize("drop_missing_columns", [False, True]) +@pytest.mark.parametrize("columns", [["category"], ["category", "country"]]) +def test_pivot_combine_preserves_column_pairs(drop_missing_columns, columns): + frame = DataFrame( + { + "row": ["first", "second"], + "category": ["A", "B"], + "country": ["US", "EU"], + "metric1": [1.0, np.nan], + "metric2": [np.nan, 2.0], + "empty": [np.nan, np.nan], + } + ) + options = { + "index": ["row"], + "columns": columns, + "aggregates": { + metric: {"operator": "mean"} for metric in ["metric1", "metric2", "empty"] + }, + "drop_missing_columns": drop_missing_columns, + } + expected = pivot(df=frame.copy(), **options) + expected.columns = expected.columns.reorder_levels([*range(1, len(columns) + 1), 0]) + actual = pivot(df=frame.copy(), combine_value_with_metric=True, **options) + + pd.testing.assert_frame_equal( + actual.sort_index(axis=1), expected.sort_index(axis=1) + ) + + def test_pivot_with_single_column(): """ Make sure pivot with single column returns correct DataFrame diff --git a/tests/unit_tests/pandas_postprocessing/test_resample.py b/tests/unit_tests/pandas_postprocessing/test_resample.py index ee6340835401..8d27fe787c75 100644 --- a/tests/unit_tests/pandas_postprocessing/test_resample.py +++ b/tests/unit_tests/pandas_postprocessing/test_resample.py @@ -31,12 +31,44 @@ ) +@pytest.mark.parametrize( + "legacy, modern", + [("1T", "1min"), ("1H", "1h"), ("1M", "1ME"), ("1AS", "1YS"), ("1A", "1YE")], +) +def test_resample_legacy_aliases(legacy: str, modern: str) -> None: + frame = pd.DataFrame( + {"value": [1, 2, 3, 4]}, + index=pd.date_range("2024-01-01", periods=4, freq="h"), + ) + pd.testing.assert_frame_equal( + pp.resample(frame, rule=legacy, method="sum"), + pp.resample(frame, rule=modern, method="sum"), + ) + + def test_resample_should_not_side_effect(): _timeseries_df = timeseries_df.copy() pp.resample(df=_timeseries_df, rule="1D", method="ffill") assert _timeseries_df.equals(timeseries_df) +@pytest.mark.parametrize("timezone", [None, "UTC"]) +def test_linear_resample_preserves_datetime_interpolation(timezone): + frame = pd.DataFrame( + { + "date": pd.date_range("2024-02-01", periods=2, freq="2D", tz=timezone), + "value": [1.0, 3.0], + "label": ["first", "last"], + }, + index=pd.to_datetime(["2024-01-01", "2024-01-03"]), + ) + result = pp.resample(df=frame, rule="D", method="linear") + assert result["date"].iloc[1] == pd.Timestamp("2024-02-02", tz=timezone) + assert result["date"].dtype == frame["date"].dtype + assert result["value"].iloc[1] == 2.0 + assert pd.isna(result["label"].iloc[1]) + + def test_resample(): post_df = pp.resample(df=timeseries_df, rule="1D", method="ffill") """ diff --git a/tests/unit_tests/pandas_postprocessing/test_rolling.py b/tests/unit_tests/pandas_postprocessing/test_rolling.py index 019002de4de7..410193200afe 100644 --- a/tests/unit_tests/pandas_postprocessing/test_rolling.py +++ b/tests/unit_tests/pandas_postprocessing/test_rolling.py @@ -179,8 +179,8 @@ def test_rolling_after_pivot_with_single_metric(): pd.DataFrame( data={ "dttm": pd.to_datetime(["2019-01-01", "2019-01-02"]), - FLAT_COLUMN_SEPARATOR.join(["sum_metric", "UK"]): [5, 12], - FLAT_COLUMN_SEPARATOR.join(["sum_metric", "US"]): [6, 14], + FLAT_COLUMN_SEPARATOR.join(["sum_metric", "UK"]): [5.0, 12.0], + FLAT_COLUMN_SEPARATOR.join(["sum_metric", "US"]): [6.0, 14.0], } ) ) @@ -230,10 +230,10 @@ def test_rolling_after_pivot_with_multiple_metrics(): pd.DataFrame( data={ "dttm": pd.to_datetime(["2019-01-01", "2019-01-02"]), - FLAT_COLUMN_SEPARATOR.join(["count_metric", "UK"]): [1, 4], - FLAT_COLUMN_SEPARATOR.join(["count_metric", "US"]): [2, 6], - FLAT_COLUMN_SEPARATOR.join(["sum_metric", "UK"]): [5, 12], - FLAT_COLUMN_SEPARATOR.join(["sum_metric", "US"]): [6, 14], + FLAT_COLUMN_SEPARATOR.join(["count_metric", "UK"]): [1.0, 4.0], + FLAT_COLUMN_SEPARATOR.join(["count_metric", "US"]): [2.0, 6.0], + FLAT_COLUMN_SEPARATOR.join(["sum_metric", "UK"]): [5.0, 12.0], + FLAT_COLUMN_SEPARATOR.join(["sum_metric", "US"]): [6.0, 14.0], } ) ) diff --git a/tests/unit_tests/pandas_postprocessing/test_utils.py b/tests/unit_tests/pandas_postprocessing/test_utils.py index 1d7bd3e3a8fe..5645f7d80e55 100644 --- a/tests/unit_tests/pandas_postprocessing/test_utils.py +++ b/tests/unit_tests/pandas_postprocessing/test_utils.py @@ -127,6 +127,17 @@ def test_append_columns_with_empty_mapping_returns_a_copy(): assert series_to_list(post_df["y"]) == [1.0, 2.0] +def test_append_columns_preserves_replacement_dtype_and_index() -> None: + """Replacing integer data must preserve fractions, nulls and index alignment.""" + base_df = DataFrame({"y": [1, 2, 3]}, index=["a", "b", "c"]) + append_df = DataFrame({"y": [3.5, float("nan"), 1.5]}, index=["c", "b", "a"]) + + post_df = _append_columns(base_df, append_df, {"y": "y"}) + + assert post_df.equals(append_df.reindex(base_df.index)) + assert series_to_list(base_df["y"]) == [1, 2, 3] + + def test_scalar_to_sequence() -> None: """ None becomes an empty list. diff --git a/tests/unit_tests/utils/test_date_parsing.py b/tests/unit_tests/utils/test_date_parsing.py index 376c4c55be38..80be2208ad3b 100644 --- a/tests/unit_tests/utils/test_date_parsing.py +++ b/tests/unit_tests/utils/test_date_parsing.py @@ -28,6 +28,29 @@ from superset.utils.pandas import detect_datetime_format +@pytest.mark.parametrize("timestamp_format", [None, "%Y-%m-%dT%H:%M:%S%z"]) +@pytest.mark.parametrize("timezone", [None, "Europe/Berlin"]) +def test_normalize_mixed_offsets( + timestamp_format: str | None, timezone: str | None +) -> None: + frame = pd.DataFrame( + {"date": ["2024-10-27T02:30:00+02:00", "2024-10-27T02:30:00+01:00"]} + ) + normalize_dttm_col( + frame, + (DateColumn("date", timestamp_format=timestamp_format, timezone=timezone),), + ) + expected = ( + [pd.Timestamp("2024-10-27T02:30:00")] * 2 + if timezone + else [ + pd.Timestamp("2024-10-27T00:30:00Z"), + pd.Timestamp("2024-10-27T01:30:00Z"), + ] + ) + assert frame["date"].tolist() == expected + + def capture_warnings(func, *args, **kwargs): """Execute function and return any format inference warnings.""" with warnings.catch_warnings(record=True) as w: @@ -138,7 +161,9 @@ def test_epoch_format_invalid_values(caplog): # The column should remain unchanged when conversion fails assert df["epoch"].dtype == "str" - assert df["epoch"].iloc[0] == "not_a_number" + pd.testing.assert_series_equal( + df["epoch"], pd.Series(["not_a_number", "invalid", "abc"], name="epoch") + ) @pytest.mark.parametrize(