From 29079cec68eafa37e6791224135c77eaf628e5e4 Mon Sep 17 00:00:00 2001 From: Jared Yu Date: Thu, 2 Jul 2026 13:20:21 -0700 Subject: [PATCH 1/3] Deprecate 'downcast-ns-timestamp-to-us-on-write' config key in favor of 'downcast-ns-timestamp-to-us' The config key 'downcast-ns-timestamp-to-us-on-write' says 'on-write' but is also used in the read path (ArrowScan). Rename the canonical key to 'downcast-ns-timestamp-to-us' which accurately describes the behavior regardless of direction. The old key still works but emits a DeprecationWarning. The new key takes precedence when both are set. --- mkdocs/docs/configuration.md | 5 +- pyiceberg/catalog/__init__.py | 4 +- pyiceberg/io/pyarrow.py | 17 ++-- pyiceberg/table/__init__.py | 43 +++++++-- pyproject.toml | 2 + tests/integration/test_add_files.py | 6 +- tests/integration/test_writes/test_writes.py | 2 +- tests/io/test_pyarrow_visitor.py | 96 +++++++++++++++++++- 8 files changed, 149 insertions(+), 26 deletions(-) diff --git a/mkdocs/docs/configuration.md b/mkdocs/docs/configuration.md index 54d33dd00e..f60083f258 100644 --- a/mkdocs/docs/configuration.md +++ b/mkdocs/docs/configuration.md @@ -931,4 +931,7 @@ Previous versions of Java (`<1.4.0`) implementations incorrectly assume the opti ## Nanoseconds Support -PyIceberg currently only supports upto microsecond precision in its TimestampType. PyArrow timestamp types in 's' and 'ms' will be upcast automatically to 'us' precision timestamps on write. Timestamps in 'ns' precision can also be downcast automatically on write if desired. This can be configured by setting the `downcast-ns-timestamp-to-us-on-write` property as "True" in the configuration file, or by setting the `PYICEBERG_DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE` environment variable. Refer to the [nanoseconds timestamp proposal document](https://docs.google.com/document/d/1bE1DcEGNzZAMiVJSZ0X1wElKLNkT9kRkk0hDlfkXzvU/edit#heading=h.ibflcctc9i1d) for more details on the long term roadmap for nanoseconds support +PyIceberg currently only supports upto microsecond precision in its TimestampType. PyArrow timestamp types in 's' and 'ms' will be upcast automatically to 'us' precision timestamps on write. Timestamps in 'ns' precision can also be downcast automatically when desired. This can be configured by setting the `downcast-ns-timestamp-to-us` property as "True" in the configuration file, or by setting the `PYICEBERG_DOWNCAST_NS_TIMESTAMP_TO_US` environment variable. Refer to the [nanoseconds timestamp proposal document](https://docs.google.com/document/d/1bE1DcEGNzZAMiVJSZ0X1wElKLNkT9kRkk0hDlfkXzvU/edit#heading=h.ibflcctc9i1d) for more details on the long term roadmap for nanoseconds support. + +!!! note "Deprecated config key" + The previous config key `downcast-ns-timestamp-to-us-on-write` (env: `PYICEBERG_DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE`) is deprecated. It still works but will emit a deprecation warning. Migrate to `downcast-ns-timestamp-to-us` (env: `PYICEBERG_DOWNCAST_NS_TIMESTAMP_TO_US`). diff --git a/pyiceberg/catalog/__init__.py b/pyiceberg/catalog/__init__.py index 8de113404c..d31d29f51d 100644 --- a/pyiceberg/catalog/__init__.py +++ b/pyiceberg/catalog/__init__.py @@ -46,12 +46,12 @@ from pyiceberg.schema import Schema from pyiceberg.serializers import ToOutputFile from pyiceberg.table import ( - DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE, CommitTableResponse, CreateTableTransaction, StagedTable, Table, TableProperties, + _get_downcast_ns_timestamp_to_us, ) from pyiceberg.table.locations import load_location_provider from pyiceberg.table.metadata import TableMetadata, TableMetadataV1, new_table_metadata @@ -847,7 +847,7 @@ def _convert_schema_if_needed( from pyiceberg.io.pyarrow import _ConvertToIcebergWithoutIDs, visit_pyarrow - downcast_ns_timestamp_to_us = Config().get_bool(DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE) or False + downcast_ns_timestamp_to_us = _get_downcast_ns_timestamp_to_us() if isinstance(schema, pa.Schema): schema: Schema = visit_pyarrow( # type: ignore schema, diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index c36f1639d9..a1e763dd05 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -146,7 +146,7 @@ visit, visit_with_partner, ) -from pyiceberg.table import DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE, TableProperties +from pyiceberg.table import TableProperties, _get_downcast_ns_timestamp_to_us from pyiceberg.table.deletion_vector import deletion_vectors_from_puffin_file from pyiceberg.table.locations import load_location_provider from pyiceberg.table.metadata import TableMetadata @@ -183,7 +183,6 @@ strtobool, ) from pyiceberg.utils.concurrent import ExecutorFactory -from pyiceberg.utils.config import Config from pyiceberg.utils.datetime import millis_to_datetime from pyiceberg.utils.decimal import unscaled_to_decimal from pyiceberg.utils.properties import get_first_property_value, property_as_bool, property_as_int @@ -1480,8 +1479,8 @@ def primitive(self, primitive: pa.DataType) -> PrimitiveType: else: raise TypeError( "Iceberg does not yet support 'ns' timestamp precision. " - "Use 'downcast-ns-timestamp-to-us-on-write' configuration property to automatically " - "downcast 'ns' to 'us' on write.", + "Use 'downcast-ns-timestamp-to-us' configuration property to automatically " + "downcast 'ns' to 'us'.", ) else: raise TypeError(f"Unsupported precision for timestamp type: {primitive.unit}") @@ -1777,7 +1776,7 @@ def __init__( self._bound_row_filter = bind(table_metadata.schema(), row_filter, case_sensitive=case_sensitive) self._case_sensitive = case_sensitive self._limit = limit - self._downcast_ns_timestamp_to_us = Config().get_bool(DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE) + self._downcast_ns_timestamp_to_us = _get_downcast_ns_timestamp_to_us() self._dictionary_columns = dictionary_columns @property @@ -2716,7 +2715,7 @@ def add_field_metadata(self, field: NestedField, metadata: dict[bytes, bytes], i def write_file(io: FileIO, table_metadata: TableMetadata, tasks: Iterator[WriteTask]) -> Iterator[DataFile]: - from pyiceberg.table import DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE, TableProperties + from pyiceberg.table import TableProperties file_format = FileFormat( table_metadata.properties.get( @@ -2734,7 +2733,7 @@ def write_data_file(task: WriteTask) -> DataFile: else: file_schema = table_schema - downcast_ns_timestamp_to_us = Config().get_bool(DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE) or False + downcast_ns_timestamp_to_us = _get_downcast_ns_timestamp_to_us() batches = [ _to_requested_schema( requested_schema=file_schema, @@ -2981,7 +2980,7 @@ def _dataframe_to_data_files( Returns: An iterable that supplies datafiles that represent the input data. """ - from pyiceberg.table import DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE, TableProperties, WriteTask + from pyiceberg.table import TableProperties, WriteTask counter = counter or itertools.count(0) write_uuid = write_uuid or uuid.uuid4() @@ -2991,7 +2990,7 @@ def _dataframe_to_data_files( default=TableProperties.WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT, ) name_mapping = table_metadata.schema().name_mapping - downcast_ns_timestamp_to_us = Config().get_bool(DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE) or False + downcast_ns_timestamp_to_us = _get_downcast_ns_timestamp_to_us() task_schema = pyarrow_to_schema( df.schema, name_mapping=name_mapping, diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 9624eac981..fc4f1cdede 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -123,9 +123,39 @@ logger = logging.getLogger(__name__) ALWAYS_TRUE = AlwaysTrue() +DOWNCAST_NS_TIMESTAMP_TO_US = "downcast-ns-timestamp-to-us" +# Deprecated: use DOWNCAST_NS_TIMESTAMP_TO_US. The old key said "on-write" but the +# config also controls read-path downcasting, so the name was misleading. DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE = "downcast-ns-timestamp-to-us-on-write" +def _get_downcast_ns_timestamp_to_us() -> bool: + """Return the effective value of the downcast-ns-timestamp-to-us config. + + Checks the new ``downcast-ns-timestamp-to-us`` key first. If not set, falls + back to the deprecated ``downcast-ns-timestamp-to-us-on-write`` key and + emits a :class:`DeprecationWarning` so callers can migrate their config. + """ + config = Config() + + value = config.get_bool(DOWNCAST_NS_TIMESTAMP_TO_US) + if value is not None: + return value + + legacy = config.get_bool(DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE) + if legacy is not None: + warnings.warn( + f"Config key '{DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE}' (env: PYICEBERG_DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE) " + f"is deprecated. Use '{DOWNCAST_NS_TIMESTAMP_TO_US}' " + "(env: PYICEBERG_DOWNCAST_NS_TIMESTAMP_TO_US) instead.", + DeprecationWarning, + stacklevel=2, + ) + return legacy + + return False + + @dataclass() class UpsertResult: """Summary the upsert operation.""" @@ -543,7 +573,7 @@ def append( if not isinstance(df, (pa.Table, pa.RecordBatchReader)): raise ValueError(f"Expected pa.Table or pa.RecordBatchReader, got: {df}") - downcast_ns_timestamp_to_us = Config().get_bool(DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE) or False + downcast_ns_timestamp_to_us = _get_downcast_ns_timestamp_to_us() _check_pyarrow_schema_compatible( self.table_metadata.schema(), provided_schema=df.schema, @@ -599,7 +629,7 @@ def dynamic_partition_overwrite( f"in the latest partition spec: {field}" ) - downcast_ns_timestamp_to_us = Config().get_bool(DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE) or False + downcast_ns_timestamp_to_us = _get_downcast_ns_timestamp_to_us() _check_pyarrow_schema_compatible( self.table_metadata.schema(), provided_schema=df.schema, @@ -708,7 +738,7 @@ def overwrite( if not isinstance(df, (pa.Table, pa.RecordBatchReader)): raise ValueError(f"Expected pa.Table or pa.RecordBatchReader, got: {df}") - downcast_ns_timestamp_to_us = Config().get_bool(DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE) or False + downcast_ns_timestamp_to_us = _get_downcast_ns_timestamp_to_us() _check_pyarrow_schema_compatible( self.table_metadata.schema(), provided_schema=df.schema, @@ -915,7 +945,7 @@ def upsert( from pyiceberg.io.pyarrow import _check_pyarrow_schema_compatible - downcast_ns_timestamp_to_us = Config().get_bool(DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE) or False + downcast_ns_timestamp_to_us = _get_downcast_ns_timestamp_to_us() _check_pyarrow_schema_compatible( self.table_metadata.schema(), provided_schema=df.schema, @@ -2665,8 +2695,9 @@ def plan_files(self) -> Iterable[FileScanTask]: options=self.options, ).plan_files( manifests=manifests, - manifest_entry_filter=lambda manifest_entry: manifest_entry.snapshot_id in append_snapshot_ids - and manifest_entry.status == ManifestEntryStatus.ADDED, + manifest_entry_filter=lambda manifest_entry: ( + manifest_entry.snapshot_id in append_snapshot_ids and manifest_entry.status == ManifestEntryStatus.ADDED + ), ) def to_arrow(self) -> pa.Table: diff --git a/pyproject.toml b/pyproject.toml index 674db6d185..6084a4f348 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -184,6 +184,8 @@ filterwarnings = [ "ignore:As the c extension couldn't be imported:RuntimeWarning:google_crc32c", # Ignore Spark 4.0.1 pandas conversion warning under pandas 3.0 "ignore:The copy keyword is deprecated and will be removed in a future version.*", + # Deprecated config key migration (backwards compat) + "ignore:Config key 'downcast-ns-timestamp-to-us-on-write'.*:DeprecationWarning", ] [tool.mypy] diff --git a/tests/integration/test_add_files.py b/tests/integration/test_add_files.py index a1d45451d8..4f8f584644 100644 --- a/tests/integration/test_add_files.py +++ b/tests/integration/test_add_files.py @@ -732,7 +732,7 @@ def test_add_files_with_timestamp_tz_ns_fails(session_catalog: Catalog, format_v ], schema=nanoseconds_schema, ) - mocker.patch.dict(os.environ, values={"PYICEBERG_DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE": "True"}) + mocker.patch.dict(os.environ, values={"PYICEBERG_DOWNCAST_NS_TIMESTAMP_TO_US": "True"}) identifier = f"default.timestamptz_ns_added{format_version}" tbl = _create_table(session_catalog, identifier, format_version, schema=nanoseconds_schema_iceberg) @@ -754,8 +754,8 @@ def test_add_files_with_timestamp_tz_ns_fails(session_catalog: Catalog, format_v exception_cause = exc_info.value.__cause__ assert isinstance(exception_cause, TypeError) assert ( - "Iceberg does not yet support 'ns' timestamp precision. Use 'downcast-ns-timestamp-to-us-on-write' " - "configuration property to automatically downcast 'ns' to 'us' on write." in exception_cause.args[0] + "Iceberg does not yet support 'ns' timestamp precision. Use 'downcast-ns-timestamp-to-us' " + "configuration property to automatically downcast 'ns' to 'us'." in exception_cause.args[0] ) diff --git a/tests/integration/test_writes/test_writes.py b/tests/integration/test_writes/test_writes.py index 30fdd76ab7..7d77c294af 100644 --- a/tests/integration/test_writes/test_writes.py +++ b/tests/integration/test_writes/test_writes.py @@ -1536,7 +1536,7 @@ def test_write_all_timestamp_precision( arrow_table_schema_with_all_microseconds_timestamp_precisions: pa.Schema, ) -> None: identifier = "default.table_all_timestamp_precision" - mocker.patch.dict(os.environ, values={"PYICEBERG_DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE": "True"}) + mocker.patch.dict(os.environ, values={"PYICEBERG_DOWNCAST_NS_TIMESTAMP_TO_US": "True"}) tbl = _create_table( session_catalog, diff --git a/tests/io/test_pyarrow_visitor.py b/tests/io/test_pyarrow_visitor.py index e98d76e262..4bfb5fd474 100644 --- a/tests/io/test_pyarrow_visitor.py +++ b/tests/io/test_pyarrow_visitor.py @@ -197,13 +197,101 @@ def test_pyarrow_timestamp_invalid_units() -> None: with pytest.raises( TypeError, match=re.escape( - "Iceberg does not yet support 'ns' timestamp precision. Use 'downcast-ns-timestamp-to-us-on-write' " - "configuration property to automatically downcast 'ns' to 'us' on write." + "Iceberg does not yet support 'ns' timestamp precision. Use 'downcast-ns-timestamp-to-us' " + "configuration property to automatically downcast 'ns' to 'us'." ), ): visit_pyarrow(pyarrow_type, _ConvertToIceberg()) +def test_downcast_ns_timestamp_legacy_env_var_is_backwards_compat() -> None: + """The deprecated PYICEBERG_DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE env var still activates downcasting.""" + import os + import warnings + + from pyiceberg.table import _get_downcast_ns_timestamp_to_us + + env_key = "PYICEBERG_DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE" + old_value = os.environ.get(env_key) + try: + os.environ[env_key] = "True" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + result = _get_downcast_ns_timestamp_to_us() + assert result is True, "Legacy env var should still activate downcasting" + deprecation_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + assert len(deprecation_warnings) == 1 + assert "downcast-ns-timestamp-to-us-on-write" in str(deprecation_warnings[0].message) + assert "downcast-ns-timestamp-to-us" in str(deprecation_warnings[0].message) + finally: + if old_value is None: + os.environ.pop(env_key, None) + else: + os.environ[env_key] = old_value + + +def test_downcast_ns_timestamp_new_env_var_takes_precedence() -> None: + """The new PYICEBERG_DOWNCAST_NS_TIMESTAMP_TO_US env var works and emits no deprecation warning.""" + import os + import warnings + + from pyiceberg.table import _get_downcast_ns_timestamp_to_us + + new_key = "PYICEBERG_DOWNCAST_NS_TIMESTAMP_TO_US" + old_key = "PYICEBERG_DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE" + old_new = os.environ.get(new_key) + old_legacy = os.environ.get(old_key) + try: + os.environ[new_key] = "True" + os.environ.pop(old_key, None) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + result = _get_downcast_ns_timestamp_to_us() + assert result is True + deprecation_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + assert len(deprecation_warnings) == 0, "New key must not emit a deprecation warning" + finally: + if old_new is None: + os.environ.pop(new_key, None) + else: + os.environ[new_key] = old_new + if old_legacy is None: + os.environ.pop(old_key, None) + else: + os.environ[old_key] = old_legacy + + +def test_downcast_ns_timestamp_new_key_overrides_legacy_key() -> None: + """When both keys are set, the new key wins and no deprecation warning is emitted.""" + import os + import warnings + + from pyiceberg.table import _get_downcast_ns_timestamp_to_us + + new_key = "PYICEBERG_DOWNCAST_NS_TIMESTAMP_TO_US" + old_key = "PYICEBERG_DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE" + old_new = os.environ.get(new_key) + old_legacy = os.environ.get(old_key) + try: + os.environ[new_key] = "False" + os.environ[old_key] = "True" # legacy says True, but new key says False + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + result = _get_downcast_ns_timestamp_to_us() + assert result is False, "New key must win over legacy key" + deprecation_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + assert len(deprecation_warnings) == 0, "New key present: no deprecation warning expected" + finally: + if old_new is None: + os.environ.pop(new_key, None) + else: + os.environ[new_key] = old_new + if old_legacy is None: + os.environ.pop(old_key, None) + else: + os.environ[old_key] = old_legacy + + def test_pyarrow_timestamp_tz_to_iceberg() -> None: pyarrow_type = pa.timestamp(unit="us", tz="UTC") pyarrow_type_zero_offset = pa.timestamp(unit="us", tz="+00:00") @@ -220,8 +308,8 @@ def test_pyarrow_timestamp_tz_invalid_units() -> None: with pytest.raises( TypeError, match=re.escape( - "Iceberg does not yet support 'ns' timestamp precision. Use 'downcast-ns-timestamp-to-us-on-write' " - "configuration property to automatically downcast 'ns' to 'us' on write." + "Iceberg does not yet support 'ns' timestamp precision. Use 'downcast-ns-timestamp-to-us' " + "configuration property to automatically downcast 'ns' to 'us'." ), ): visit_pyarrow(pyarrow_type, _ConvertToIceberg()) From 563aa3c76d3821f7317047e25032e1104d76fc56 Mon Sep 17 00:00:00 2001 From: Jared Yu Date: Fri, 21 Aug 2026 11:35:09 -0700 Subject: [PATCH 2/3] Make helper public and use deprecation_message utility --- pyiceberg/catalog/__init__.py | 4 ++-- pyiceberg/io/pyarrow.py | 8 ++++---- pyiceberg/table/__init__.py | 22 +++++++++++----------- tests/io/test_pyarrow_visitor.py | 12 ++++++------ 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/pyiceberg/catalog/__init__.py b/pyiceberg/catalog/__init__.py index d31d29f51d..ed6e9571c9 100644 --- a/pyiceberg/catalog/__init__.py +++ b/pyiceberg/catalog/__init__.py @@ -51,7 +51,7 @@ StagedTable, Table, TableProperties, - _get_downcast_ns_timestamp_to_us, + get_downcast_ns_timestamp_to_us, ) from pyiceberg.table.locations import load_location_provider from pyiceberg.table.metadata import TableMetadata, TableMetadataV1, new_table_metadata @@ -847,7 +847,7 @@ def _convert_schema_if_needed( from pyiceberg.io.pyarrow import _ConvertToIcebergWithoutIDs, visit_pyarrow - downcast_ns_timestamp_to_us = _get_downcast_ns_timestamp_to_us() + downcast_ns_timestamp_to_us = get_downcast_ns_timestamp_to_us() if isinstance(schema, pa.Schema): schema: Schema = visit_pyarrow( # type: ignore schema, diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index a1e763dd05..af79d51198 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -146,7 +146,7 @@ visit, visit_with_partner, ) -from pyiceberg.table import TableProperties, _get_downcast_ns_timestamp_to_us +from pyiceberg.table import TableProperties, get_downcast_ns_timestamp_to_us from pyiceberg.table.deletion_vector import deletion_vectors_from_puffin_file from pyiceberg.table.locations import load_location_provider from pyiceberg.table.metadata import TableMetadata @@ -1776,7 +1776,7 @@ def __init__( self._bound_row_filter = bind(table_metadata.schema(), row_filter, case_sensitive=case_sensitive) self._case_sensitive = case_sensitive self._limit = limit - self._downcast_ns_timestamp_to_us = _get_downcast_ns_timestamp_to_us() + self._downcast_ns_timestamp_to_us = get_downcast_ns_timestamp_to_us() self._dictionary_columns = dictionary_columns @property @@ -2733,7 +2733,7 @@ def write_data_file(task: WriteTask) -> DataFile: else: file_schema = table_schema - downcast_ns_timestamp_to_us = _get_downcast_ns_timestamp_to_us() + downcast_ns_timestamp_to_us = get_downcast_ns_timestamp_to_us() batches = [ _to_requested_schema( requested_schema=file_schema, @@ -2990,7 +2990,7 @@ def _dataframe_to_data_files( default=TableProperties.WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT, ) name_mapping = table_metadata.schema().name_mapping - downcast_ns_timestamp_to_us = _get_downcast_ns_timestamp_to_us() + downcast_ns_timestamp_to_us = get_downcast_ns_timestamp_to_us() task_schema = pyarrow_to_schema( df.schema, name_mapping=name_mapping, diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index fc4f1cdede..5029953ae3 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -104,6 +104,7 @@ from pyiceberg.types import strtobool from pyiceberg.utils.concurrent import ExecutorFactory from pyiceberg.utils.config import Config +from pyiceberg.utils.deprecated import deprecation_message from pyiceberg.utils.properties import property_as_bool, property_as_int if TYPE_CHECKING: @@ -129,7 +130,7 @@ DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE = "downcast-ns-timestamp-to-us-on-write" -def _get_downcast_ns_timestamp_to_us() -> bool: +def get_downcast_ns_timestamp_to_us() -> bool: """Return the effective value of the downcast-ns-timestamp-to-us config. Checks the new ``downcast-ns-timestamp-to-us`` key first. If not set, falls @@ -144,12 +145,11 @@ def _get_downcast_ns_timestamp_to_us() -> bool: legacy = config.get_bool(DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE) if legacy is not None: - warnings.warn( - f"Config key '{DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE}' (env: PYICEBERG_DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE) " - f"is deprecated. Use '{DOWNCAST_NS_TIMESTAMP_TO_US}' " - "(env: PYICEBERG_DOWNCAST_NS_TIMESTAMP_TO_US) instead.", - DeprecationWarning, - stacklevel=2, + deprecation_message( + deprecated_in="0.9.0", + removed_in="0.10.0", + help_message="Config key 'downcast-ns-timestamp-to-us-on-write' is deprecated. " + "Use 'downcast-ns-timestamp-to-us' (env: PYICEBERG_DOWNCAST_NS_TIMESTAMP_TO_US) instead.", ) return legacy @@ -573,7 +573,7 @@ def append( if not isinstance(df, (pa.Table, pa.RecordBatchReader)): raise ValueError(f"Expected pa.Table or pa.RecordBatchReader, got: {df}") - downcast_ns_timestamp_to_us = _get_downcast_ns_timestamp_to_us() + downcast_ns_timestamp_to_us = get_downcast_ns_timestamp_to_us() _check_pyarrow_schema_compatible( self.table_metadata.schema(), provided_schema=df.schema, @@ -629,7 +629,7 @@ def dynamic_partition_overwrite( f"in the latest partition spec: {field}" ) - downcast_ns_timestamp_to_us = _get_downcast_ns_timestamp_to_us() + downcast_ns_timestamp_to_us = get_downcast_ns_timestamp_to_us() _check_pyarrow_schema_compatible( self.table_metadata.schema(), provided_schema=df.schema, @@ -738,7 +738,7 @@ def overwrite( if not isinstance(df, (pa.Table, pa.RecordBatchReader)): raise ValueError(f"Expected pa.Table or pa.RecordBatchReader, got: {df}") - downcast_ns_timestamp_to_us = _get_downcast_ns_timestamp_to_us() + downcast_ns_timestamp_to_us = get_downcast_ns_timestamp_to_us() _check_pyarrow_schema_compatible( self.table_metadata.schema(), provided_schema=df.schema, @@ -945,7 +945,7 @@ def upsert( from pyiceberg.io.pyarrow import _check_pyarrow_schema_compatible - downcast_ns_timestamp_to_us = _get_downcast_ns_timestamp_to_us() + downcast_ns_timestamp_to_us = get_downcast_ns_timestamp_to_us() _check_pyarrow_schema_compatible( self.table_metadata.schema(), provided_schema=df.schema, diff --git a/tests/io/test_pyarrow_visitor.py b/tests/io/test_pyarrow_visitor.py index 4bfb5fd474..4a1a454fc7 100644 --- a/tests/io/test_pyarrow_visitor.py +++ b/tests/io/test_pyarrow_visitor.py @@ -209,7 +209,7 @@ def test_downcast_ns_timestamp_legacy_env_var_is_backwards_compat() -> None: import os import warnings - from pyiceberg.table import _get_downcast_ns_timestamp_to_us + from pyiceberg.table import get_downcast_ns_timestamp_to_us env_key = "PYICEBERG_DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE" old_value = os.environ.get(env_key) @@ -217,7 +217,7 @@ def test_downcast_ns_timestamp_legacy_env_var_is_backwards_compat() -> None: os.environ[env_key] = "True" with warnings.catch_warnings(record=True) as caught: warnings.simplefilter("always") - result = _get_downcast_ns_timestamp_to_us() + result = get_downcast_ns_timestamp_to_us() assert result is True, "Legacy env var should still activate downcasting" deprecation_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] assert len(deprecation_warnings) == 1 @@ -235,7 +235,7 @@ def test_downcast_ns_timestamp_new_env_var_takes_precedence() -> None: import os import warnings - from pyiceberg.table import _get_downcast_ns_timestamp_to_us + from pyiceberg.table import get_downcast_ns_timestamp_to_us new_key = "PYICEBERG_DOWNCAST_NS_TIMESTAMP_TO_US" old_key = "PYICEBERG_DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE" @@ -246,7 +246,7 @@ def test_downcast_ns_timestamp_new_env_var_takes_precedence() -> None: os.environ.pop(old_key, None) with warnings.catch_warnings(record=True) as caught: warnings.simplefilter("always") - result = _get_downcast_ns_timestamp_to_us() + result = get_downcast_ns_timestamp_to_us() assert result is True deprecation_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] assert len(deprecation_warnings) == 0, "New key must not emit a deprecation warning" @@ -266,7 +266,7 @@ def test_downcast_ns_timestamp_new_key_overrides_legacy_key() -> None: import os import warnings - from pyiceberg.table import _get_downcast_ns_timestamp_to_us + from pyiceberg.table import get_downcast_ns_timestamp_to_us new_key = "PYICEBERG_DOWNCAST_NS_TIMESTAMP_TO_US" old_key = "PYICEBERG_DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE" @@ -277,7 +277,7 @@ def test_downcast_ns_timestamp_new_key_overrides_legacy_key() -> None: os.environ[old_key] = "True" # legacy says True, but new key says False with warnings.catch_warnings(record=True) as caught: warnings.simplefilter("always") - result = _get_downcast_ns_timestamp_to_us() + result = get_downcast_ns_timestamp_to_us() assert result is False, "New key must win over legacy key" deprecation_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] assert len(deprecation_warnings) == 0, "New key present: no deprecation warning expected" From c541c98523b6fc26cc77634e195699690ef0a574 Mon Sep 17 00:00:00 2001 From: Jared Yu Date: Fri, 21 Aug 2026 14:22:54 -0700 Subject: [PATCH 3/3] Address review: fix deprecation versions and trim comment --- pyiceberg/table/__init__.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 5029953ae3..bb9c12e26a 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -125,8 +125,7 @@ ALWAYS_TRUE = AlwaysTrue() DOWNCAST_NS_TIMESTAMP_TO_US = "downcast-ns-timestamp-to-us" -# Deprecated: use DOWNCAST_NS_TIMESTAMP_TO_US. The old key said "on-write" but the -# config also controls read-path downcasting, so the name was misleading. +# Deprecated: use DOWNCAST_NS_TIMESTAMP_TO_US DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE = "downcast-ns-timestamp-to-us-on-write" @@ -146,8 +145,8 @@ def get_downcast_ns_timestamp_to_us() -> bool: legacy = config.get_bool(DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE) if legacy is not None: deprecation_message( - deprecated_in="0.9.0", - removed_in="0.10.0", + deprecated_in="0.12.0", + removed_in="0.13.0", help_message="Config key 'downcast-ns-timestamp-to-us-on-write' is deprecated. " "Use 'downcast-ns-timestamp-to-us' (env: PYICEBERG_DOWNCAST_NS_TIMESTAMP_TO_US) instead.", )