From 4022071990d6da217ab69b486ae6365b4eb0ba2f Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Tue, 3 Nov 2026 16:05:57 +0800 Subject: [PATCH 1/2] [python] Remove deletion vector and dedicated vector defaults for multimodal tables --- docs/docs/pypaimon/multimodal-tables.mdx | 22 +++++++++++-------- .../pypaimon/multimodal/connection.py | 2 -- .../pypaimon/tests/multimodal_table_test.py | 4 ++-- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/docs/docs/pypaimon/multimodal-tables.mdx b/docs/docs/pypaimon/multimodal-tables.mdx index dbb7e5c71812..d295802987d0 100644 --- a/docs/docs/pypaimon/multimodal-tables.mdx +++ b/docs/docs/pypaimon/multimodal-tables.mdx @@ -35,12 +35,14 @@ primary keys. Each `add`, `overwrite`, `update`, or `delete` call handles its ow commit; use the [Python API](./python-api) for explicit writer and commit control. Creating a multimodal table enables `row-tracking.enabled`, -`data-evolution.enabled`, `deletion-vectors.enabled`, and -`blob-as-descriptor` by default. Global index scans use full fallback coverage +`data-evolution.enabled`, and `blob-as-descriptor` by default. +Global index scans use full fallback coverage through `global-index.search-mode` set to `FULL`. Data files use Paimon's -default Parquet format, and vector columns are stored in dedicated Parquet -files through `vector.file.format` set to `parquet`. These defaults require no -file-format extra: +default Parquet format, including vector columns. To store vectors in dedicated +files, explicitly set `vector.file.format`, for example to `parquet`. +Deletion vectors are disabled by default; enable them with +`deletion-vectors.enabled` set to `true` in the table creation options. +These defaults require no file-format extra: ```shell pip install pypaimon @@ -365,9 +367,10 @@ or delete prepared files after a commit error. ## Delete `delete` removes rows matched by a SQL-like predicate. It uses the same -predicate syntax as `scan().where(...)` and `update(...)`. Multimodal tables -enable deletion vectors by default; on partitioned tables, a predicate that -references only partition columns uses the partition drop path. +predicate syntax as `scan().where(...)` and `update(...)`. Row-level deletes +require `deletion-vectors.enabled` set to `true` when creating the table. +On partitioned tables, a predicate that references only partition columns uses +the partition drop path. ```python docs.delete(where="id = 2") @@ -405,7 +408,8 @@ docs.merge("id") \ ]) ``` -Use `when_matched_delete()` for matched rows that should be removed: +Use `when_matched_delete()` for matched rows that should be removed. The table +must have `deletion-vectors.enabled` set to `true`: ```python docs.merge("id") \ diff --git a/paimon-python/pypaimon/multimodal/connection.py b/paimon-python/pypaimon/multimodal/connection.py index 647f412e0cc8..cdf83205dd6f 100644 --- a/paimon-python/pypaimon/multimodal/connection.py +++ b/paimon-python/pypaimon/multimodal/connection.py @@ -31,10 +31,8 @@ _DEFAULT_OPTIONS = { "row-tracking.enabled": "true", "data-evolution.enabled": "true", - "deletion-vectors.enabled": "true", "blob-as-descriptor": "true", "global-index.search-mode": "full", - "vector.file.format": "parquet", } _DEFAULT_DATABASE = Catalog.DEFAULT_DATABASE diff --git a/paimon-python/pypaimon/tests/multimodal_table_test.py b/paimon-python/pypaimon/tests/multimodal_table_test.py index 60a62bf936e0..a41b7fda7e9a 100644 --- a/paimon-python/pypaimon/tests/multimodal_table_test.py +++ b/paimon-python/pypaimon/tests/multimodal_table_test.py @@ -137,11 +137,11 @@ def test_create_table_defaults_data_evolution_options(self): options = table.raw_table.table_schema.options self.assertEqual("true", options["row-tracking.enabled"]) self.assertEqual("true", options["data-evolution.enabled"]) - self.assertEqual("true", options["deletion-vectors.enabled"]) + self.assertNotIn("deletion-vectors.enabled", options) self.assertEqual("true", options["blob-as-descriptor"]) self.assertNotIn("data-evolution.row-sidecar.enabled", options) self.assertEqual("full", options["global-index.search-mode"]) - self.assertEqual("parquet", options["vector.file.format"]) + self.assertNotIn("vector.file.format", options) self.assertEqual("default.docs", self.conn.get_table("docs").identifier) self.assertEqual(["id", "content", "embedding", "payload"], [field.name for field in table.raw_table.fields]) From 95ce97a7619a46d347b18baef921da86aac19633 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Tue, 3 Nov 2026 16:47:49 +0800 Subject: [PATCH 2/2] [python] Explicitly enable deletion vectors in multimodal tests --- paimon-python/pypaimon/tests/batch_vector_lookup_test.py | 1 + .../pypaimon/tests/full_text_scalar_filter_test.py | 1 + .../pypaimon/tests/global_index_live_row_filter_test.py | 3 ++- paimon-python/pypaimon/tests/multimodal_columns_test.py | 3 ++- paimon-python/pypaimon/tests/multimodal_lance_test.py | 2 +- .../pypaimon/tests/ray_vector_filter_exactness_test.py | 1 + paimon-python/pypaimon/tests/ray_vector_search_test.py | 1 + .../pypaimon/tests/search_result_metadata_test.py | 3 ++- paimon-python/pypaimon/tests/search_snapshot_test.py | 2 +- paimon-python/pypaimon/tests/torch_read_test.py | 4 +++- .../pypaimon/tests/vector_filter_exactness_test.py | 2 +- .../pypaimon/tests/vector_index_dv_build_test.py | 8 ++++++-- 12 files changed, 22 insertions(+), 9 deletions(-) diff --git a/paimon-python/pypaimon/tests/batch_vector_lookup_test.py b/paimon-python/pypaimon/tests/batch_vector_lookup_test.py index 229c9cf3a5eb..8784972dfd15 100644 --- a/paimon-python/pypaimon/tests/batch_vector_lookup_test.py +++ b/paimon-python/pypaimon/tests/batch_vector_lookup_test.py @@ -36,6 +36,7 @@ def docs(tmp_path, request): table = pm.connect(options={"warehouse": str(tmp_path)}).create_table( "docs", schema=schema, partitioned=["category"], options={ "file.format": "parquet", "vector.file.format": "parquet", + "deletion-vectors.enabled": "true", "vector-index.search-mode": "full", "read.parallelism": str(request.param), "read.batch-size": "2", "blob-as-descriptor": "false", }) diff --git a/paimon-python/pypaimon/tests/full_text_scalar_filter_test.py b/paimon-python/pypaimon/tests/full_text_scalar_filter_test.py index f0f0a4d99166..b1953ba65806 100644 --- a/paimon-python/pypaimon/tests/full_text_scalar_filter_test.py +++ b/paimon-python/pypaimon/tests/full_text_scalar_filter_test.py @@ -33,6 +33,7 @@ def docs(tmp_path): table = pm.connect(options={"warehouse": str(tmp_path)}).create_table( "docs", schema=schema, partitioned=["pt"], options={ "file.format": "parquet", "vector.file.format": "parquet", "read.batch-size": "1", + "deletion-vectors.enabled": "true", "full-text-index.search-mode": "full", "vector-index.search-mode": "full"}) table.add(pa.table({"id": [0, 1, 2], "text": ["paimon", "paimon paimon", "paimon long document"], "label": ["other", "other", "target"], "pt": ["a"] * 3, diff --git a/paimon-python/pypaimon/tests/global_index_live_row_filter_test.py b/paimon-python/pypaimon/tests/global_index_live_row_filter_test.py index 1380ad0e38e0..1ac0e8d21374 100644 --- a/paimon-python/pypaimon/tests/global_index_live_row_filter_test.py +++ b/paimon-python/pypaimon/tests/global_index_live_row_filter_test.py @@ -27,7 +27,8 @@ def test_scoped_live_rows_only_load_relevant_deletions(tmp_path): table = pm.connect(options={"warehouse": str(tmp_path)}).create_table( - "rows", schema=pa.schema([("id", pa.int64())]), options={"file.format": "parquet"}) + "rows", schema=pa.schema([("id", pa.int64())]), options={ + "file.format": "parquet", "deletion-vectors.enabled": "true"}) table.add(pa.table({"id": [0, 1, 2]})) table.add(pa.table({"id": [3, 4, 5]})) snapshot = table.raw_table.snapshot_manager().get_latest_snapshot() diff --git a/paimon-python/pypaimon/tests/multimodal_columns_test.py b/paimon-python/pypaimon/tests/multimodal_columns_test.py index ebe3b4cccd59..8c9e93dc93e8 100644 --- a/paimon-python/pypaimon/tests/multimodal_columns_test.py +++ b/paimon-python/pypaimon/tests/multimodal_columns_test.py @@ -36,7 +36,8 @@ def frames(tmp_path): ]), data=[{"id": i, "label": "old", "image": bytes([i]) * 100} for i in range(5)], - options={"file.format": "parquet", "read.batch-size": "2"}, + options={"file.format": "parquet", "read.batch-size": "2", + "deletion-vectors.enabled": "true"}, ) diff --git a/paimon-python/pypaimon/tests/multimodal_lance_test.py b/paimon-python/pypaimon/tests/multimodal_lance_test.py index d4599b376065..37f04e479eb0 100644 --- a/paimon-python/pypaimon/tests/multimodal_lance_test.py +++ b/paimon-python/pypaimon/tests/multimodal_lance_test.py @@ -92,7 +92,7 @@ def test_export_lance_exports_logical_updates_deletes_and_historical_schema(conn }, [ {"id": 1, "text": "before", "image": b"before"}, {"id": 2, "text": "deleted", "image": b"deleted"}, - ]) + ], options={"deletion-vectors.enabled": "true"}) snapshot_id = table.raw_table.snapshot_manager().get_latest_snapshot().id table.raw_table.create_tag("training-v1", snapshot_id=snapshot_id) conn.catalog.alter_table("default.history", [ diff --git a/paimon-python/pypaimon/tests/ray_vector_filter_exactness_test.py b/paimon-python/pypaimon/tests/ray_vector_filter_exactness_test.py index ed1d3c02cb1f..63350a5ddf6a 100644 --- a/paimon-python/pypaimon/tests/ray_vector_filter_exactness_test.py +++ b/paimon-python/pypaimon/tests/ray_vector_filter_exactness_test.py @@ -108,6 +108,7 @@ def test_worker_filters_overlapping_scalar_shards_at_planned_snapshot(tmp_path, table = pm.connect(options={"warehouse": str(tmp_path)}).create_table( "vectors", schema=schema, options={ "file.format": "parquet", "vector.file.format": "parquet", + "deletion-vectors.enabled": "true", "global-index.filter.refine-from-data": "true"}) for start in range(0, 12, 3): row_ids = list(range(start, start + 3)) diff --git a/paimon-python/pypaimon/tests/ray_vector_search_test.py b/paimon-python/pypaimon/tests/ray_vector_search_test.py index b6e1c74f5ca0..f40ce3bcd0a9 100644 --- a/paimon-python/pypaimon/tests/ray_vector_search_test.py +++ b/paimon-python/pypaimon/tests/ray_vector_search_test.py @@ -57,6 +57,7 @@ def table(tmp_path): return pm.connect(options={"warehouse": str(tmp_path)}).create_table( "vectors", schema=SCHEMA, options={"file.format": "parquet", "vector.file.format": "parquet", + "deletion-vectors.enabled": "true", "vector-index.search-mode": "full", "read.batch-size": "2", "source.split.target-size": "1 b", "source.split.open-file-cost": "1 b"}) diff --git a/paimon-python/pypaimon/tests/search_result_metadata_test.py b/paimon-python/pypaimon/tests/search_result_metadata_test.py index 36fb34f413c3..54d1c98f09b0 100644 --- a/paimon-python/pypaimon/tests/search_result_metadata_test.py +++ b/paimon-python/pypaimon/tests/search_result_metadata_test.py @@ -30,7 +30,8 @@ def docs(tmp_path): ("embedding", pa.list_(pa.float32(), 2))]) table = pm.connect(options={"warehouse": str(tmp_path)}).create_table( "docs", schema=schema, options={"file.format": "parquet", "vector.file.format": "parquet", - "vector-index.search-mode": "full", "read.batch-size": "1"}) + "vector-index.search-mode": "full", "read.batch-size": "1", + "deletion-vectors.enabled": "true"}) table.add(pa.table({"id": [0, 1, 2, 3], "text": ["paimon long document", "paimon", "other", "paimon"], "embedding": [[2., 1.], [0., 1.], [1., 1.], [0., 1.]]}, schema=schema)) return table diff --git a/paimon-python/pypaimon/tests/search_snapshot_test.py b/paimon-python/pypaimon/tests/search_snapshot_test.py index 78c614fe475c..785e2af619af 100644 --- a/paimon-python/pypaimon/tests/search_snapshot_test.py +++ b/paimon-python/pypaimon/tests/search_snapshot_test.py @@ -40,7 +40,7 @@ def table(tmp_path): return pm.connect(options={"warehouse": str(tmp_path)}).create_table( "vectors", schema=SCHEMA, options={"file.format": "parquet", "vector.file.format": "parquet", - "vector-index.search-mode": "full"}) + "vector-index.search-mode": "full", "deletion-vectors.enabled": "true"}) def search(table, kind="vector", **options): diff --git a/paimon-python/pypaimon/tests/torch_read_test.py b/paimon-python/pypaimon/tests/torch_read_test.py index c961827600b3..23cad98c3e61 100644 --- a/paimon-python/pypaimon/tests/torch_read_test.py +++ b/paimon-python/pypaimon/tests/torch_read_test.py @@ -2045,7 +2045,9 @@ def test_torch_batch_shuffle_vectors_deletions_and_snapshot(self): schema = pa.schema([('id', pa.int64()), ('vector', pa.list_(pa.float32(), 2))]) table = pm.connect(options={'warehouse': self.warehouse}).create_table( - 'batch_shuffle_vectors', schema=schema, options={'file.format': 'parquet'}) + 'batch_shuffle_vectors', schema=schema, options={ + 'file.format': 'parquet', 'vector.file.format': 'parquet', + 'deletion-vectors.enabled': 'true'}) table.add(pa.table({'id': list(range(8)), 'vector': [[i, i + 1] for i in range(8)]}, schema=schema)) snapshot = table.raw_table.snapshot_manager().get_latest_snapshot().id table.delete('id = 2') diff --git a/paimon-python/pypaimon/tests/vector_filter_exactness_test.py b/paimon-python/pypaimon/tests/vector_filter_exactness_test.py index 1d6fef297941..3c096a4ee8f6 100644 --- a/paimon-python/pypaimon/tests/vector_filter_exactness_test.py +++ b/paimon-python/pypaimon/tests/vector_filter_exactness_test.py @@ -42,7 +42,7 @@ def table(tmp_path): schema = pa.schema([("id", pa.int64()), ("name", pa.string()), ("embedding", pa.list_(pa.float32(), 2))]) table = pm.connect(options={"warehouse": str(tmp_path)}).create_table( "vectors", schema=schema, options={"file.format": "parquet", "vector.file.format": "parquet", - "read.batch-size": "1"}) + "read.batch-size": "1", "deletion-vectors.enabled": "true"}) table.add(pa.table({"id": [0, 1, 2], "name": ["alpha", "beta zeta", "gamma"], "embedding": [[0., 1.], [1., 1.], [2., 1.]]}, schema=schema)) table.raw_table.create_global_index( diff --git a/paimon-python/pypaimon/tests/vector_index_dv_build_test.py b/paimon-python/pypaimon/tests/vector_index_dv_build_test.py index f8766fbc9f06..07f2befd7e08 100644 --- a/paimon-python/pypaimon/tests/vector_index_dv_build_test.py +++ b/paimon-python/pypaimon/tests/vector_index_dv_build_test.py @@ -46,6 +46,7 @@ def create_table(tmp_path, null_vector=False): table = pm.connect(options={'warehouse': str(tmp_path)}).create_table( 'vectors', schema=schema, options={ 'file.format': 'parquet', 'vector.file.format': 'parquet', 'read.batch-size': '2', + 'deletion-vectors.enabled': 'true', 'global-index.row-count-per-shard': '4', 'global-index.build.parallelism': '1'}) append(table, list(range(7)), null_vector) return table @@ -123,7 +124,8 @@ def test_python_ivfpq_search_keeps_live_null_vectors_out_of_results(tmp_path, ba schema = pa.schema([('id', pa.int64()), ('embedding', pa.list_(pa.float32()))]) table = pm.connect(options={'warehouse': str(tmp_path)}).create_table( 'vectors', schema=schema, options={ - 'file.format': 'parquet', 'vector.file.format': 'parquet', 'vector-index.search-mode': 'fast'}) + 'file.format': 'parquet', 'vector.file.format': 'parquet', 'vector-index.search-mode': 'fast', + 'deletion-vectors.enabled': 'true'}) table.add(pa.table({ 'id': list(range(5)), 'embedding': [[1., 0.], [0., 1.], None, [1., 1.], [2., 2.]]}, schema=schema)) table.delete('id = 0 OR id = 3') @@ -178,7 +180,9 @@ def test_native_incremental_build_with_concurrent_delete(table): def test_native_partition_scoped_build_after_deletion(tmp_path): pytest.importorskip('paimon_vindex') table = pm.connect(options={'warehouse': str(tmp_path)}).create_table( - 'vectors', schema=SCHEMA, partitioned=['category'], options={'file.format': 'parquet'}) + 'vectors', schema=SCHEMA, partitioned=['category'], options={ + 'file.format': 'parquet', 'vector.file.format': 'parquet', + 'deletion-vectors.enabled': 'true'}) append(table, [0, 1, 2], category='other') append(table, [3, 4, 5]) table.delete("category = 'yes' AND id = 4")