From f574ebcbd468121b58f94938583fd632ea8ae676 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 2 Sep 2026 15:33:29 -0600 Subject: [PATCH] test: exercise the Iceberg write split-operator plan in Iceberg's own suites The split-operator plan (IcebergCommit -> IcebergWrite) ships off by default, so Iceberg's Spark suites have never run against it. All four failure buckets catalogued in #5259 have since been fixed, but nothing in CI covers three of them. Enable spark.comet.write.iceberg.splitOperator.enabled in every Comet-configured SparkSession builder in the Iceberg test diffs, alongside the existing native scan config. No user-facing default changes. The diffs are regenerated rather than hand-edited, which also restores canonical path ordering: the TestSystemFunctionPushDownInRowLevelOperations hunks had been appended at the end of each diff instead of sorted into place. --- dev/diffs/iceberg/1.10.0.diff | 449 +++++++++-------- dev/diffs/iceberg/1.11.0.diff | 231 +++++---- dev/diffs/iceberg/1.8.1.diff | 453 ++++++++++-------- dev/diffs/iceberg/1.9.1.diff | 453 ++++++++++-------- .../contributor-guide/iceberg-spark-tests.md | 5 + 5 files changed, 908 insertions(+), 683 deletions(-) diff --git a/dev/diffs/iceberg/1.10.0.diff b/dev/diffs/iceberg/1.10.0.diff index 310b5414995..7bbf584c31e 100644 --- a/dev/diffs/iceberg/1.10.0.diff +++ b/dev/diffs/iceberg/1.10.0.diff @@ -1,5 +1,5 @@ diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml -index eeabe54f5f..a76d7bdcbc 100644 +index eeabe54f5f..72873ccbc9 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -37,7 +37,7 @@ awssdk-s3accessgrants = "2.3.0" @@ -24,10 +24,10 @@ index 714be0831d..dede65f213 100644 // runtime dependencies for running Hive Catalog based integration test integrationRuntimeOnly project(':iceberg-hive-metastore') diff --git a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java -index 4c1a509591..6220ae2b98 100644 +index 4c1a509591..186a264a88 100644 --- a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java +++ b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java -@@ -59,7 +59,23 @@ public abstract class ExtensionsTestBase extends CatalogTestBase { +@@ -59,7 +59,25 @@ public abstract class ExtensionsTestBase extends CatalogTestBase { .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") .config( SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), String.valueOf(RANDOM.nextBoolean())) @@ -37,6 +37,7 @@ index 4c1a509591..6220ae2b98 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -46,16 +47,17 @@ index 4c1a509591..6220ae2b98 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); TestBase.sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext()); diff --git a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java -index 58d054bd05..61b307b92b 100644 +index 58d054bd05..ef1c1cb6d1 100644 --- a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java +++ b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java -@@ -56,6 +56,14 @@ public class TestCallStatementParser { +@@ -56,6 +56,15 @@ public class TestCallStatementParser { .master("local[2]") .config("spark.sql.extensions", IcebergSparkSessionExtensions.class.getName()) .config("spark.extra.prop", "value") @@ -65,16 +67,45 @@ index 58d054bd05..61b307b92b 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); TestCallStatementParser.parser = spark.sessionState().sqlParser(); } +diff --git a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java +index 5a0a04edba..afe92a8b72 100644 +--- a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java ++++ b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java +@@ -38,6 +38,7 @@ import org.apache.spark.sql.catalyst.expressions.ApplyFunctionExpression; + import org.apache.spark.sql.catalyst.expressions.Expression; + import org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke; + import org.apache.spark.sql.execution.CommandResultExec; ++import org.apache.spark.sql.execution.SparkPlan; + import org.apache.spark.sql.execution.datasources.v2.V2TableWriteExec; + import org.junit.jupiter.api.AfterEach; + import org.junit.jupiter.api.BeforeEach; +@@ -313,9 +314,13 @@ public class TestSystemFunctionPushDownInRowLevelOperations extends ExtensionsTe + + private List executeAndCollectFunctionCalls(String query, Object... args) { + CommandResultExec command = (CommandResultExec) executeAndKeepPlan(query, args); +- V2TableWriteExec write = (V2TableWriteExec) command.commandPhysicalPlan(); ++ // Comet's split-operator Iceberg write plans the command as IcebergCommitExec, which is not ++ // a V2TableWriteExec; collect over the whole command plan in that case. ++ SparkPlan plan = command.commandPhysicalPlan(); ++ SparkPlan queryPlan = ++ plan instanceof V2TableWriteExec ? ((V2TableWriteExec) plan).query() : plan; + return SparkPlanUtil.collectExprs( +- write.query(), ++ queryPlan, + expr -> expr instanceof StaticInvoke || expr instanceof ApplyFunctionExpression); + } + diff --git a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java -index b6ade2bff3..b52d011691 100644 +index b6ade2bff3..90a4839dad 100644 --- a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java +++ b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java -@@ -170,6 +170,14 @@ public class DeleteOrphanFilesBenchmark { +@@ -170,6 +170,15 @@ public class DeleteOrphanFilesBenchmark { .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", catalogWarehouse()) @@ -84,16 +115,17 @@ index b6ade2bff3..b52d011691 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local"); spark = builder.getOrCreate(); } diff --git a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java -index 7cd41bd7e7..7830b9d495 100644 +index 7cd41bd7e7..0b6d490529 100644 --- a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java +++ b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java -@@ -386,6 +386,14 @@ public class IcebergSortCompactionBenchmark { +@@ -386,6 +386,15 @@ public class IcebergSortCompactionBenchmark { "spark.sql.catalog.spark_catalog", "org.apache.iceberg.spark.SparkSessionCatalog") .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", getCatalogWarehouse()) @@ -103,16 +135,17 @@ index 7cd41bd7e7..7830b9d495 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[*]"); spark = builder.getOrCreate(); Configuration sparkHadoopConf = spark.sessionState().newHadoopConf(); diff --git a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java -index 68c537e34a..5ef031963a 100644 +index 68c537e34a..2e421c7706 100644 --- a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java +++ b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java -@@ -94,7 +94,17 @@ public abstract class IcebergSourceBenchmark { +@@ -94,7 +94,18 @@ public abstract class IcebergSourceBenchmark { } protected void setupSpark(boolean enableDictionaryEncoding) { @@ -126,6 +159,7 @@ index 68c537e34a..5ef031963a 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g"); if (!enableDictionaryEncoding) { @@ -913,10 +947,10 @@ index a0f45e7610..78e7845911 100644 .filter(residual) .caseSensitive(caseSensitive()) diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java -index 404ba72846..5bd4e2b351 100644 +index 404ba72846..b375458f05 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java -@@ -90,6 +90,14 @@ public abstract class SparkDistributedDataScanTestBase +@@ -90,6 +90,15 @@ public abstract class SparkDistributedDataScanTestBase .master("local[2]") .config("spark.serializer", serializer) .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -926,16 +960,17 @@ index 404ba72846..5bd4e2b351 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java -index 9361c63176..44c59c8971 100644 +index 9361c63176..16ce1c614f 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java -@@ -69,6 +69,14 @@ public class TestSparkDistributedDataScanDeletes +@@ -69,6 +69,15 @@ public class TestSparkDistributedDataScanDeletes .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -945,16 +980,17 @@ index 9361c63176..44c59c8971 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java -index a218f965ea..b1e0fcf2ca 100644 +index a218f965ea..e30a8d6c6f 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java -@@ -62,6 +62,14 @@ public class TestSparkDistributedDataScanFilterFiles +@@ -62,6 +62,15 @@ public class TestSparkDistributedDataScanFilterFiles .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -964,16 +1000,17 @@ index a218f965ea..b1e0fcf2ca 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java -index acd4688440..fffc604b48 100644 +index acd4688440..33732e6654 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java -@@ -59,6 +59,14 @@ public class TestSparkDistributedDataScanReporting +@@ -59,6 +59,15 @@ public class TestSparkDistributedDataScanReporting .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -983,16 +1020,17 @@ index acd4688440..fffc604b48 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/TestBase.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/TestBase.java -index 3c32b46936..76c7683aa9 100644 +index 3c32b46936..2a39c4605c 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/TestBase.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/TestBase.java -@@ -79,6 +79,14 @@ public abstract class TestBase extends SparkTestHelperBase { +@@ -79,6 +79,15 @@ public abstract class TestBase extends SparkTestHelperBase { .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") @@ -1002,16 +1040,17 @@ index 3c32b46936..76c7683aa9 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() .getOrCreate(); diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java -index 6647a1b483..0f11bb8e0c 100644 +index 6647a1b483..aeb16f85ac 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java -@@ -62,6 +62,14 @@ public abstract class ScanTestBase extends AvroDataTestBase { +@@ -62,6 +62,15 @@ public abstract class ScanTestBase extends AvroDataTestBase { SparkSession.builder() .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) .master("local[2]") @@ -1021,16 +1060,17 @@ index 6647a1b483..0f11bb8e0c 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); ScanTestBase.sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java -index 24a14bb64d..531193574c 100644 +index 24a14bb64d..9329cd99b3 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java -@@ -144,7 +144,18 @@ public class TestCompressionSettings extends CatalogTestBase { +@@ -144,7 +144,19 @@ public class TestCompressionSettings extends CatalogTestBase { @BeforeAll public static void startSpark() { @@ -1044,6 +1084,7 @@ index 24a14bb64d..531193574c 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1051,10 +1092,10 @@ index 24a14bb64d..531193574c 100644 @BeforeEach diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java -index 99d3f38ee7..3049a1d314 100644 +index 99d3f38ee7..f1a74280dd 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java -@@ -74,7 +74,18 @@ public class TestDataSourceOptions extends TestBaseWithCatalog { +@@ -74,7 +74,19 @@ public class TestDataSourceOptions extends TestBaseWithCatalog { @BeforeAll public static void startSpark() { @@ -1068,6 +1109,7 @@ index 99d3f38ee7..3049a1d314 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1075,10 +1117,10 @@ index 99d3f38ee7..3049a1d314 100644 @AfterAll diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java -index a70aeb1ba6..95b47ac0d5 100644 +index a70aeb1ba6..fae2359d01 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java -@@ -123,6 +123,14 @@ public class TestFilteredScan { +@@ -123,6 +123,15 @@ public class TestFilteredScan { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -1088,16 +1130,17 @@ index a70aeb1ba6..95b47ac0d5 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); // define UDFs used by partition tests diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java -index c03f7b94ec..56a29fc6f7 100644 +index c03f7b94ec..30032fa107 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java -@@ -99,6 +99,14 @@ public class TestForwardCompatibility { +@@ -99,6 +99,15 @@ public class TestForwardCompatibility { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -1107,16 +1150,17 @@ index c03f7b94ec..56a29fc6f7 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java -index f4f57157e4..d8f3fa1cd0 100644 +index f4f57157e4..418364e3da 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java -@@ -51,6 +51,14 @@ public class TestIcebergSpark { +@@ -51,6 +51,15 @@ public class TestIcebergSpark { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -1126,16 +1170,17 @@ index f4f57157e4..d8f3fa1cd0 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java -index e1402396fa..5f02d6a2b7 100644 +index e1402396fa..3092902873 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java -@@ -118,6 +118,14 @@ public class TestPartitionPruning { +@@ -118,6 +118,15 @@ public class TestPartitionPruning { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -1145,16 +1190,17 @@ index e1402396fa..5f02d6a2b7 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); TestPartitionPruning.sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext()); diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java -index b0ad930487..9f7c024dbf 100644 +index b0ad930487..6764bf645a 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java -@@ -112,6 +112,14 @@ public class TestPartitionValues { +@@ -112,6 +112,15 @@ public class TestPartitionValues { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -1164,16 +1210,17 @@ index b0ad930487..9f7c024dbf 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java -index 11865db7fc..1d7030e73e 100644 +index 11865db7fc..c7c1e6a02f 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java -@@ -91,6 +91,14 @@ public class TestSnapshotSelection { +@@ -91,6 +91,15 @@ public class TestSnapshotSelection { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -1183,16 +1230,17 @@ index 11865db7fc..1d7030e73e 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java -index 833a0b6b5e..b888968253 100644 +index 833a0b6b5e..eb29e23aac 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java -@@ -125,6 +125,14 @@ public class TestSparkDataFile { +@@ -125,6 +125,15 @@ public class TestSparkDataFile { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -1202,16 +1250,17 @@ index 833a0b6b5e..b888968253 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); TestSparkDataFile.sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext()); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java -index 8884654fe0..bcb7c96073 100644 +index 8884654fe0..fde607663e 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java -@@ -100,6 +100,14 @@ public class TestSparkDataWrite { +@@ -100,6 +100,15 @@ public class TestSparkDataWrite { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -1221,12 +1270,13 @@ index 8884654fe0..bcb7c96073 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } -@@ -144,7 +152,7 @@ public class TestSparkDataWrite { +@@ -144,7 +153,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1235,7 +1285,7 @@ index 8884654fe0..bcb7c96073 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); for (ManifestFile manifest : SnapshotUtil.latestSnapshot(table, branch).allManifests(table.io())) { -@@ -213,7 +221,7 @@ public class TestSparkDataWrite { +@@ -213,7 +222,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1244,7 +1294,7 @@ index 8884654fe0..bcb7c96073 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -258,7 +266,7 @@ public class TestSparkDataWrite { +@@ -258,7 +267,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1253,7 +1303,7 @@ index 8884654fe0..bcb7c96073 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -310,7 +318,7 @@ public class TestSparkDataWrite { +@@ -310,7 +319,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1262,7 +1312,7 @@ index 8884654fe0..bcb7c96073 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -352,7 +360,7 @@ public class TestSparkDataWrite { +@@ -352,7 +361,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1271,7 +1321,7 @@ index 8884654fe0..bcb7c96073 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -391,7 +399,7 @@ public class TestSparkDataWrite { +@@ -391,7 +400,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1280,7 +1330,7 @@ index 8884654fe0..bcb7c96073 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); List files = Lists.newArrayList(); -@@ -459,7 +467,7 @@ public class TestSparkDataWrite { +@@ -459,7 +468,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1289,7 +1339,7 @@ index 8884654fe0..bcb7c96073 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -707,7 +715,7 @@ public class TestSparkDataWrite { +@@ -707,7 +716,7 @@ public class TestSparkDataWrite { // Since write and commit succeeded, the rows should be readable Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1299,10 +1349,10 @@ index 8884654fe0..bcb7c96073 100644 .hasSize(records.size() + records2.size()) .containsExactlyInAnyOrder( diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java -index 596d05d30b..ac83e92fdf 100644 +index 596d05d30b..acbacf65e0 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java -@@ -88,6 +88,14 @@ public class TestSparkReadProjection extends TestReadProjection { +@@ -88,6 +88,15 @@ public class TestSparkReadProjection extends TestReadProjection { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -1312,16 +1362,17 @@ index 596d05d30b..ac83e92fdf 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); ImmutableMap config = ImmutableMap.of( diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java -index 16fa726032..d708206b96 100644 +index 16fa726032..ce27c5cd4d 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java -@@ -132,7 +132,23 @@ public class TestSparkReaderDeletes extends DeleteReadTests { +@@ -132,7 +132,25 @@ public class TestSparkReaderDeletes extends DeleteReadTests { .config("spark.ui.liveUpdate.period", 0) .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) @@ -1331,6 +1382,7 @@ index 16fa726032..d708206b96 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -1340,12 +1392,13 @@ index 16fa726032..d708206b96 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); catalog = -@@ -200,7 +216,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { +@@ -200,7 +218,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { } protected boolean countDeletes() { @@ -1356,10 +1409,10 @@ index 16fa726032..d708206b96 100644 @Override diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java -index baf7fa8f88..681e94afb6 100644 +index baf7fa8f88..fb677d7114 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java -@@ -182,7 +182,23 @@ public class TestSparkReaderWithBloomFilter { +@@ -182,7 +182,25 @@ public class TestSparkReaderWithBloomFilter { SparkSession.builder() .master("local[2]") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) @@ -1369,6 +1422,7 @@ index baf7fa8f88..681e94afb6 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -1378,16 +1432,17 @@ index baf7fa8f88..681e94afb6 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); catalog = diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java -index dc4fc7e187..d71f67fcda 100644 +index dc4fc7e187..2ab6dd12c2 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java -@@ -69,6 +69,14 @@ public class TestStructuredStreaming { +@@ -69,6 +69,15 @@ public class TestStructuredStreaming { .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) .config("spark.sql.shuffle.partitions", 4) @@ -1397,16 +1452,17 @@ index dc4fc7e187..d71f67fcda 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java -index 8b1e3fbfc7..917088073c 100644 +index 8b1e3fbfc7..c519a2be0f 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java -@@ -75,7 +75,18 @@ public class TestTimestampWithoutZone extends TestBase { +@@ -75,7 +75,19 @@ public class TestTimestampWithoutZone extends TestBase { @BeforeAll public static void startSpark() { @@ -1420,6 +1476,7 @@ index 8b1e3fbfc7..917088073c 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1427,10 +1484,10 @@ index 8b1e3fbfc7..917088073c 100644 @AfterAll diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java -index c3fac70dd3..6d630c0c68 100644 +index c3fac70dd3..d3ffd034d2 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java -@@ -84,6 +84,14 @@ public class TestWriteMetricsConfig { +@@ -84,6 +84,15 @@ public class TestWriteMetricsConfig { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -1440,16 +1497,17 @@ index c3fac70dd3..6d630c0c68 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); TestWriteMetricsConfig.sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java -index 5ce56b4fec..cf2a7e1d3b 100644 +index 5ce56b4fec..3b614e3fc9 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java -@@ -63,7 +63,23 @@ public class TestAggregatePushDown extends CatalogTestBase { +@@ -63,7 +63,25 @@ public class TestAggregatePushDown extends CatalogTestBase { SparkSession.builder() .master("local[2]") .config("spark.sql.iceberg.aggregate_pushdown", "true") @@ -1459,6 +1517,7 @@ index 5ce56b4fec..cf2a7e1d3b 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -1468,6 +1527,7 @@ index 5ce56b4fec..cf2a7e1d3b 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); @@ -1501,10 +1561,10 @@ index 69700d8436..49ea338a45 100644 // runtime dependencies for running Hive Catalog based integration test integrationRuntimeOnly project(':iceberg-hive-metastore') diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java -index 4c1a509591..b22f3f9ad9 100644 +index 4c1a509591..3b4e44cc13 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java -@@ -59,6 +59,14 @@ public abstract class ExtensionsTestBase extends CatalogTestBase { +@@ -59,6 +59,15 @@ public abstract class ExtensionsTestBase extends CatalogTestBase { .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") .config( SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), String.valueOf(RANDOM.nextBoolean())) @@ -1514,6 +1574,7 @@ index 4c1a509591..b22f3f9ad9 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -1534,10 +1595,10 @@ index 893f9931cf..17e5fef217 100644 protected void createTableWithDeleteGranularity( diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java -index ecf9e6f8a5..23dbcf7af2 100644 +index ecf9e6f8a5..beca5f8521 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java -@@ -56,6 +56,14 @@ public class TestCallStatementParser { +@@ -56,6 +56,15 @@ public class TestCallStatementParser { .master("local[2]") .config("spark.sql.extensions", IcebergSparkSessionExtensions.class.getName()) .config("spark.extra.prop", "value") @@ -1547,16 +1608,45 @@ index ecf9e6f8a5..23dbcf7af2 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); TestCallStatementParser.parser = spark.sessionState().sqlParser(); } +diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java +index 8cb92224fc..777b9bb1dd 100644 +--- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java ++++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java +@@ -38,6 +38,7 @@ import org.apache.spark.sql.catalyst.expressions.ApplyFunctionExpression; + import org.apache.spark.sql.catalyst.expressions.Expression; + import org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke; + import org.apache.spark.sql.execution.CommandResultExec; ++import org.apache.spark.sql.execution.SparkPlan; + import org.apache.spark.sql.execution.datasources.v2.V2TableWriteExec; + import org.junit.jupiter.api.AfterEach; + import org.junit.jupiter.api.BeforeEach; +@@ -313,9 +314,13 @@ public class TestSystemFunctionPushDownInRowLevelOperations extends ExtensionsTe + + private List executeAndCollectFunctionCalls(String query, Object... args) { + CommandResultExec command = (CommandResultExec) executeAndKeepPlan(query, args); +- V2TableWriteExec write = (V2TableWriteExec) command.commandPhysicalPlan(); ++ // Comet's split-operator Iceberg write plans the command as IcebergCommitExec, which is not ++ // a V2TableWriteExec; collect over the whole command plan in that case. ++ SparkPlan plan = command.commandPhysicalPlan(); ++ SparkPlan queryPlan = ++ plan instanceof V2TableWriteExec ? ((V2TableWriteExec) plan).query() : plan; + return SparkPlanUtil.collectExprs( +- write.query(), ++ queryPlan, + expr -> expr instanceof StaticInvoke || expr instanceof ApplyFunctionExpression); + } + diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java -index 64edb1002e..4706106c2e 100644 +index 64edb1002e..57767da086 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java -@@ -179,6 +179,14 @@ public class DeleteOrphanFilesBenchmark { +@@ -179,6 +179,15 @@ public class DeleteOrphanFilesBenchmark { .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", catalogWarehouse()) @@ -1566,16 +1656,17 @@ index 64edb1002e..4706106c2e 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local"); spark = builder.getOrCreate(); } diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java -index 77b79384a6..1c64ac124e 100644 +index 77b79384a6..8708ffa72a 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java -@@ -392,6 +392,14 @@ public class IcebergSortCompactionBenchmark { +@@ -392,6 +392,15 @@ public class IcebergSortCompactionBenchmark { "spark.sql.catalog.spark_catalog", "org.apache.iceberg.spark.SparkSessionCatalog") .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", getCatalogWarehouse()) @@ -1585,16 +1676,17 @@ index 77b79384a6..1c64ac124e 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[*]"); spark = builder.getOrCreate(); Configuration sparkHadoopConf = spark.sessionState().newHadoopConf(); diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java -index c6794e43c6..ada3a83e3c 100644 +index c6794e43c6..4c1a165a44 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java -@@ -239,6 +239,14 @@ public class DVReaderBenchmark { +@@ -239,6 +239,15 @@ public class DVReaderBenchmark { .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", newWarehouseDir()) @@ -1604,16 +1696,17 @@ index c6794e43c6..ada3a83e3c 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[*]") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java -index ac74fb5a10..bafca20a1c 100644 +index ac74fb5a10..4aa5d5b361 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java -@@ -223,6 +223,14 @@ public class DVWriterBenchmark { +@@ -223,6 +223,15 @@ public class DVWriterBenchmark { .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", newWarehouseDir()) @@ -1623,16 +1716,17 @@ index ac74fb5a10..bafca20a1c 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[*]") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java -index 68c537e34a..5ef031963a 100644 +index 68c537e34a..2e421c7706 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java -@@ -94,7 +94,17 @@ public abstract class IcebergSourceBenchmark { +@@ -94,7 +94,18 @@ public abstract class IcebergSourceBenchmark { } protected void setupSpark(boolean enableDictionaryEncoding) { @@ -1646,6 +1740,7 @@ index 68c537e34a..5ef031963a 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g"); if (!enableDictionaryEncoding) { @@ -2433,10 +2528,10 @@ index a0f45e7610..78e7845911 100644 .filter(residual) .caseSensitive(caseSensitive()) diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java -index 404ba72846..5bd4e2b351 100644 +index 404ba72846..b375458f05 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java -@@ -90,6 +90,14 @@ public abstract class SparkDistributedDataScanTestBase +@@ -90,6 +90,15 @@ public abstract class SparkDistributedDataScanTestBase .master("local[2]") .config("spark.serializer", serializer) .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -2446,16 +2541,17 @@ index 404ba72846..5bd4e2b351 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java -index 659507e4c5..74182efbb2 100644 +index 659507e4c5..2a3a87f079 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java -@@ -73,6 +73,14 @@ public class TestSparkDistributedDataScanDeletes +@@ -73,6 +73,15 @@ public class TestSparkDistributedDataScanDeletes .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -2465,16 +2561,17 @@ index 659507e4c5..74182efbb2 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java -index a218f965ea..b1e0fcf2ca 100644 +index a218f965ea..e30a8d6c6f 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java -@@ -62,6 +62,14 @@ public class TestSparkDistributedDataScanFilterFiles +@@ -62,6 +62,15 @@ public class TestSparkDistributedDataScanFilterFiles .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -2484,16 +2581,17 @@ index a218f965ea..b1e0fcf2ca 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java -index 2665d7ba8d..3539020102 100644 +index 2665d7ba8d..c24c82341a 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java -@@ -63,6 +63,14 @@ public class TestSparkDistributedDataScanReporting +@@ -63,6 +63,15 @@ public class TestSparkDistributedDataScanReporting .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -2503,16 +2601,17 @@ index 2665d7ba8d..3539020102 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java -index daf4e29ac0..0c7d376f80 100644 +index daf4e29ac0..235cf60557 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java -@@ -79,6 +79,14 @@ public abstract class TestBase extends SparkTestHelperBase { +@@ -79,6 +79,15 @@ public abstract class TestBase extends SparkTestHelperBase { .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") @@ -2522,6 +2621,7 @@ index daf4e29ac0..0c7d376f80 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -2553,10 +2653,10 @@ index 0db6a65fd3..9ee8b9e084 100644 } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/vectorized/parquet/TestParquetDictionaryEncodedVectorizedReads.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/vectorized/parquet/TestParquetDictionaryEncodedVectorizedReads.java -index 973a17c9a3..d7402c1409 100644 +index 973a17c9a3..4e4d3c02f1 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/vectorized/parquet/TestParquetDictionaryEncodedVectorizedReads.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/vectorized/parquet/TestParquetDictionaryEncodedVectorizedReads.java -@@ -65,6 +65,14 @@ public class TestParquetDictionaryEncodedVectorizedReads extends TestParquetVect +@@ -65,6 +65,15 @@ public class TestParquetDictionaryEncodedVectorizedReads extends TestParquetVect SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -2566,16 +2666,17 @@ index 973a17c9a3..d7402c1409 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java -index 1c5905744a..17316c7440 100644 +index 1c5905744a..b9cf548290 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java -@@ -61,6 +61,14 @@ public abstract class ScanTestBase extends AvroDataTestBase { +@@ -61,6 +61,15 @@ public abstract class ScanTestBase extends AvroDataTestBase { ScanTestBase.spark = SparkSession.builder() .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -2585,16 +2686,17 @@ index 1c5905744a..17316c7440 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[2]") .getOrCreate(); ScanTestBase.sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java -index 19ec6d13dd..2451667033 100644 +index 19ec6d13dd..c5f73bf666 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java -@@ -144,7 +144,18 @@ public class TestCompressionSettings extends CatalogTestBase { +@@ -144,7 +144,19 @@ public class TestCompressionSettings extends CatalogTestBase { @BeforeAll public static void startSpark() { @@ -2608,6 +2710,7 @@ index 19ec6d13dd..2451667033 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2615,10 +2718,10 @@ index 19ec6d13dd..2451667033 100644 @BeforeEach diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java -index a7702b169a..3f5648dede 100644 +index a7702b169a..67fabf3cdc 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java -@@ -74,7 +74,18 @@ public class TestDataSourceOptions extends TestBaseWithCatalog { +@@ -74,7 +74,19 @@ public class TestDataSourceOptions extends TestBaseWithCatalog { @BeforeAll public static void startSpark() { @@ -2632,6 +2735,7 @@ index a7702b169a..3f5648dede 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2639,10 +2743,10 @@ index a7702b169a..3f5648dede 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java -index fd7d52178f..9bfc596a15 100644 +index fd7d52178f..36831cfb07 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java -@@ -114,6 +114,14 @@ public class TestFilteredScan { +@@ -114,6 +114,15 @@ public class TestFilteredScan { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -2652,16 +2756,17 @@ index fd7d52178f..9bfc596a15 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java -index 153564f7d1..65f44e8986 100644 +index 153564f7d1..973eac1878 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java -@@ -98,6 +98,14 @@ public class TestForwardCompatibility { +@@ -98,6 +98,15 @@ public class TestForwardCompatibility { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -2671,16 +2776,17 @@ index 153564f7d1..65f44e8986 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java -index f4f57157e4..d8f3fa1cd0 100644 +index f4f57157e4..418364e3da 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java -@@ -51,6 +51,14 @@ public class TestIcebergSpark { +@@ -51,6 +51,15 @@ public class TestIcebergSpark { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -2690,16 +2796,17 @@ index f4f57157e4..d8f3fa1cd0 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java -index e1402396fa..5f02d6a2b7 100644 +index e1402396fa..3092902873 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java -@@ -118,6 +118,14 @@ public class TestPartitionPruning { +@@ -118,6 +118,15 @@ public class TestPartitionPruning { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -2709,16 +2816,17 @@ index e1402396fa..5f02d6a2b7 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); TestPartitionPruning.sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext()); diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java -index 0b6ab2052b..a4786a3014 100644 +index 0b6ab2052b..8682b4a7da 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java -@@ -112,6 +112,14 @@ public class TestPartitionValues { +@@ -112,6 +112,15 @@ public class TestPartitionValues { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -2728,16 +2836,17 @@ index 0b6ab2052b..a4786a3014 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java -index 11865db7fc..1d7030e73e 100644 +index 11865db7fc..c7c1e6a02f 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java -@@ -91,6 +91,14 @@ public class TestSnapshotSelection { +@@ -91,6 +91,15 @@ public class TestSnapshotSelection { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -2747,16 +2856,17 @@ index 11865db7fc..1d7030e73e 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java -index 3051e27d72..b41cb2560c 100644 +index 3051e27d72..cb3011b222 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java -@@ -125,6 +125,14 @@ public class TestSparkDataFile { +@@ -125,6 +125,15 @@ public class TestSparkDataFile { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -2766,16 +2876,17 @@ index 3051e27d72..b41cb2560c 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); TestSparkDataFile.sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext()); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java -index 4ccbf86f12..22148d2938 100644 +index 4ccbf86f12..f915f22b21 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java -@@ -100,6 +100,14 @@ public class TestSparkDataWrite { +@@ -100,6 +100,15 @@ public class TestSparkDataWrite { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -2785,12 +2896,13 @@ index 4ccbf86f12..22148d2938 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } -@@ -144,7 +152,7 @@ public class TestSparkDataWrite { +@@ -144,7 +153,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -2799,7 +2911,7 @@ index 4ccbf86f12..22148d2938 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); for (ManifestFile manifest : SnapshotUtil.latestSnapshot(table, branch).allManifests(table.io())) { -@@ -213,7 +221,7 @@ public class TestSparkDataWrite { +@@ -213,7 +222,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -2808,7 +2920,7 @@ index 4ccbf86f12..22148d2938 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -258,7 +266,7 @@ public class TestSparkDataWrite { +@@ -258,7 +267,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -2817,7 +2929,7 @@ index 4ccbf86f12..22148d2938 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -310,7 +318,7 @@ public class TestSparkDataWrite { +@@ -310,7 +319,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -2826,7 +2938,7 @@ index 4ccbf86f12..22148d2938 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -352,7 +360,7 @@ public class TestSparkDataWrite { +@@ -352,7 +361,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -2835,7 +2947,7 @@ index 4ccbf86f12..22148d2938 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -391,7 +399,7 @@ public class TestSparkDataWrite { +@@ -391,7 +400,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -2844,7 +2956,7 @@ index 4ccbf86f12..22148d2938 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); List files = Lists.newArrayList(); -@@ -459,7 +467,7 @@ public class TestSparkDataWrite { +@@ -459,7 +468,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -2853,7 +2965,7 @@ index 4ccbf86f12..22148d2938 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -706,7 +714,7 @@ public class TestSparkDataWrite { +@@ -706,7 +715,7 @@ public class TestSparkDataWrite { // Since write and commit succeeded, the rows should be readable Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -2863,10 +2975,10 @@ index 4ccbf86f12..22148d2938 100644 .hasSize(records.size() + records2.size()) .containsExactlyInAnyOrder( diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java -index 596d05d30b..ac83e92fdf 100644 +index 596d05d30b..acbacf65e0 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java -@@ -88,6 +88,14 @@ public class TestSparkReadProjection extends TestReadProjection { +@@ -88,6 +88,15 @@ public class TestSparkReadProjection extends TestReadProjection { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -2876,16 +2988,17 @@ index 596d05d30b..ac83e92fdf 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); ImmutableMap config = ImmutableMap.of( diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java -index 42699f4662..185b2f8d7a 100644 +index 42699f4662..b8bf79886e 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java -@@ -138,6 +138,14 @@ public class TestSparkReaderDeletes extends DeleteReadTests { +@@ -138,6 +138,15 @@ public class TestSparkReaderDeletes extends DeleteReadTests { .config("spark.ui.liveUpdate.period", 0) .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) @@ -2895,12 +3008,13 @@ index 42699f4662..185b2f8d7a 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() .getOrCreate(); -@@ -206,7 +214,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { +@@ -206,7 +215,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { } protected boolean countDeletes() { @@ -2911,10 +3025,10 @@ index 42699f4662..185b2f8d7a 100644 @Override diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java -index baf7fa8f88..5ada29b4af 100644 +index baf7fa8f88..b58321b761 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java -@@ -182,6 +182,14 @@ public class TestSparkReaderWithBloomFilter { +@@ -182,6 +182,15 @@ public class TestSparkReaderWithBloomFilter { SparkSession.builder() .master("local[2]") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) @@ -2924,16 +3038,17 @@ index baf7fa8f88..5ada29b4af 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() .getOrCreate(); diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java -index 54048bbf21..06e3eb0e56 100644 +index 54048bbf21..2ebe9ab993 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java -@@ -69,6 +69,14 @@ public class TestStructuredStreaming { +@@ -69,6 +69,15 @@ public class TestStructuredStreaming { .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) .config("spark.sql.shuffle.partitions", 4) @@ -2943,16 +3058,17 @@ index 54048bbf21..06e3eb0e56 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java -index 8b1e3fbfc7..917088073c 100644 +index 8b1e3fbfc7..c519a2be0f 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java -@@ -75,7 +75,18 @@ public class TestTimestampWithoutZone extends TestBase { +@@ -75,7 +75,19 @@ public class TestTimestampWithoutZone extends TestBase { @BeforeAll public static void startSpark() { @@ -2966,6 +3082,7 @@ index 8b1e3fbfc7..917088073c 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2973,10 +3090,10 @@ index 8b1e3fbfc7..917088073c 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java -index c3fac70dd3..6d630c0c68 100644 +index c3fac70dd3..d3ffd034d2 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java -@@ -84,6 +84,14 @@ public class TestWriteMetricsConfig { +@@ -84,6 +84,15 @@ public class TestWriteMetricsConfig { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -2986,16 +3103,17 @@ index c3fac70dd3..6d630c0c68 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); TestWriteMetricsConfig.sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java -index 5ce56b4fec..00735f62d1 100644 +index 5ce56b4fec..d089cc1f69 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java -@@ -63,6 +63,14 @@ public class TestAggregatePushDown extends CatalogTestBase { +@@ -63,6 +63,15 @@ public class TestAggregatePushDown extends CatalogTestBase { SparkSession.builder() .master("local[2]") .config("spark.sql.iceberg.aggregate_pushdown", "true") @@ -3005,6 +3123,7 @@ index 5ce56b4fec..00735f62d1 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -3025,59 +3144,3 @@ index 9d2ce2b388..5e23368848 100644 } else { assertThat(planAsString).as("Should be no post scan filter").doesNotContain("Filter ("); } -diff --git a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java -index 5a0a04edb..afe92a8b7 100644 ---- a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java -+++ b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java -@@ -38,6 +38,7 @@ import org.apache.spark.sql.catalyst.expressions.ApplyFunctionExpression; - import org.apache.spark.sql.catalyst.expressions.Expression; - import org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke; - import org.apache.spark.sql.execution.CommandResultExec; -+import org.apache.spark.sql.execution.SparkPlan; - import org.apache.spark.sql.execution.datasources.v2.V2TableWriteExec; - import org.junit.jupiter.api.AfterEach; - import org.junit.jupiter.api.BeforeEach; -@@ -313,9 +314,13 @@ public class TestSystemFunctionPushDownInRowLevelOperations extends ExtensionsTe - - private List executeAndCollectFunctionCalls(String query, Object... args) { - CommandResultExec command = (CommandResultExec) executeAndKeepPlan(query, args); -- V2TableWriteExec write = (V2TableWriteExec) command.commandPhysicalPlan(); -+ // Comet's split-operator Iceberg write plans the command as IcebergCommitExec, which is not -+ // a V2TableWriteExec; collect over the whole command plan in that case. -+ SparkPlan plan = command.commandPhysicalPlan(); -+ SparkPlan queryPlan = -+ plan instanceof V2TableWriteExec ? ((V2TableWriteExec) plan).query() : plan; - return SparkPlanUtil.collectExprs( -- write.query(), -+ queryPlan, - expr -> expr instanceof StaticInvoke || expr instanceof ApplyFunctionExpression); - } - -diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java -index 8cb92224f..777b9bb1d 100644 ---- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java -+++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java -@@ -38,6 +38,7 @@ import org.apache.spark.sql.catalyst.expressions.ApplyFunctionExpression; - import org.apache.spark.sql.catalyst.expressions.Expression; - import org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke; - import org.apache.spark.sql.execution.CommandResultExec; -+import org.apache.spark.sql.execution.SparkPlan; - import org.apache.spark.sql.execution.datasources.v2.V2TableWriteExec; - import org.junit.jupiter.api.AfterEach; - import org.junit.jupiter.api.BeforeEach; -@@ -313,9 +314,13 @@ public class TestSystemFunctionPushDownInRowLevelOperations extends ExtensionsTe - - private List executeAndCollectFunctionCalls(String query, Object... args) { - CommandResultExec command = (CommandResultExec) executeAndKeepPlan(query, args); -- V2TableWriteExec write = (V2TableWriteExec) command.commandPhysicalPlan(); -+ // Comet's split-operator Iceberg write plans the command as IcebergCommitExec, which is not -+ // a V2TableWriteExec; collect over the whole command plan in that case. -+ SparkPlan plan = command.commandPhysicalPlan(); -+ SparkPlan queryPlan = -+ plan instanceof V2TableWriteExec ? ((V2TableWriteExec) plan).query() : plan; - return SparkPlanUtil.collectExprs( -- write.query(), -+ queryPlan, - expr -> expr instanceof StaticInvoke || expr instanceof ApplyFunctionExpression); - } - diff --git a/dev/diffs/iceberg/1.11.0.diff b/dev/diffs/iceberg/1.11.0.diff index 050e2107d35..555002b51fa 100644 --- a/dev/diffs/iceberg/1.11.0.diff +++ b/dev/diffs/iceberg/1.11.0.diff @@ -1,5 +1,5 @@ diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml -index db659dc06b..78f5e3440f 100644 +index db659dc06b..f45ab41ce2 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -40,6 +40,7 @@ bouncycastle = "1.84" @@ -39,10 +39,10 @@ index e6455fa34f..08b685f61a 100644 // runtime dependencies for running Hive Catalog based integration test integrationRuntimeOnly project(':iceberg-hive-metastore') diff --git a/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java b/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java -index f766fbb79a..c5e31185a9 100644 +index f766fbb79a..59e4e6c285 100644 --- a/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java +++ b/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java -@@ -71,6 +71,14 @@ public abstract class ExtensionsTestBase extends CatalogTestBase { +@@ -71,6 +71,15 @@ public abstract class ExtensionsTestBase extends CatalogTestBase { .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") .config( SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), String.valueOf(RANDOM.nextBoolean())) @@ -52,6 +52,7 @@ index f766fbb79a..c5e31185a9 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) @@ -71,11 +72,39 @@ index b5d6415763..0763a723d3 100644 } protected void createTableWithDeleteGranularity( +diff --git a/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java b/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java +index 934220e5d3..92132d4b8d 100644 +--- a/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java ++++ b/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java +@@ -40,6 +40,7 @@ import org.apache.spark.sql.catalyst.expressions.ApplyFunctionExpression; + import org.apache.spark.sql.catalyst.expressions.Expression; + import org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke; + import org.apache.spark.sql.execution.CommandResultExec; ++import org.apache.spark.sql.execution.SparkPlan; + import org.apache.spark.sql.execution.datasources.v2.V2TableWriteExec; + import org.junit.jupiter.api.AfterEach; + import org.junit.jupiter.api.BeforeEach; +@@ -330,9 +331,13 @@ public class TestSystemFunctionPushDownInRowLevelOperations extends ExtensionsTe + + private List executeAndCollectFunctionCalls(String query, Object... args) { + CommandResultExec command = (CommandResultExec) executeAndKeepPlan(query, args); +- V2TableWriteExec write = (V2TableWriteExec) command.commandPhysicalPlan(); ++ // Comet's split-operator Iceberg write plans the command as IcebergCommitExec, which is not ++ // a V2TableWriteExec; collect over the whole command plan in that case. ++ SparkPlan plan = command.commandPhysicalPlan(); ++ SparkPlan queryPlan = ++ plan instanceof V2TableWriteExec ? ((V2TableWriteExec) plan).query() : plan; + return SparkPlanUtil.collectExprs( +- write.query(), ++ queryPlan, + expr -> expr instanceof StaticInvoke || expr instanceof ApplyFunctionExpression); + } + diff --git a/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java b/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java -index 3fd84553f0..350b4a7561 100644 +index 3fd84553f0..db71a30730 100644 --- a/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java +++ b/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java -@@ -180,6 +180,14 @@ public class DeleteOrphanFilesBenchmark { +@@ -180,6 +180,15 @@ public class DeleteOrphanFilesBenchmark { .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", catalogWarehouse()) @@ -85,16 +114,17 @@ index 3fd84553f0..350b4a7561 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .master("local"); spark = builder.getOrCreate(); diff --git a/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java b/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java -index 683f6bb46d..402561a241 100644 +index 683f6bb46d..bd5ef5d3c0 100644 --- a/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java +++ b/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java -@@ -395,6 +395,14 @@ public class IcebergSortCompactionBenchmark { +@@ -395,6 +395,15 @@ public class IcebergSortCompactionBenchmark { "spark.sql.catalog.spark_catalog", "org.apache.iceberg.spark.SparkSessionCatalog") .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", getCatalogWarehouse()) @@ -104,16 +134,17 @@ index 683f6bb46d..402561a241 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .master("local[*]"); spark = builder.getOrCreate(); diff --git a/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java b/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java -index 3f242ce228..004db59f1f 100644 +index 3f242ce228..ee19cc1248 100644 --- a/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java +++ b/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java -@@ -240,6 +240,14 @@ public class DVReaderBenchmark { +@@ -240,6 +240,15 @@ public class DVReaderBenchmark { .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", newWarehouseDir()) @@ -123,16 +154,17 @@ index 3f242ce228..004db59f1f 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[*]") .getOrCreate(); } diff --git a/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java b/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java -index db57897240..b3fde306c8 100644 +index db57897240..f3b47a62f6 100644 --- a/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java +++ b/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java -@@ -224,6 +224,14 @@ public class DVWriterBenchmark { +@@ -224,6 +224,15 @@ public class DVWriterBenchmark { .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", newWarehouseDir()) @@ -142,16 +174,17 @@ index db57897240..b3fde306c8 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[*]") .getOrCreate(); } diff --git a/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java b/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java -index debe37866f..62daec8291 100644 +index debe37866f..f48e355a4c 100644 --- a/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java +++ b/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java -@@ -95,7 +95,17 @@ public abstract class IcebergSourceBenchmark { +@@ -95,7 +95,18 @@ public abstract class IcebergSourceBenchmark { } protected void setupSpark(boolean enableDictionaryEncoding) { @@ -165,16 +198,17 @@ index debe37866f..62daec8291 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g"); if (!enableDictionaryEncoding) { builder .config("parquet.dictionary.page.size", "1") diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java -index d1c724425c..325dcb95d1 100644 +index d1c724425c..da7450dd95 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java -@@ -90,6 +90,14 @@ public abstract class SparkDistributedDataScanTestBase +@@ -90,6 +90,15 @@ public abstract class SparkDistributedDataScanTestBase .master("local[2]") .config("spark.serializer", serializer) .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -184,16 +218,17 @@ index d1c724425c..325dcb95d1 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java -index a21c6a08ec..1292883f55 100644 +index a21c6a08ec..104bfcd580 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java -@@ -73,6 +73,14 @@ public class TestSparkDistributedDataScanDeletes +@@ -73,6 +73,15 @@ public class TestSparkDistributedDataScanDeletes .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -203,16 +238,17 @@ index a21c6a08ec..1292883f55 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java -index 5edf482822..92166f7b48 100644 +index 5edf482822..24bd0ee141 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java -@@ -62,6 +62,14 @@ public class TestSparkDistributedDataScanFilterFiles +@@ -62,6 +62,15 @@ public class TestSparkDistributedDataScanFilterFiles .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -222,16 +258,17 @@ index 5edf482822..92166f7b48 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java -index e6f3c75475..0a9a958b2c 100644 +index e6f3c75475..6e2a7c6fb7 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java -@@ -63,6 +63,14 @@ public class TestSparkDistributedDataScanReporting +@@ -63,6 +63,15 @@ public class TestSparkDistributedDataScanReporting .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -241,16 +278,17 @@ index e6f3c75475..0a9a958b2c 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestBase.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestBase.java -index 507d7b313b..3b73dcc014 100644 +index 507d7b313b..8fbdb40228 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestBase.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestBase.java -@@ -86,6 +86,14 @@ public abstract class TestBase extends SparkTestHelperBase { +@@ -86,6 +86,15 @@ public abstract class TestBase extends SparkTestHelperBase { .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") @@ -260,6 +298,7 @@ index 507d7b313b..3b73dcc014 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config("spark.ui.enabled", "false") @@ -291,10 +330,10 @@ index 8ce60f6275..75112b4354 100644 } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/vectorized/parquet/TestParquetDictionaryEncodedVectorizedReads.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/vectorized/parquet/TestParquetDictionaryEncodedVectorizedReads.java -index b61ecfa2f4..d696e85139 100644 +index b61ecfa2f4..a3aafae4ee 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/vectorized/parquet/TestParquetDictionaryEncodedVectorizedReads.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/vectorized/parquet/TestParquetDictionaryEncodedVectorizedReads.java -@@ -66,6 +66,14 @@ public class TestParquetDictionaryEncodedVectorizedReads extends TestParquetVect +@@ -66,6 +66,15 @@ public class TestParquetDictionaryEncodedVectorizedReads extends TestParquetVect SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -304,16 +343,17 @@ index b61ecfa2f4..d696e85139 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java -index da9cd63921..fae1357a40 100644 +index da9cd63921..c713c27456 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java -@@ -62,6 +62,14 @@ public abstract class ScanTestBase extends AvroDataTestBase { +@@ -62,6 +62,15 @@ public abstract class ScanTestBase extends AvroDataTestBase { ScanTestBase.spark = SparkSession.builder() .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -323,16 +363,17 @@ index da9cd63921..fae1357a40 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[2]") .config(TestBase.DISABLE_UI) .getOrCreate(); diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java -index d381822483..043d6596c0 100644 +index d381822483..f090b5bf14 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java -@@ -144,7 +144,18 @@ public class TestCompressionSettings extends CatalogTestBase { +@@ -144,7 +144,19 @@ public class TestCompressionSettings extends CatalogTestBase { @BeforeAll public static void startSpark() { @@ -346,6 +387,7 @@ index d381822483..043d6596c0 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -353,10 +395,10 @@ index d381822483..043d6596c0 100644 @BeforeEach diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java -index e67ec5fd62..ed47a307c3 100644 +index e67ec5fd62..ef19a1d349 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java -@@ -79,7 +79,18 @@ public class TestDataSourceOptions extends TestBaseWithCatalog { +@@ -79,7 +79,19 @@ public class TestDataSourceOptions extends TestBaseWithCatalog { @BeforeAll public static void startSpark() { @@ -370,6 +412,7 @@ index e67ec5fd62..ed47a307c3 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -377,10 +420,10 @@ index e67ec5fd62..ed47a307c3 100644 @AfterAll diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java -index 24fecf4eb2..abab7798bc 100644 +index 24fecf4eb2..964a5bc003 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java -@@ -117,6 +117,14 @@ public class TestFilteredScan { +@@ -117,6 +117,15 @@ public class TestFilteredScan { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -390,16 +433,17 @@ index 24fecf4eb2..abab7798bc 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java -index d0103ff46e..62a7bdc04d 100644 +index d0103ff46e..c634782764 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java -@@ -99,6 +99,14 @@ public class TestForwardCompatibility { +@@ -99,6 +99,15 @@ public class TestForwardCompatibility { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -409,16 +453,17 @@ index d0103ff46e..62a7bdc04d 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java -index a637b975fe..26304e8894 100644 +index a637b975fe..dc31aab58b 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java -@@ -52,6 +52,14 @@ public class TestIcebergSpark { +@@ -52,6 +52,15 @@ public class TestIcebergSpark { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -428,16 +473,17 @@ index a637b975fe..26304e8894 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java -index 8098db81f9..0e83b02f45 100644 +index 8098db81f9..a3b023a43e 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java -@@ -120,6 +120,14 @@ public class TestPartitionPruning { +@@ -120,6 +120,15 @@ public class TestPartitionPruning { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -447,16 +493,17 @@ index 8098db81f9..0e83b02f45 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); TestPartitionPruning.sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext()); diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java -index 9b5b22a73f..5d41bbeb5e 100644 +index 9b5b22a73f..4aed7f06dc 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java -@@ -113,6 +113,14 @@ public class TestPartitionValues { +@@ -113,6 +113,15 @@ public class TestPartitionValues { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -466,16 +513,17 @@ index 9b5b22a73f..5d41bbeb5e 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java -index 3004e8fa5c..38b6a1f6a4 100644 +index 3004e8fa5c..82ebbe9e81 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java -@@ -98,6 +98,14 @@ public class TestSnapshotSelection { +@@ -98,6 +98,15 @@ public class TestSnapshotSelection { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -485,16 +533,17 @@ index 3004e8fa5c..38b6a1f6a4 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java -index d719ca6751..eba566120e 100644 +index d719ca6751..00ec4f9cdf 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java -@@ -127,6 +127,14 @@ public class TestSparkDataFile { +@@ -127,6 +127,15 @@ public class TestSparkDataFile { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -504,16 +553,17 @@ index d719ca6751..eba566120e 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); TestSparkDataFile.sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext()); diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java -index c2f5afef0e..0dac36653c 100644 +index c2f5afef0e..d970524cd8 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java -@@ -104,6 +104,14 @@ public class TestSparkDataWrite { +@@ -104,6 +104,15 @@ public class TestSparkDataWrite { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -523,12 +573,13 @@ index c2f5afef0e..0dac36653c 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); } -@@ -149,7 +157,7 @@ public class TestSparkDataWrite { +@@ -149,7 +158,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -537,7 +588,7 @@ index c2f5afef0e..0dac36653c 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); for (ManifestFile manifest : SnapshotUtil.latestSnapshot(table, branch).allManifests(table.io())) { -@@ -219,7 +227,7 @@ public class TestSparkDataWrite { +@@ -219,7 +228,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -546,7 +597,7 @@ index c2f5afef0e..0dac36653c 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -264,7 +272,7 @@ public class TestSparkDataWrite { +@@ -264,7 +273,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -555,7 +606,7 @@ index c2f5afef0e..0dac36653c 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -316,7 +324,7 @@ public class TestSparkDataWrite { +@@ -316,7 +325,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -564,7 +615,7 @@ index c2f5afef0e..0dac36653c 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -358,7 +366,7 @@ public class TestSparkDataWrite { +@@ -358,7 +367,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -573,7 +624,7 @@ index c2f5afef0e..0dac36653c 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -397,7 +405,7 @@ public class TestSparkDataWrite { +@@ -397,7 +406,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -582,7 +633,7 @@ index c2f5afef0e..0dac36653c 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); List files = Lists.newArrayList(); -@@ -461,7 +469,7 @@ public class TestSparkDataWrite { +@@ -461,7 +470,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -591,7 +642,7 @@ index c2f5afef0e..0dac36653c 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -814,7 +822,7 @@ public class TestSparkDataWrite { +@@ -814,7 +823,7 @@ public class TestSparkDataWrite { // Since write and commit succeeded, the rows should be readable Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -601,10 +652,10 @@ index c2f5afef0e..0dac36653c 100644 .hasSize(records.size() + records2.size()) .containsExactlyInAnyOrder( diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java -index de6a5e5902..e220f0dd31 100644 +index de6a5e5902..e98a15fe39 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java -@@ -89,6 +89,14 @@ public class TestSparkReadProjection extends TestReadProjection { +@@ -89,6 +89,15 @@ public class TestSparkReadProjection extends TestReadProjection { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -614,16 +665,17 @@ index de6a5e5902..e220f0dd31 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); ImmutableMap config = diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java -index 0d61930571..108d31b965 100644 +index 0d61930571..0a6208cc21 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java -@@ -138,6 +138,14 @@ public class TestSparkReaderDeletes extends DeleteReadTests { +@@ -138,6 +138,15 @@ public class TestSparkReaderDeletes extends DeleteReadTests { .config("spark.ui.liveUpdate.period", 0) .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) @@ -633,12 +685,13 @@ index 0d61930571..108d31b965 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .enableHiveSupport() .getOrCreate(); -@@ -210,7 +218,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { +@@ -210,7 +219,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { } protected boolean countDeletes() { @@ -649,10 +702,10 @@ index 0d61930571..108d31b965 100644 @Override diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java -index cb2f866fab..a161d541cf 100644 +index cb2f866fab..02c42b7299 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java -@@ -183,6 +183,14 @@ public class TestSparkReaderWithBloomFilter { +@@ -183,6 +183,15 @@ public class TestSparkReaderWithBloomFilter { SparkSession.builder() .master("local[2]") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) @@ -662,16 +715,17 @@ index cb2f866fab..a161d541cf 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .enableHiveSupport() .getOrCreate(); diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java -index 5e900ea0ba..c9299d1ebf 100644 +index 5e900ea0ba..2f7d6a08a1 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java -@@ -73,6 +73,14 @@ public class TestStructuredStreaming { +@@ -73,6 +73,15 @@ public class TestStructuredStreaming { .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) .config("spark.sql.shuffle.partitions", 4) @@ -681,16 +735,17 @@ index 5e900ea0ba..c9299d1ebf 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java -index 79781a8fc3..748b0a8327 100644 +index 79781a8fc3..08bfa8098b 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java -@@ -75,7 +75,18 @@ public class TestTimestampWithoutZone extends TestBase { +@@ -75,7 +75,19 @@ public class TestTimestampWithoutZone extends TestBase { @BeforeAll public static void startSpark() { @@ -704,6 +759,7 @@ index 79781a8fc3..748b0a8327 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -711,10 +767,10 @@ index 79781a8fc3..748b0a8327 100644 @AfterAll diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java -index ab2479d610..9b2864bb33 100644 +index ab2479d610..49383ca4a6 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java -@@ -86,6 +86,14 @@ public class TestWriteMetricsConfig { +@@ -86,6 +86,15 @@ public class TestWriteMetricsConfig { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -724,16 +780,17 @@ index ab2479d610..9b2864bb33 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); TestWriteMetricsConfig.sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java -index 1669301d2d..f48b4f674a 100644 +index 1669301d2d..422c7e4ace 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java -@@ -63,6 +63,14 @@ public class TestAggregatePushDown extends CatalogTestBase { +@@ -63,6 +63,15 @@ public class TestAggregatePushDown extends CatalogTestBase { SparkSession.builder() .master("local[2]") .config("spark.sql.iceberg.aggregate_pushdown", "true") @@ -743,6 +800,7 @@ index 1669301d2d..f48b4f674a 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) @@ -766,10 +824,10 @@ index e5a9d63b68..220d445851 100644 assertThat(planAsString).as("Should be no post scan filter").doesNotContain("Filter ("); } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/variant/TestVariantShredding.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/variant/TestVariantShredding.java -index 8cdcf22e58..820f9fc03b 100644 +index 8cdcf22e58..da3ea3560f 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/variant/TestVariantShredding.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/variant/TestVariantShredding.java -@@ -100,6 +100,14 @@ public class TestVariantShredding extends CatalogTestBase { +@@ -100,6 +100,15 @@ public class TestVariantShredding extends CatalogTestBase { .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") .config(DISABLE_UI) @@ -779,36 +837,9 @@ index 8cdcf22e58..820f9fc03b 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() .getOrCreate(); -diff --git a/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java b/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java -index 934220e5d..92132d4b8 100644 ---- a/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java -+++ b/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java -@@ -40,6 +40,7 @@ import org.apache.spark.sql.catalyst.expressions.ApplyFunctionExpression; - import org.apache.spark.sql.catalyst.expressions.Expression; - import org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke; - import org.apache.spark.sql.execution.CommandResultExec; -+import org.apache.spark.sql.execution.SparkPlan; - import org.apache.spark.sql.execution.datasources.v2.V2TableWriteExec; - import org.junit.jupiter.api.AfterEach; - import org.junit.jupiter.api.BeforeEach; -@@ -330,9 +331,13 @@ public class TestSystemFunctionPushDownInRowLevelOperations extends ExtensionsTe - - private List executeAndCollectFunctionCalls(String query, Object... args) { - CommandResultExec command = (CommandResultExec) executeAndKeepPlan(query, args); -- V2TableWriteExec write = (V2TableWriteExec) command.commandPhysicalPlan(); -+ // Comet's split-operator Iceberg write plans the command as IcebergCommitExec, which is not -+ // a V2TableWriteExec; collect over the whole command plan in that case. -+ SparkPlan plan = command.commandPhysicalPlan(); -+ SparkPlan queryPlan = -+ plan instanceof V2TableWriteExec ? ((V2TableWriteExec) plan).query() : plan; - return SparkPlanUtil.collectExprs( -- write.query(), -+ queryPlan, - expr -> expr instanceof StaticInvoke || expr instanceof ApplyFunctionExpression); - } - diff --git a/dev/diffs/iceberg/1.8.1.diff b/dev/diffs/iceberg/1.8.1.diff index b674f432448..718eeccd7e6 100644 --- a/dev/diffs/iceberg/1.8.1.diff +++ b/dev/diffs/iceberg/1.8.1.diff @@ -1,5 +1,5 @@ diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml -index 04ffa8f4ed..5a925fd594 100644 +index 04ffa8f4ed..5c0a54a84a 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -34,6 +34,7 @@ azuresdk-bom = "1.2.31" @@ -51,10 +51,10 @@ index 6eb26e8b73..38d9cf3ca3 100644 // runtime dependencies for running Hive Catalog based integration test integrationRuntimeOnly project(':iceberg-hive-metastore') diff --git a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkExtensionsTestBase.java b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkExtensionsTestBase.java -index 4f137f5b8d..145ef2d00c 100644 +index 4f137f5b8d..f1122f0843 100644 --- a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkExtensionsTestBase.java +++ b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkExtensionsTestBase.java -@@ -60,7 +60,23 @@ public abstract class SparkExtensionsTestBase extends SparkCatalogTestBase { +@@ -60,7 +60,25 @@ public abstract class SparkExtensionsTestBase extends SparkCatalogTestBase { .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") .config( SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), String.valueOf(RANDOM.nextBoolean())) @@ -64,6 +64,7 @@ index 4f137f5b8d..145ef2d00c 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -73,6 +74,7 @@ index 4f137f5b8d..145ef2d00c 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); @@ -93,10 +95,10 @@ index 5fa76f9da6..9184aaba83 100644 protected void createTableWithDeleteGranularity( diff --git a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java -index 55a413063e..a1895c2faf 100644 +index 55a413063e..61a9385ff3 100644 --- a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java +++ b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java -@@ -61,6 +61,14 @@ public class TestCallStatementParser { +@@ -61,6 +61,15 @@ public class TestCallStatementParser { .master("local[2]") .config("spark.sql.extensions", IcebergSparkSessionExtensions.class.getName()) .config("spark.extra.prop", "value") @@ -106,16 +108,45 @@ index 55a413063e..a1895c2faf 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); TestCallStatementParser.parser = spark.sessionState().sqlParser(); } +diff --git a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java +index 49119d319c..04fd3d4859 100644 +--- a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java ++++ b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java +@@ -37,6 +37,7 @@ import org.apache.spark.sql.catalyst.expressions.ApplyFunctionExpression; + import org.apache.spark.sql.catalyst.expressions.Expression; + import org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke; + import org.apache.spark.sql.execution.CommandResultExec; ++import org.apache.spark.sql.execution.SparkPlan; + import org.apache.spark.sql.execution.datasources.v2.V2TableWriteExec; + import org.junit.After; + import org.junit.Before; +@@ -316,9 +317,13 @@ public class TestSystemFunctionPushDownInRowLevelOperations extends SparkExtensi + + private List executeAndCollectFunctionCalls(String query, Object... args) { + CommandResultExec command = (CommandResultExec) executeAndKeepPlan(query, args); +- V2TableWriteExec write = (V2TableWriteExec) command.commandPhysicalPlan(); ++ // Comet's split-operator Iceberg write plans the command as IcebergCommitExec, which is not ++ // a V2TableWriteExec; collect over the whole command plan in that case. ++ SparkPlan plan = command.commandPhysicalPlan(); ++ SparkPlan queryPlan = ++ plan instanceof V2TableWriteExec ? ((V2TableWriteExec) plan).query() : plan; + return SparkPlanUtil.collectExprs( +- write.query(), ++ queryPlan, + expr -> expr instanceof StaticInvoke || expr instanceof ApplyFunctionExpression); + } + diff --git a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java -index b6ade2bff3..b52d011691 100644 +index b6ade2bff3..90a4839dad 100644 --- a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java +++ b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java -@@ -170,6 +170,14 @@ public class DeleteOrphanFilesBenchmark { +@@ -170,6 +170,15 @@ public class DeleteOrphanFilesBenchmark { .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", catalogWarehouse()) @@ -125,16 +156,17 @@ index b6ade2bff3..b52d011691 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local"); spark = builder.getOrCreate(); } diff --git a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java -index b08c352819..d2ca76e07a 100644 +index b08c352819..63880b089d 100644 --- a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java +++ b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java -@@ -386,6 +386,14 @@ public class IcebergSortCompactionBenchmark { +@@ -386,6 +386,15 @@ public class IcebergSortCompactionBenchmark { "spark.sql.catalog.spark_catalog", "org.apache.iceberg.spark.SparkSessionCatalog") .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", getCatalogWarehouse()) @@ -144,16 +176,17 @@ index b08c352819..d2ca76e07a 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[*]"); spark = builder.getOrCreate(); Configuration sparkHadoopConf = spark.sessionState().newHadoopConf(); diff --git a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java -index 68c537e34a..5ef031963a 100644 +index 68c537e34a..2e421c7706 100644 --- a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java +++ b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java -@@ -94,7 +94,17 @@ public abstract class IcebergSourceBenchmark { +@@ -94,7 +94,18 @@ public abstract class IcebergSourceBenchmark { } protected void setupSpark(boolean enableDictionaryEncoding) { @@ -167,6 +200,7 @@ index 68c537e34a..5ef031963a 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g"); if (!enableDictionaryEncoding) { @@ -970,10 +1004,10 @@ index 780e1750a5..25f253eede 100644 .filter(residual) .caseSensitive(caseSensitive()) diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java -index 404ba72846..5bd4e2b351 100644 +index 404ba72846..b375458f05 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java -@@ -90,6 +90,14 @@ public abstract class SparkDistributedDataScanTestBase +@@ -90,6 +90,15 @@ public abstract class SparkDistributedDataScanTestBase .master("local[2]") .config("spark.serializer", serializer) .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -983,16 +1017,17 @@ index 404ba72846..5bd4e2b351 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java -index 9361c63176..44c59c8971 100644 +index 9361c63176..16ce1c614f 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java -@@ -69,6 +69,14 @@ public class TestSparkDistributedDataScanDeletes +@@ -69,6 +69,15 @@ public class TestSparkDistributedDataScanDeletes .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -1002,16 +1037,17 @@ index 9361c63176..44c59c8971 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java -index a218f965ea..b1e0fcf2ca 100644 +index a218f965ea..e30a8d6c6f 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java -@@ -62,6 +62,14 @@ public class TestSparkDistributedDataScanFilterFiles +@@ -62,6 +62,15 @@ public class TestSparkDistributedDataScanFilterFiles .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -1021,16 +1057,17 @@ index a218f965ea..b1e0fcf2ca 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java -index acd4688440..fffc604b48 100644 +index acd4688440..33732e6654 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java -@@ -59,6 +59,14 @@ public class TestSparkDistributedDataScanReporting +@@ -59,6 +59,15 @@ public class TestSparkDistributedDataScanReporting .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -1040,16 +1077,17 @@ index acd4688440..fffc604b48 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/SparkTestBase.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/SparkTestBase.java -index 3e8953fb95..65a8d8ca35 100644 +index 3e8953fb95..51ce392476 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/SparkTestBase.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/SparkTestBase.java -@@ -77,6 +77,14 @@ public abstract class SparkTestBase extends SparkTestHelperBase { +@@ -77,6 +77,15 @@ public abstract class SparkTestBase extends SparkTestHelperBase { .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") @@ -1059,6 +1097,7 @@ index 3e8953fb95..65a8d8ca35 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -1090,10 +1129,10 @@ index ad969384c5..f3c2b833e3 100644 } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java -index 3a269740b7..2035edf93a 100644 +index 3a269740b7..fc63449215 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java -@@ -54,7 +54,18 @@ public abstract class ScanTestBase extends AvroDataTest { +@@ -54,7 +54,19 @@ public abstract class ScanTestBase extends AvroDataTest { @BeforeAll public static void startSpark() { @@ -1107,6 +1146,7 @@ index 3a269740b7..2035edf93a 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1114,10 +1154,10 @@ index 3a269740b7..2035edf93a 100644 } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java -index 724c6edde2..13c7be5103 100644 +index 724c6edde2..ff01c10e1a 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java -@@ -103,7 +103,18 @@ public class TestCompressionSettings extends SparkCatalogTestBase { +@@ -103,7 +103,19 @@ public class TestCompressionSettings extends SparkCatalogTestBase { @BeforeClass public static void startSpark() { @@ -1131,6 +1171,7 @@ index 724c6edde2..13c7be5103 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1138,10 +1179,10 @@ index 724c6edde2..13c7be5103 100644 @Parameterized.AfterParam diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java -index 013b8d4386..4c62e88fe9 100644 +index 013b8d4386..007b1af65b 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java -@@ -76,7 +76,18 @@ public class TestDataSourceOptions extends SparkTestBaseWithCatalog { +@@ -76,7 +76,19 @@ public class TestDataSourceOptions extends SparkTestBaseWithCatalog { @BeforeClass public static void startSpark() { @@ -1155,6 +1196,7 @@ index 013b8d4386..4c62e88fe9 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1162,10 +1204,10 @@ index 013b8d4386..4c62e88fe9 100644 @AfterClass diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java -index ba13d005bd..8c1223f93c 100644 +index ba13d005bd..b9778c1986 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java -@@ -118,7 +118,18 @@ public class TestFilteredScan { +@@ -118,7 +118,19 @@ public class TestFilteredScan { @BeforeClass public static void startSpark() { @@ -1179,6 +1221,7 @@ index ba13d005bd..8c1223f93c 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1186,10 +1229,10 @@ index ba13d005bd..8c1223f93c 100644 // define UDFs used by partition tests Function bucket4 = Transforms.bucket(4).bind(Types.LongType.get()); diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java -index 9f97753094..d715c6a3a5 100644 +index 9f97753094..fb5bfbad1b 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java -@@ -94,7 +94,18 @@ public class TestForwardCompatibility { +@@ -94,7 +94,19 @@ public class TestForwardCompatibility { @BeforeClass public static void startSpark() { @@ -1203,6 +1246,7 @@ index 9f97753094..d715c6a3a5 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1210,10 +1254,10 @@ index 9f97753094..d715c6a3a5 100644 @AfterClass diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java -index 0154506f86..edd49d5e5b 100644 +index 0154506f86..12e916cbe4 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java -@@ -46,7 +46,18 @@ public class TestIcebergSpark { +@@ -46,7 +46,19 @@ public class TestIcebergSpark { @BeforeClass public static void startSpark() { @@ -1227,6 +1271,7 @@ index 0154506f86..edd49d5e5b 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1234,10 +1279,10 @@ index 0154506f86..edd49d5e5b 100644 @AfterClass diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java -index 639d37c793..0e69c8d992 100644 +index 639d37c793..5e397909d4 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java -@@ -111,7 +111,18 @@ public class TestPartitionPruning { +@@ -111,7 +111,19 @@ public class TestPartitionPruning { @BeforeClass public static void startSpark() { @@ -1251,6 +1296,7 @@ index 639d37c793..0e69c8d992 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1258,10 +1304,10 @@ index 639d37c793..0e69c8d992 100644 String optionKey = String.format("fs.%s.impl", CountOpenLocalFileSystem.scheme); diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java -index ad0984ef42..42a539ffd9 100644 +index ad0984ef42..99af33c86c 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java -@@ -104,7 +104,18 @@ public class TestPartitionValues { +@@ -104,7 +104,19 @@ public class TestPartitionValues { @BeforeClass public static void startSpark() { @@ -1275,6 +1321,7 @@ index ad0984ef42..42a539ffd9 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1282,10 +1329,10 @@ index ad0984ef42..42a539ffd9 100644 @AfterClass diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java -index 9fc576dde5..69913e15cc 100644 +index 9fc576dde5..69a49f5091 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java -@@ -83,7 +83,18 @@ public class TestSnapshotSelection { +@@ -83,7 +83,19 @@ public class TestSnapshotSelection { @BeforeClass public static void startSpark() { @@ -1299,6 +1346,7 @@ index 9fc576dde5..69913e15cc 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1306,10 +1354,10 @@ index 9fc576dde5..69913e15cc 100644 @AfterClass diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java -index 16fde3c954..b28970ca18 100644 +index 16fde3c954..26e94f6dde 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java -@@ -121,7 +121,18 @@ public class TestSparkDataFile { +@@ -121,7 +121,19 @@ public class TestSparkDataFile { @BeforeClass public static void startSpark() { @@ -1323,6 +1371,7 @@ index 16fde3c954..b28970ca18 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1330,10 +1379,10 @@ index 16fde3c954..b28970ca18 100644 } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java -index 63c18277aa..3c00c4a20a 100644 +index 63c18277aa..971f14b647 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java -@@ -89,7 +89,18 @@ public class TestSparkDataWrite { +@@ -89,7 +89,19 @@ public class TestSparkDataWrite { @BeforeClass public static void startSpark() { @@ -1347,13 +1396,14 @@ index 63c18277aa..3c00c4a20a 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); } @Parameterized.AfterParam -@@ -138,7 +149,7 @@ public class TestSparkDataWrite { +@@ -138,7 +150,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1362,7 +1412,7 @@ index 63c18277aa..3c00c4a20a 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); for (ManifestFile manifest : -@@ -208,7 +219,7 @@ public class TestSparkDataWrite { +@@ -208,7 +220,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1371,7 +1421,7 @@ index 63c18277aa..3c00c4a20a 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); } -@@ -254,7 +265,7 @@ public class TestSparkDataWrite { +@@ -254,7 +266,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1380,7 +1430,7 @@ index 63c18277aa..3c00c4a20a 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); } -@@ -307,7 +318,7 @@ public class TestSparkDataWrite { +@@ -307,7 +319,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1389,7 +1439,7 @@ index 63c18277aa..3c00c4a20a 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); } -@@ -350,7 +361,7 @@ public class TestSparkDataWrite { +@@ -350,7 +362,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1398,7 +1448,7 @@ index 63c18277aa..3c00c4a20a 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); } -@@ -390,7 +401,7 @@ public class TestSparkDataWrite { +@@ -390,7 +402,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1407,7 +1457,7 @@ index 63c18277aa..3c00c4a20a 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); -@@ -455,7 +466,7 @@ public class TestSparkDataWrite { +@@ -455,7 +467,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1416,7 +1466,7 @@ index 63c18277aa..3c00c4a20a 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); } -@@ -619,7 +630,7 @@ public class TestSparkDataWrite { +@@ -619,7 +631,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1425,7 +1475,7 @@ index 63c18277aa..3c00c4a20a 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); -@@ -705,7 +716,7 @@ public class TestSparkDataWrite { +@@ -705,7 +717,7 @@ public class TestSparkDataWrite { // Since write and commit succeeded, the rows should be readable Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1435,10 +1485,10 @@ index 63c18277aa..3c00c4a20a 100644 "Number of rows should match", records.size() + records2.size(), actual.size()); assertThat(actual) diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java -index 3a4b235c46..9beb95022d 100644 +index 3a4b235c46..7633a1792d 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java -@@ -86,7 +86,18 @@ public class TestSparkReadProjection extends TestReadProjection { +@@ -86,7 +86,19 @@ public class TestSparkReadProjection extends TestReadProjection { @BeforeClass public static void startSpark() { @@ -1452,6 +1502,7 @@ index 3a4b235c46..9beb95022d 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1459,10 +1510,10 @@ index 3a4b235c46..9beb95022d 100644 ImmutableMap.of( "type", "hive", diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java -index dda49b4946..edf2b24a1c 100644 +index dda49b4946..69d3a0d24f 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java -@@ -131,7 +131,23 @@ public class TestSparkReaderDeletes extends DeleteReadTests { +@@ -131,7 +131,25 @@ public class TestSparkReaderDeletes extends DeleteReadTests { .config("spark.ui.liveUpdate.period", 0) .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) @@ -1472,6 +1523,7 @@ index dda49b4946..edf2b24a1c 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -1481,12 +1533,13 @@ index dda49b4946..edf2b24a1c 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); catalog = -@@ -199,7 +215,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { +@@ -199,7 +217,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { } protected boolean countDeletes() { @@ -1497,10 +1550,10 @@ index dda49b4946..edf2b24a1c 100644 @Override diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java -index e5831b76e4..bcc5cd0a58 100644 +index e5831b76e4..2433013d3e 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java -@@ -176,7 +176,23 @@ public class TestSparkReaderWithBloomFilter { +@@ -176,7 +176,25 @@ public class TestSparkReaderWithBloomFilter { SparkSession.builder() .master("local[2]") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) @@ -1510,6 +1563,7 @@ index e5831b76e4..bcc5cd0a58 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -1519,16 +1573,17 @@ index e5831b76e4..bcc5cd0a58 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); catalog = diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java -index f420f1b955..f5fd3cd91d 100644 +index f420f1b955..21273826ba 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java -@@ -66,6 +66,14 @@ public class TestStructuredStreaming { +@@ -66,6 +66,15 @@ public class TestStructuredStreaming { SparkSession.builder() .master("local[2]") .config("spark.sql.shuffle.partitions", 4) @@ -1538,16 +1593,17 @@ index f420f1b955..f5fd3cd91d 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java -index ac674e2e62..00e064c64d 100644 +index ac674e2e62..c18fed7cfb 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java -@@ -73,7 +73,18 @@ public class TestTimestampWithoutZone extends SparkTestBase { +@@ -73,7 +73,19 @@ public class TestTimestampWithoutZone extends SparkTestBase { @BeforeClass public static void startSpark() { @@ -1561,6 +1617,7 @@ index ac674e2e62..00e064c64d 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1568,10 +1625,10 @@ index ac674e2e62..00e064c64d 100644 @AfterClass diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java -index 73827b309b..6eaa915d5b 100644 +index 73827b309b..ace66913d1 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java -@@ -80,7 +80,18 @@ public class TestWriteMetricsConfig { +@@ -80,7 +80,19 @@ public class TestWriteMetricsConfig { @BeforeClass public static void startSpark() { @@ -1585,6 +1642,7 @@ index 73827b309b..6eaa915d5b 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1592,10 +1650,10 @@ index 73827b309b..6eaa915d5b 100644 } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java -index 1a4e2f3e1c..f2228098ec 100644 +index 1a4e2f3e1c..047e0f1fb4 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java -@@ -65,7 +65,23 @@ public class TestAggregatePushDown extends SparkCatalogTestBase { +@@ -65,7 +65,25 @@ public class TestAggregatePushDown extends SparkCatalogTestBase { SparkSession.builder() .master("local[2]") .config("spark.sql.iceberg.aggregate_pushdown", "true") @@ -1605,6 +1663,7 @@ index 1a4e2f3e1c..f2228098ec 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -1614,6 +1673,7 @@ index 1a4e2f3e1c..f2228098ec 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); @@ -1678,10 +1738,10 @@ index e2d2c7a7ac..95cd3c08b5 100644 // runtime dependencies for running Hive Catalog based integration test integrationRuntimeOnly project(':iceberg-hive-metastore') diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java -index 578845e3da..3cc1598035 100644 +index 578845e3da..02b6dc270f 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java -@@ -57,6 +57,14 @@ public abstract class ExtensionsTestBase extends CatalogTestBase { +@@ -57,6 +57,15 @@ public abstract class ExtensionsTestBase extends CatalogTestBase { .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") .config( SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), String.valueOf(RANDOM.nextBoolean())) @@ -1691,16 +1751,17 @@ index 578845e3da..3cc1598035 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() .getOrCreate(); diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java -index ade19de36f..366a094cfd 100644 +index ade19de36f..579b68e73f 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java -@@ -56,6 +56,14 @@ public class TestCallStatementParser { +@@ -56,6 +56,15 @@ public class TestCallStatementParser { .master("local[2]") .config("spark.sql.extensions", IcebergSparkSessionExtensions.class.getName()) .config("spark.extra.prop", "value") @@ -1710,16 +1771,45 @@ index ade19de36f..366a094cfd 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); TestCallStatementParser.parser = spark.sessionState().sqlParser(); } +diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java +index 8cb92224fc..777b9bb1dd 100644 +--- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java ++++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java +@@ -38,6 +38,7 @@ import org.apache.spark.sql.catalyst.expressions.ApplyFunctionExpression; + import org.apache.spark.sql.catalyst.expressions.Expression; + import org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke; + import org.apache.spark.sql.execution.CommandResultExec; ++import org.apache.spark.sql.execution.SparkPlan; + import org.apache.spark.sql.execution.datasources.v2.V2TableWriteExec; + import org.junit.jupiter.api.AfterEach; + import org.junit.jupiter.api.BeforeEach; +@@ -313,9 +314,13 @@ public class TestSystemFunctionPushDownInRowLevelOperations extends ExtensionsTe + + private List executeAndCollectFunctionCalls(String query, Object... args) { + CommandResultExec command = (CommandResultExec) executeAndKeepPlan(query, args); +- V2TableWriteExec write = (V2TableWriteExec) command.commandPhysicalPlan(); ++ // Comet's split-operator Iceberg write plans the command as IcebergCommitExec, which is not ++ // a V2TableWriteExec; collect over the whole command plan in that case. ++ SparkPlan plan = command.commandPhysicalPlan(); ++ SparkPlan queryPlan = ++ plan instanceof V2TableWriteExec ? ((V2TableWriteExec) plan).query() : plan; + return SparkPlanUtil.collectExprs( +- write.query(), ++ queryPlan, + expr -> expr instanceof StaticInvoke || expr instanceof ApplyFunctionExpression); + } + diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java -index 64edb1002e..4706106c2e 100644 +index 64edb1002e..57767da086 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java -@@ -179,6 +179,14 @@ public class DeleteOrphanFilesBenchmark { +@@ -179,6 +179,15 @@ public class DeleteOrphanFilesBenchmark { .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", catalogWarehouse()) @@ -1729,16 +1819,17 @@ index 64edb1002e..4706106c2e 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local"); spark = builder.getOrCreate(); } diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java -index a5d0456b0b..815de72f4a 100644 +index a5d0456b0b..f52c387943 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java -@@ -392,6 +392,14 @@ public class IcebergSortCompactionBenchmark { +@@ -392,6 +392,15 @@ public class IcebergSortCompactionBenchmark { "spark.sql.catalog.spark_catalog", "org.apache.iceberg.spark.SparkSessionCatalog") .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", getCatalogWarehouse()) @@ -1748,16 +1839,17 @@ index a5d0456b0b..815de72f4a 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[*]"); spark = builder.getOrCreate(); Configuration sparkHadoopConf = spark.sessionState().newHadoopConf(); diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java -index c6794e43c6..ada3a83e3c 100644 +index c6794e43c6..4c1a165a44 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java -@@ -239,6 +239,14 @@ public class DVReaderBenchmark { +@@ -239,6 +239,15 @@ public class DVReaderBenchmark { .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", newWarehouseDir()) @@ -1767,16 +1859,17 @@ index c6794e43c6..ada3a83e3c 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[*]") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java -index ac74fb5a10..bafca20a1c 100644 +index ac74fb5a10..4aa5d5b361 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java -@@ -223,6 +223,14 @@ public class DVWriterBenchmark { +@@ -223,6 +223,15 @@ public class DVWriterBenchmark { .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", newWarehouseDir()) @@ -1786,16 +1879,17 @@ index ac74fb5a10..bafca20a1c 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[*]") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java -index 68c537e34a..5ef031963a 100644 +index 68c537e34a..2e421c7706 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java -@@ -94,7 +94,17 @@ public abstract class IcebergSourceBenchmark { +@@ -94,7 +94,18 @@ public abstract class IcebergSourceBenchmark { } protected void setupSpark(boolean enableDictionaryEncoding) { @@ -1809,6 +1903,7 @@ index 68c537e34a..5ef031963a 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g"); if (!enableDictionaryEncoding) { @@ -2612,10 +2707,10 @@ index 780e1750a5..25f253eede 100644 .filter(residual) .caseSensitive(caseSensitive()) diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java -index 404ba72846..5bd4e2b351 100644 +index 404ba72846..b375458f05 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java -@@ -90,6 +90,14 @@ public abstract class SparkDistributedDataScanTestBase +@@ -90,6 +90,15 @@ public abstract class SparkDistributedDataScanTestBase .master("local[2]") .config("spark.serializer", serializer) .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -2625,16 +2720,17 @@ index 404ba72846..5bd4e2b351 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java -index 659507e4c5..74182efbb2 100644 +index 659507e4c5..2a3a87f079 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java -@@ -73,6 +73,14 @@ public class TestSparkDistributedDataScanDeletes +@@ -73,6 +73,15 @@ public class TestSparkDistributedDataScanDeletes .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -2644,16 +2740,17 @@ index 659507e4c5..74182efbb2 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java -index a218f965ea..b1e0fcf2ca 100644 +index a218f965ea..e30a8d6c6f 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java -@@ -62,6 +62,14 @@ public class TestSparkDistributedDataScanFilterFiles +@@ -62,6 +62,15 @@ public class TestSparkDistributedDataScanFilterFiles .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -2663,16 +2760,17 @@ index a218f965ea..b1e0fcf2ca 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java -index 2665d7ba8d..3539020102 100644 +index 2665d7ba8d..c24c82341a 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java -@@ -63,6 +63,14 @@ public class TestSparkDistributedDataScanReporting +@@ -63,6 +63,15 @@ public class TestSparkDistributedDataScanReporting .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -2682,16 +2780,17 @@ index 2665d7ba8d..3539020102 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java -index de68351f6e..785f7a3b58 100644 +index de68351f6e..dea8226742 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java -@@ -77,6 +77,14 @@ public abstract class TestBase extends SparkTestHelperBase { +@@ -77,6 +77,15 @@ public abstract class TestBase extends SparkTestHelperBase { .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") @@ -2701,16 +2800,17 @@ index de68351f6e..785f7a3b58 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() .getOrCreate(); diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetDictionaryEncodedVectorizedReads.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetDictionaryEncodedVectorizedReads.java -index bc4e722bc8..c7a417e105 100644 +index bc4e722bc8..57770308b3 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetDictionaryEncodedVectorizedReads.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetDictionaryEncodedVectorizedReads.java -@@ -59,7 +59,18 @@ public class TestParquetDictionaryEncodedVectorizedReads extends TestParquetVect +@@ -59,7 +59,19 @@ public class TestParquetDictionaryEncodedVectorizedReads extends TestParquetVect @BeforeAll public static void startSpark() { @@ -2724,6 +2824,7 @@ index bc4e722bc8..c7a417e105 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2731,10 +2832,10 @@ index bc4e722bc8..c7a417e105 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java -index 3a269740b7..2035edf93a 100644 +index 3a269740b7..fc63449215 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java -@@ -54,7 +54,18 @@ public abstract class ScanTestBase extends AvroDataTest { +@@ -54,7 +54,19 @@ public abstract class ScanTestBase extends AvroDataTest { @BeforeAll public static void startSpark() { @@ -2748,6 +2849,7 @@ index 3a269740b7..2035edf93a 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2755,10 +2857,10 @@ index 3a269740b7..2035edf93a 100644 } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java -index f411920a5d..cf3b710d2d 100644 +index f411920a5d..ec8eebe117 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java -@@ -144,7 +144,18 @@ public class TestCompressionSettings extends CatalogTestBase { +@@ -144,7 +144,19 @@ public class TestCompressionSettings extends CatalogTestBase { @BeforeAll public static void startSpark() { @@ -2772,6 +2874,7 @@ index f411920a5d..cf3b710d2d 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2779,10 +2882,10 @@ index f411920a5d..cf3b710d2d 100644 @BeforeEach diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java -index c4ba96e634..5faeec110c 100644 +index c4ba96e634..dcf72c19b6 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java -@@ -75,7 +75,18 @@ public class TestDataSourceOptions extends TestBaseWithCatalog { +@@ -75,7 +75,19 @@ public class TestDataSourceOptions extends TestBaseWithCatalog { @BeforeAll public static void startSpark() { @@ -2796,6 +2899,7 @@ index c4ba96e634..5faeec110c 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2803,10 +2907,10 @@ index c4ba96e634..5faeec110c 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java -index 348173596e..828fe716a5 100644 +index 348173596e..03156f786b 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java -@@ -110,7 +110,18 @@ public class TestFilteredScan { +@@ -110,7 +110,19 @@ public class TestFilteredScan { @BeforeAll public static void startSpark() { @@ -2820,6 +2924,7 @@ index 348173596e..828fe716a5 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2827,10 +2932,10 @@ index 348173596e..828fe716a5 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java -index 84c99a575c..79dfc3aff3 100644 +index 84c99a575c..e647ab924e 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java -@@ -93,7 +93,18 @@ public class TestForwardCompatibility { +@@ -93,7 +93,19 @@ public class TestForwardCompatibility { @BeforeAll public static void startSpark() { @@ -2844,6 +2949,7 @@ index 84c99a575c..79dfc3aff3 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2851,10 +2957,10 @@ index 84c99a575c..79dfc3aff3 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java -index 7eff93d204..aead2a6924 100644 +index 7eff93d204..2fdc87a0a8 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java -@@ -46,7 +46,18 @@ public class TestIcebergSpark { +@@ -46,7 +46,19 @@ public class TestIcebergSpark { @BeforeAll public static void startSpark() { @@ -2868,6 +2974,7 @@ index 7eff93d204..aead2a6924 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2875,10 +2982,10 @@ index 7eff93d204..aead2a6924 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java -index 9464f687b0..4850759d61 100644 +index 9464f687b0..8653560334 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java -@@ -112,7 +112,18 @@ public class TestPartitionPruning { +@@ -112,7 +112,19 @@ public class TestPartitionPruning { @BeforeAll public static void startSpark() { @@ -2892,6 +2999,7 @@ index 9464f687b0..4850759d61 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2899,10 +3007,10 @@ index 9464f687b0..4850759d61 100644 String optionKey = String.format("fs.%s.impl", CountOpenLocalFileSystem.scheme); diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java -index 5c218f21c4..dd4aeca9ef 100644 +index 5c218f21c4..2f17343d59 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java -@@ -107,7 +107,18 @@ public class TestPartitionValues { +@@ -107,7 +107,19 @@ public class TestPartitionValues { @BeforeAll public static void startSpark() { @@ -2916,6 +3024,7 @@ index 5c218f21c4..dd4aeca9ef 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2923,10 +3032,10 @@ index 5c218f21c4..dd4aeca9ef 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java -index a7334a580c..f6f034240b 100644 +index a7334a580c..68210dc8e8 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java -@@ -87,7 +87,18 @@ public class TestSnapshotSelection { +@@ -87,7 +87,19 @@ public class TestSnapshotSelection { @BeforeAll public static void startSpark() { @@ -2940,6 +3049,7 @@ index a7334a580c..f6f034240b 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2947,10 +3057,10 @@ index a7334a580c..f6f034240b 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java -index 182b1ef8f5..47e6f0bd01 100644 +index 182b1ef8f5..76e04b1472 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java -@@ -120,7 +120,18 @@ public class TestSparkDataFile { +@@ -120,7 +120,19 @@ public class TestSparkDataFile { @BeforeAll public static void startSpark() { @@ -2964,6 +3074,7 @@ index 182b1ef8f5..47e6f0bd01 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2971,10 +3082,10 @@ index 182b1ef8f5..47e6f0bd01 100644 } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java -index fb2b312bed..25b6170fd3 100644 +index fb2b312bed..846f50e0c9 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java -@@ -96,7 +96,18 @@ public class TestSparkDataWrite { +@@ -96,7 +96,19 @@ public class TestSparkDataWrite { @BeforeAll public static void startSpark() { @@ -2988,13 +3099,14 @@ index fb2b312bed..25b6170fd3 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); } @AfterEach -@@ -140,7 +151,7 @@ public class TestSparkDataWrite { +@@ -140,7 +152,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3003,7 +3115,7 @@ index fb2b312bed..25b6170fd3 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); for (ManifestFile manifest : -@@ -210,7 +221,7 @@ public class TestSparkDataWrite { +@@ -210,7 +222,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3012,7 +3124,7 @@ index fb2b312bed..25b6170fd3 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); } -@@ -256,7 +267,7 @@ public class TestSparkDataWrite { +@@ -256,7 +268,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3021,7 +3133,7 @@ index fb2b312bed..25b6170fd3 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); } -@@ -309,7 +320,7 @@ public class TestSparkDataWrite { +@@ -309,7 +321,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3030,7 +3142,7 @@ index fb2b312bed..25b6170fd3 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); } -@@ -352,7 +363,7 @@ public class TestSparkDataWrite { +@@ -352,7 +364,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3039,7 +3151,7 @@ index fb2b312bed..25b6170fd3 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); } -@@ -392,7 +403,7 @@ public class TestSparkDataWrite { +@@ -392,7 +404,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3048,7 +3160,7 @@ index fb2b312bed..25b6170fd3 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); -@@ -458,7 +469,7 @@ public class TestSparkDataWrite { +@@ -458,7 +470,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3057,7 +3169,7 @@ index fb2b312bed..25b6170fd3 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); } -@@ -622,7 +633,7 @@ public class TestSparkDataWrite { +@@ -622,7 +634,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3066,7 +3178,7 @@ index fb2b312bed..25b6170fd3 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); -@@ -708,7 +719,7 @@ public class TestSparkDataWrite { +@@ -708,7 +720,7 @@ public class TestSparkDataWrite { // Since write and commit succeeded, the rows should be readable Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3076,10 +3188,10 @@ index fb2b312bed..25b6170fd3 100644 assertThat(actual) .describedAs("Result rows should match") diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java -index becf6a064d..f00afdf707 100644 +index becf6a064d..91451c268e 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java -@@ -83,7 +83,18 @@ public class TestSparkReadProjection extends TestReadProjection { +@@ -83,7 +83,19 @@ public class TestSparkReadProjection extends TestReadProjection { @BeforeAll public static void startSpark() { @@ -3093,6 +3205,7 @@ index becf6a064d..f00afdf707 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -3100,10 +3213,10 @@ index becf6a064d..f00afdf707 100644 ImmutableMap.of( "type", "hive", diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java -index 4f1cef5d37..e1b1716dcd 100644 +index 4f1cef5d37..4a1a602ed4 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java -@@ -136,6 +136,14 @@ public class TestSparkReaderDeletes extends DeleteReadTests { +@@ -136,6 +136,15 @@ public class TestSparkReaderDeletes extends DeleteReadTests { .config("spark.ui.liveUpdate.period", 0) .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) @@ -3113,12 +3226,13 @@ index 4f1cef5d37..e1b1716dcd 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() .getOrCreate(); -@@ -204,7 +212,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { +@@ -204,7 +213,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { } protected boolean countDeletes() { @@ -3129,10 +3243,10 @@ index 4f1cef5d37..e1b1716dcd 100644 @Override diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java -index baf7fa8f88..5ada29b4af 100644 +index baf7fa8f88..b58321b761 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java -@@ -182,6 +182,14 @@ public class TestSparkReaderWithBloomFilter { +@@ -182,6 +182,15 @@ public class TestSparkReaderWithBloomFilter { SparkSession.builder() .master("local[2]") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) @@ -3142,16 +3256,17 @@ index baf7fa8f88..5ada29b4af 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() .getOrCreate(); diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java -index 17db46b85c..93dfa45e9c 100644 +index 17db46b85c..d424b97d2a 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java -@@ -65,6 +65,14 @@ public class TestStructuredStreaming { +@@ -65,6 +65,15 @@ public class TestStructuredStreaming { SparkSession.builder() .master("local[2]") .config("spark.sql.shuffle.partitions", 4) @@ -3161,16 +3276,17 @@ index 17db46b85c..93dfa45e9c 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java -index 306444b9f2..e4c828110e 100644 +index 306444b9f2..e4082d9cd9 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java -@@ -75,7 +75,18 @@ public class TestTimestampWithoutZone extends TestBase { +@@ -75,7 +75,19 @@ public class TestTimestampWithoutZone extends TestBase { @BeforeAll public static void startSpark() { @@ -3184,6 +3300,7 @@ index 306444b9f2..e4c828110e 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -3191,10 +3308,10 @@ index 306444b9f2..e4c828110e 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java -index 841268a6be..ec29f30204 100644 +index 841268a6be..97f7187314 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java -@@ -80,7 +80,18 @@ public class TestWriteMetricsConfig { +@@ -80,7 +80,19 @@ public class TestWriteMetricsConfig { @BeforeAll public static void startSpark() { @@ -3208,6 +3325,7 @@ index 841268a6be..ec29f30204 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -3215,10 +3333,10 @@ index 841268a6be..ec29f30204 100644 } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java -index 6e09252704..2ed54a2e60 100644 +index 6e09252704..4b0f2e4d58 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java -@@ -60,6 +60,14 @@ public class TestAggregatePushDown extends CatalogTestBase { +@@ -60,6 +60,15 @@ public class TestAggregatePushDown extends CatalogTestBase { SparkSession.builder() .master("local[2]") .config("spark.sql.iceberg.aggregate_pushdown", "true") @@ -3228,6 +3346,7 @@ index 6e09252704..2ed54a2e60 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -3261,59 +3380,3 @@ index 6719c45ca9..2515454401 100644 sourceColumnName, tableName, tableName(OTHER_TABLE_NAME), -diff --git a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java -index 49119d319..04fd3d485 100644 ---- a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java -+++ b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java -@@ -37,6 +37,7 @@ import org.apache.spark.sql.catalyst.expressions.ApplyFunctionExpression; - import org.apache.spark.sql.catalyst.expressions.Expression; - import org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke; - import org.apache.spark.sql.execution.CommandResultExec; -+import org.apache.spark.sql.execution.SparkPlan; - import org.apache.spark.sql.execution.datasources.v2.V2TableWriteExec; - import org.junit.After; - import org.junit.Before; -@@ -316,9 +317,13 @@ public class TestSystemFunctionPushDownInRowLevelOperations extends SparkExtensi - - private List executeAndCollectFunctionCalls(String query, Object... args) { - CommandResultExec command = (CommandResultExec) executeAndKeepPlan(query, args); -- V2TableWriteExec write = (V2TableWriteExec) command.commandPhysicalPlan(); -+ // Comet's split-operator Iceberg write plans the command as IcebergCommitExec, which is not -+ // a V2TableWriteExec; collect over the whole command plan in that case. -+ SparkPlan plan = command.commandPhysicalPlan(); -+ SparkPlan queryPlan = -+ plan instanceof V2TableWriteExec ? ((V2TableWriteExec) plan).query() : plan; - return SparkPlanUtil.collectExprs( -- write.query(), -+ queryPlan, - expr -> expr instanceof StaticInvoke || expr instanceof ApplyFunctionExpression); - } - -diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java -index 8cb92224f..777b9bb1d 100644 ---- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java -+++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java -@@ -38,6 +38,7 @@ import org.apache.spark.sql.catalyst.expressions.ApplyFunctionExpression; - import org.apache.spark.sql.catalyst.expressions.Expression; - import org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke; - import org.apache.spark.sql.execution.CommandResultExec; -+import org.apache.spark.sql.execution.SparkPlan; - import org.apache.spark.sql.execution.datasources.v2.V2TableWriteExec; - import org.junit.jupiter.api.AfterEach; - import org.junit.jupiter.api.BeforeEach; -@@ -313,9 +314,13 @@ public class TestSystemFunctionPushDownInRowLevelOperations extends ExtensionsTe - - private List executeAndCollectFunctionCalls(String query, Object... args) { - CommandResultExec command = (CommandResultExec) executeAndKeepPlan(query, args); -- V2TableWriteExec write = (V2TableWriteExec) command.commandPhysicalPlan(); -+ // Comet's split-operator Iceberg write plans the command as IcebergCommitExec, which is not -+ // a V2TableWriteExec; collect over the whole command plan in that case. -+ SparkPlan plan = command.commandPhysicalPlan(); -+ SparkPlan queryPlan = -+ plan instanceof V2TableWriteExec ? ((V2TableWriteExec) plan).query() : plan; - return SparkPlanUtil.collectExprs( -- write.query(), -+ queryPlan, - expr -> expr instanceof StaticInvoke || expr instanceof ApplyFunctionExpression); - } - diff --git a/dev/diffs/iceberg/1.9.1.diff b/dev/diffs/iceberg/1.9.1.diff index 636f7e958f9..139003cae3b 100644 --- a/dev/diffs/iceberg/1.9.1.diff +++ b/dev/diffs/iceberg/1.9.1.diff @@ -1,5 +1,5 @@ diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml -index c50991c5fc..2892f11068 100644 +index c50991c5fc..4c6db98159 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -36,6 +36,7 @@ awssdk-s3accessgrants = "2.3.0" @@ -41,10 +41,10 @@ index 2cd8666892..4077116abb 100644 // runtime dependencies for running Hive Catalog based integration test integrationRuntimeOnly project(':iceberg-hive-metastore') diff --git a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java -index 578845e3da..5cdda2a394 100644 +index 578845e3da..19a200892f 100644 --- a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java +++ b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java -@@ -57,7 +57,23 @@ public abstract class ExtensionsTestBase extends CatalogTestBase { +@@ -57,7 +57,25 @@ public abstract class ExtensionsTestBase extends CatalogTestBase { .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") .config( SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), String.valueOf(RANDOM.nextBoolean())) @@ -54,6 +54,7 @@ index 578845e3da..5cdda2a394 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -63,16 +64,17 @@ index 578845e3da..5cdda2a394 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); TestBase.sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext()); diff --git a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java -index 58d054bd05..61b307b92b 100644 +index 58d054bd05..ef1c1cb6d1 100644 --- a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java +++ b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java -@@ -56,6 +56,14 @@ public class TestCallStatementParser { +@@ -56,6 +56,15 @@ public class TestCallStatementParser { .master("local[2]") .config("spark.sql.extensions", IcebergSparkSessionExtensions.class.getName()) .config("spark.extra.prop", "value") @@ -82,16 +84,45 @@ index 58d054bd05..61b307b92b 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); TestCallStatementParser.parser = spark.sessionState().sqlParser(); } +diff --git a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java +index 5a0a04edba..afe92a8b72 100644 +--- a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java ++++ b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java +@@ -38,6 +38,7 @@ import org.apache.spark.sql.catalyst.expressions.ApplyFunctionExpression; + import org.apache.spark.sql.catalyst.expressions.Expression; + import org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke; + import org.apache.spark.sql.execution.CommandResultExec; ++import org.apache.spark.sql.execution.SparkPlan; + import org.apache.spark.sql.execution.datasources.v2.V2TableWriteExec; + import org.junit.jupiter.api.AfterEach; + import org.junit.jupiter.api.BeforeEach; +@@ -313,9 +314,13 @@ public class TestSystemFunctionPushDownInRowLevelOperations extends ExtensionsTe + + private List executeAndCollectFunctionCalls(String query, Object... args) { + CommandResultExec command = (CommandResultExec) executeAndKeepPlan(query, args); +- V2TableWriteExec write = (V2TableWriteExec) command.commandPhysicalPlan(); ++ // Comet's split-operator Iceberg write plans the command as IcebergCommitExec, which is not ++ // a V2TableWriteExec; collect over the whole command plan in that case. ++ SparkPlan plan = command.commandPhysicalPlan(); ++ SparkPlan queryPlan = ++ plan instanceof V2TableWriteExec ? ((V2TableWriteExec) plan).query() : plan; + return SparkPlanUtil.collectExprs( +- write.query(), ++ queryPlan, + expr -> expr instanceof StaticInvoke || expr instanceof ApplyFunctionExpression); + } + diff --git a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java -index b6ade2bff3..b52d011691 100644 +index b6ade2bff3..90a4839dad 100644 --- a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java +++ b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java -@@ -170,6 +170,14 @@ public class DeleteOrphanFilesBenchmark { +@@ -170,6 +170,15 @@ public class DeleteOrphanFilesBenchmark { .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", catalogWarehouse()) @@ -101,16 +132,17 @@ index b6ade2bff3..b52d011691 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local"); spark = builder.getOrCreate(); } diff --git a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java -index b08c352819..d2ca76e07a 100644 +index b08c352819..63880b089d 100644 --- a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java +++ b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java -@@ -386,6 +386,14 @@ public class IcebergSortCompactionBenchmark { +@@ -386,6 +386,15 @@ public class IcebergSortCompactionBenchmark { "spark.sql.catalog.spark_catalog", "org.apache.iceberg.spark.SparkSessionCatalog") .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", getCatalogWarehouse()) @@ -120,16 +152,17 @@ index b08c352819..d2ca76e07a 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[*]"); spark = builder.getOrCreate(); Configuration sparkHadoopConf = spark.sessionState().newHadoopConf(); diff --git a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java -index 68c537e34a..5ef031963a 100644 +index 68c537e34a..2e421c7706 100644 --- a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java +++ b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java -@@ -94,7 +94,17 @@ public abstract class IcebergSourceBenchmark { +@@ -94,7 +94,18 @@ public abstract class IcebergSourceBenchmark { } protected void setupSpark(boolean enableDictionaryEncoding) { @@ -143,6 +176,7 @@ index 68c537e34a..5ef031963a 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g"); if (!enableDictionaryEncoding) { @@ -930,10 +964,10 @@ index 780e1750a5..25f253eede 100644 .filter(residual) .caseSensitive(caseSensitive()) diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java -index 404ba72846..5bd4e2b351 100644 +index 404ba72846..b375458f05 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java -@@ -90,6 +90,14 @@ public abstract class SparkDistributedDataScanTestBase +@@ -90,6 +90,15 @@ public abstract class SparkDistributedDataScanTestBase .master("local[2]") .config("spark.serializer", serializer) .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -943,16 +977,17 @@ index 404ba72846..5bd4e2b351 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java -index 9361c63176..44c59c8971 100644 +index 9361c63176..16ce1c614f 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java -@@ -69,6 +69,14 @@ public class TestSparkDistributedDataScanDeletes +@@ -69,6 +69,15 @@ public class TestSparkDistributedDataScanDeletes .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -962,16 +997,17 @@ index 9361c63176..44c59c8971 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java -index a218f965ea..b1e0fcf2ca 100644 +index a218f965ea..e30a8d6c6f 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java -@@ -62,6 +62,14 @@ public class TestSparkDistributedDataScanFilterFiles +@@ -62,6 +62,15 @@ public class TestSparkDistributedDataScanFilterFiles .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -981,16 +1017,17 @@ index a218f965ea..b1e0fcf2ca 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java -index acd4688440..fffc604b48 100644 +index acd4688440..33732e6654 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java -@@ -59,6 +59,14 @@ public class TestSparkDistributedDataScanReporting +@@ -59,6 +59,15 @@ public class TestSparkDistributedDataScanReporting .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -1000,16 +1037,17 @@ index acd4688440..fffc604b48 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/SparkTestBase.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/SparkTestBase.java -index 3e8953fb95..65a8d8ca35 100644 +index 3e8953fb95..51ce392476 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/SparkTestBase.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/SparkTestBase.java -@@ -77,6 +77,14 @@ public abstract class SparkTestBase extends SparkTestHelperBase { +@@ -77,6 +77,15 @@ public abstract class SparkTestBase extends SparkTestHelperBase { .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") @@ -1019,16 +1057,17 @@ index 3e8953fb95..65a8d8ca35 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() .getOrCreate(); diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java -index 06d5e0c44f..fdef2072e1 100644 +index 06d5e0c44f..effc2bcc69 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java -@@ -57,7 +57,18 @@ public abstract class ScanTestBase extends AvroDataTest { +@@ -57,7 +57,19 @@ public abstract class ScanTestBase extends AvroDataTest { @BeforeAll public static void startSpark() { @@ -1042,6 +1081,7 @@ index 06d5e0c44f..fdef2072e1 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1049,10 +1089,10 @@ index 06d5e0c44f..fdef2072e1 100644 } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java -index 26cb4dcc95..39a4a64d88 100644 +index 26cb4dcc95..d185f1175b 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java -@@ -104,7 +104,18 @@ public class TestCompressionSettings extends SparkCatalogTestBase { +@@ -104,7 +104,19 @@ public class TestCompressionSettings extends SparkCatalogTestBase { @BeforeClass public static void startSpark() { @@ -1066,6 +1106,7 @@ index 26cb4dcc95..39a4a64d88 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1073,10 +1114,10 @@ index 26cb4dcc95..39a4a64d88 100644 @Before diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java -index 013b8d4386..4c62e88fe9 100644 +index 013b8d4386..007b1af65b 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java -@@ -76,7 +76,18 @@ public class TestDataSourceOptions extends SparkTestBaseWithCatalog { +@@ -76,7 +76,19 @@ public class TestDataSourceOptions extends SparkTestBaseWithCatalog { @BeforeClass public static void startSpark() { @@ -1090,6 +1131,7 @@ index 013b8d4386..4c62e88fe9 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1097,10 +1139,10 @@ index 013b8d4386..4c62e88fe9 100644 @AfterClass diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java -index ba13d005bd..8c1223f93c 100644 +index ba13d005bd..b9778c1986 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java -@@ -118,7 +118,18 @@ public class TestFilteredScan { +@@ -118,7 +118,19 @@ public class TestFilteredScan { @BeforeClass public static void startSpark() { @@ -1114,6 +1156,7 @@ index ba13d005bd..8c1223f93c 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1121,10 +1164,10 @@ index ba13d005bd..8c1223f93c 100644 // define UDFs used by partition tests Function bucket4 = Transforms.bucket(4).bind(Types.LongType.get()); diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java -index 9f97753094..d715c6a3a5 100644 +index 9f97753094..fb5bfbad1b 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java -@@ -94,7 +94,18 @@ public class TestForwardCompatibility { +@@ -94,7 +94,19 @@ public class TestForwardCompatibility { @BeforeClass public static void startSpark() { @@ -1138,6 +1181,7 @@ index 9f97753094..d715c6a3a5 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1145,10 +1189,10 @@ index 9f97753094..d715c6a3a5 100644 @AfterClass diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java -index 0154506f86..edd49d5e5b 100644 +index 0154506f86..12e916cbe4 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java -@@ -46,7 +46,18 @@ public class TestIcebergSpark { +@@ -46,7 +46,19 @@ public class TestIcebergSpark { @BeforeClass public static void startSpark() { @@ -1162,6 +1206,7 @@ index 0154506f86..edd49d5e5b 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1169,10 +1214,10 @@ index 0154506f86..edd49d5e5b 100644 @AfterClass diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java -index 639d37c793..0e69c8d992 100644 +index 639d37c793..5e397909d4 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java -@@ -111,7 +111,18 @@ public class TestPartitionPruning { +@@ -111,7 +111,19 @@ public class TestPartitionPruning { @BeforeClass public static void startSpark() { @@ -1186,6 +1231,7 @@ index 639d37c793..0e69c8d992 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1193,10 +1239,10 @@ index 639d37c793..0e69c8d992 100644 String optionKey = String.format("fs.%s.impl", CountOpenLocalFileSystem.scheme); diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java -index ad0984ef42..42a539ffd9 100644 +index ad0984ef42..99af33c86c 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java -@@ -104,7 +104,18 @@ public class TestPartitionValues { +@@ -104,7 +104,19 @@ public class TestPartitionValues { @BeforeClass public static void startSpark() { @@ -1210,6 +1256,7 @@ index ad0984ef42..42a539ffd9 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1217,10 +1264,10 @@ index ad0984ef42..42a539ffd9 100644 @AfterClass diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java -index 9fc576dde5..69913e15cc 100644 +index 9fc576dde5..69a49f5091 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java -@@ -83,7 +83,18 @@ public class TestSnapshotSelection { +@@ -83,7 +83,19 @@ public class TestSnapshotSelection { @BeforeClass public static void startSpark() { @@ -1234,6 +1281,7 @@ index 9fc576dde5..69913e15cc 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1241,10 +1289,10 @@ index 9fc576dde5..69913e15cc 100644 @AfterClass diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java -index 16fde3c954..b28970ca18 100644 +index 16fde3c954..26e94f6dde 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java -@@ -121,7 +121,18 @@ public class TestSparkDataFile { +@@ -121,7 +121,19 @@ public class TestSparkDataFile { @BeforeClass public static void startSpark() { @@ -1258,6 +1306,7 @@ index 16fde3c954..b28970ca18 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1265,10 +1314,10 @@ index 16fde3c954..b28970ca18 100644 } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java -index 63c18277aa..3c00c4a20a 100644 +index 63c18277aa..971f14b647 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java -@@ -89,7 +89,18 @@ public class TestSparkDataWrite { +@@ -89,7 +89,19 @@ public class TestSparkDataWrite { @BeforeClass public static void startSpark() { @@ -1282,13 +1331,14 @@ index 63c18277aa..3c00c4a20a 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); } @Parameterized.AfterParam -@@ -138,7 +149,7 @@ public class TestSparkDataWrite { +@@ -138,7 +150,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1297,7 +1347,7 @@ index 63c18277aa..3c00c4a20a 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); for (ManifestFile manifest : -@@ -208,7 +219,7 @@ public class TestSparkDataWrite { +@@ -208,7 +220,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1306,7 +1356,7 @@ index 63c18277aa..3c00c4a20a 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); } -@@ -254,7 +265,7 @@ public class TestSparkDataWrite { +@@ -254,7 +266,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1315,7 +1365,7 @@ index 63c18277aa..3c00c4a20a 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); } -@@ -307,7 +318,7 @@ public class TestSparkDataWrite { +@@ -307,7 +319,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1324,7 +1374,7 @@ index 63c18277aa..3c00c4a20a 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); } -@@ -350,7 +361,7 @@ public class TestSparkDataWrite { +@@ -350,7 +362,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1333,7 +1383,7 @@ index 63c18277aa..3c00c4a20a 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); } -@@ -390,7 +401,7 @@ public class TestSparkDataWrite { +@@ -390,7 +402,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1342,7 +1392,7 @@ index 63c18277aa..3c00c4a20a 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); -@@ -455,7 +466,7 @@ public class TestSparkDataWrite { +@@ -455,7 +467,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1351,7 +1401,7 @@ index 63c18277aa..3c00c4a20a 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); } -@@ -619,7 +630,7 @@ public class TestSparkDataWrite { +@@ -619,7 +631,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1360,7 +1410,7 @@ index 63c18277aa..3c00c4a20a 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); -@@ -705,7 +716,7 @@ public class TestSparkDataWrite { +@@ -705,7 +717,7 @@ public class TestSparkDataWrite { // Since write and commit succeeded, the rows should be readable Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1370,10 +1420,10 @@ index 63c18277aa..3c00c4a20a 100644 "Number of rows should match", records.size() + records2.size(), actual.size()); assertThat(actual) diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java -index 584a6b1c70..9d7fc94684 100644 +index 584a6b1c70..b93a23d6f1 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java -@@ -86,7 +86,18 @@ public class TestSparkReadProjection extends TestReadProjection { +@@ -86,7 +86,19 @@ public class TestSparkReadProjection extends TestReadProjection { @BeforeClass public static void startSpark() { @@ -1387,6 +1437,7 @@ index 584a6b1c70..9d7fc94684 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1394,10 +1445,10 @@ index 584a6b1c70..9d7fc94684 100644 ImmutableMap.of( "type", "hive", diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java -index dda49b4946..edf2b24a1c 100644 +index dda49b4946..69d3a0d24f 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java -@@ -131,7 +131,23 @@ public class TestSparkReaderDeletes extends DeleteReadTests { +@@ -131,7 +131,25 @@ public class TestSparkReaderDeletes extends DeleteReadTests { .config("spark.ui.liveUpdate.period", 0) .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) @@ -1407,6 +1458,7 @@ index dda49b4946..edf2b24a1c 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -1416,12 +1468,13 @@ index dda49b4946..edf2b24a1c 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); catalog = -@@ -199,7 +215,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { +@@ -199,7 +217,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { } protected boolean countDeletes() { @@ -1432,10 +1485,10 @@ index dda49b4946..edf2b24a1c 100644 @Override diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java -index e5831b76e4..bcc5cd0a58 100644 +index e5831b76e4..2433013d3e 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java -@@ -176,7 +176,23 @@ public class TestSparkReaderWithBloomFilter { +@@ -176,7 +176,25 @@ public class TestSparkReaderWithBloomFilter { SparkSession.builder() .master("local[2]") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) @@ -1445,6 +1498,7 @@ index e5831b76e4..bcc5cd0a58 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -1454,16 +1508,17 @@ index e5831b76e4..bcc5cd0a58 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); catalog = diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java -index 961d69b721..d00d87e487 100644 +index 961d69b721..f12ac3e83b 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java -@@ -68,6 +68,14 @@ public class TestStructuredStreaming { +@@ -68,6 +68,15 @@ public class TestStructuredStreaming { SparkSession.builder() .master("local[2]") .config("spark.sql.shuffle.partitions", 4) @@ -1473,16 +1528,17 @@ index 961d69b721..d00d87e487 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java -index ac674e2e62..00e064c64d 100644 +index ac674e2e62..c18fed7cfb 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java -@@ -73,7 +73,18 @@ public class TestTimestampWithoutZone extends SparkTestBase { +@@ -73,7 +73,19 @@ public class TestTimestampWithoutZone extends SparkTestBase { @BeforeClass public static void startSpark() { @@ -1496,6 +1552,7 @@ index ac674e2e62..00e064c64d 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1503,10 +1560,10 @@ index ac674e2e62..00e064c64d 100644 @AfterClass diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java -index 73827b309b..6eaa915d5b 100644 +index 73827b309b..ace66913d1 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java -@@ -80,7 +80,18 @@ public class TestWriteMetricsConfig { +@@ -80,7 +80,19 @@ public class TestWriteMetricsConfig { @BeforeClass public static void startSpark() { @@ -1520,6 +1577,7 @@ index 73827b309b..6eaa915d5b 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1527,10 +1585,10 @@ index 73827b309b..6eaa915d5b 100644 } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java -index 1a4e2f3e1c..f2228098ec 100644 +index 1a4e2f3e1c..047e0f1fb4 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java -@@ -65,7 +65,23 @@ public class TestAggregatePushDown extends SparkCatalogTestBase { +@@ -65,7 +65,25 @@ public class TestAggregatePushDown extends SparkCatalogTestBase { SparkSession.builder() .master("local[2]") .config("spark.sql.iceberg.aggregate_pushdown", "true") @@ -1540,6 +1598,7 @@ index 1a4e2f3e1c..f2228098ec 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -1549,6 +1608,7 @@ index 1a4e2f3e1c..f2228098ec 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); @@ -1613,10 +1673,10 @@ index 572c32f929..29013d26c0 100644 // runtime dependencies for running Hive Catalog based integration test integrationRuntimeOnly project(':iceberg-hive-metastore') diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java -index 578845e3da..3cc1598035 100644 +index 578845e3da..02b6dc270f 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java -@@ -57,6 +57,14 @@ public abstract class ExtensionsTestBase extends CatalogTestBase { +@@ -57,6 +57,15 @@ public abstract class ExtensionsTestBase extends CatalogTestBase { .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") .config( SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), String.valueOf(RANDOM.nextBoolean())) @@ -1626,6 +1686,7 @@ index 578845e3da..3cc1598035 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -1646,10 +1707,10 @@ index b5d6415763..0763a723d3 100644 protected void createTableWithDeleteGranularity( diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java -index ecf9e6f8a5..23dbcf7af2 100644 +index ecf9e6f8a5..beca5f8521 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java -@@ -56,6 +56,14 @@ public class TestCallStatementParser { +@@ -56,6 +56,15 @@ public class TestCallStatementParser { .master("local[2]") .config("spark.sql.extensions", IcebergSparkSessionExtensions.class.getName()) .config("spark.extra.prop", "value") @@ -1659,16 +1720,45 @@ index ecf9e6f8a5..23dbcf7af2 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); TestCallStatementParser.parser = spark.sessionState().sqlParser(); } +diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java +index 8cb92224fc..777b9bb1dd 100644 +--- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java ++++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java +@@ -38,6 +38,7 @@ import org.apache.spark.sql.catalyst.expressions.ApplyFunctionExpression; + import org.apache.spark.sql.catalyst.expressions.Expression; + import org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke; + import org.apache.spark.sql.execution.CommandResultExec; ++import org.apache.spark.sql.execution.SparkPlan; + import org.apache.spark.sql.execution.datasources.v2.V2TableWriteExec; + import org.junit.jupiter.api.AfterEach; + import org.junit.jupiter.api.BeforeEach; +@@ -313,9 +314,13 @@ public class TestSystemFunctionPushDownInRowLevelOperations extends ExtensionsTe + + private List executeAndCollectFunctionCalls(String query, Object... args) { + CommandResultExec command = (CommandResultExec) executeAndKeepPlan(query, args); +- V2TableWriteExec write = (V2TableWriteExec) command.commandPhysicalPlan(); ++ // Comet's split-operator Iceberg write plans the command as IcebergCommitExec, which is not ++ // a V2TableWriteExec; collect over the whole command plan in that case. ++ SparkPlan plan = command.commandPhysicalPlan(); ++ SparkPlan queryPlan = ++ plan instanceof V2TableWriteExec ? ((V2TableWriteExec) plan).query() : plan; + return SparkPlanUtil.collectExprs( +- write.query(), ++ queryPlan, + expr -> expr instanceof StaticInvoke || expr instanceof ApplyFunctionExpression); + } + diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java -index 64edb1002e..4706106c2e 100644 +index 64edb1002e..57767da086 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java -@@ -179,6 +179,14 @@ public class DeleteOrphanFilesBenchmark { +@@ -179,6 +179,15 @@ public class DeleteOrphanFilesBenchmark { .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", catalogWarehouse()) @@ -1678,16 +1768,17 @@ index 64edb1002e..4706106c2e 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local"); spark = builder.getOrCreate(); } diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java -index a5d0456b0b..815de72f4a 100644 +index a5d0456b0b..f52c387943 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java -@@ -392,6 +392,14 @@ public class IcebergSortCompactionBenchmark { +@@ -392,6 +392,15 @@ public class IcebergSortCompactionBenchmark { "spark.sql.catalog.spark_catalog", "org.apache.iceberg.spark.SparkSessionCatalog") .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", getCatalogWarehouse()) @@ -1697,16 +1788,17 @@ index a5d0456b0b..815de72f4a 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[*]"); spark = builder.getOrCreate(); Configuration sparkHadoopConf = spark.sessionState().newHadoopConf(); diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java -index c6794e43c6..ada3a83e3c 100644 +index c6794e43c6..4c1a165a44 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java -@@ -239,6 +239,14 @@ public class DVReaderBenchmark { +@@ -239,6 +239,15 @@ public class DVReaderBenchmark { .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", newWarehouseDir()) @@ -1716,16 +1808,17 @@ index c6794e43c6..ada3a83e3c 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[*]") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java -index ac74fb5a10..bafca20a1c 100644 +index ac74fb5a10..4aa5d5b361 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java -@@ -223,6 +223,14 @@ public class DVWriterBenchmark { +@@ -223,6 +223,15 @@ public class DVWriterBenchmark { .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", newWarehouseDir()) @@ -1735,16 +1828,17 @@ index ac74fb5a10..bafca20a1c 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[*]") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java -index 68c537e34a..5ef031963a 100644 +index 68c537e34a..2e421c7706 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java -@@ -94,7 +94,17 @@ public abstract class IcebergSourceBenchmark { +@@ -94,7 +94,18 @@ public abstract class IcebergSourceBenchmark { } protected void setupSpark(boolean enableDictionaryEncoding) { @@ -1758,6 +1852,7 @@ index 68c537e34a..5ef031963a 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g"); if (!enableDictionaryEncoding) { @@ -2545,10 +2640,10 @@ index 780e1750a5..25f253eede 100644 .filter(residual) .caseSensitive(caseSensitive()) diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java -index 404ba72846..5bd4e2b351 100644 +index 404ba72846..b375458f05 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java -@@ -90,6 +90,14 @@ public abstract class SparkDistributedDataScanTestBase +@@ -90,6 +90,15 @@ public abstract class SparkDistributedDataScanTestBase .master("local[2]") .config("spark.serializer", serializer) .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -2558,16 +2653,17 @@ index 404ba72846..5bd4e2b351 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java -index 659507e4c5..74182efbb2 100644 +index 659507e4c5..2a3a87f079 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java -@@ -73,6 +73,14 @@ public class TestSparkDistributedDataScanDeletes +@@ -73,6 +73,15 @@ public class TestSparkDistributedDataScanDeletes .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -2577,16 +2673,17 @@ index 659507e4c5..74182efbb2 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java -index a218f965ea..b1e0fcf2ca 100644 +index a218f965ea..e30a8d6c6f 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java -@@ -62,6 +62,14 @@ public class TestSparkDistributedDataScanFilterFiles +@@ -62,6 +62,15 @@ public class TestSparkDistributedDataScanFilterFiles .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -2596,16 +2693,17 @@ index a218f965ea..b1e0fcf2ca 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java -index 2665d7ba8d..3539020102 100644 +index 2665d7ba8d..c24c82341a 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java -@@ -63,6 +63,14 @@ public class TestSparkDistributedDataScanReporting +@@ -63,6 +63,15 @@ public class TestSparkDistributedDataScanReporting .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -2615,16 +2713,17 @@ index 2665d7ba8d..3539020102 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java -index 3e9f3334ef..fbc33dc157 100644 +index 3e9f3334ef..0f96c07739 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java -@@ -77,6 +77,14 @@ public abstract class TestBase extends SparkTestHelperBase { +@@ -77,6 +77,15 @@ public abstract class TestBase extends SparkTestHelperBase { .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") @@ -2634,6 +2733,7 @@ index 3e9f3334ef..fbc33dc157 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -2665,10 +2765,10 @@ index a31138ae01..501c8ac93e 100644 } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetDictionaryEncodedVectorizedReads.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetDictionaryEncodedVectorizedReads.java -index bc4e722bc8..c7a417e105 100644 +index bc4e722bc8..57770308b3 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetDictionaryEncodedVectorizedReads.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetDictionaryEncodedVectorizedReads.java -@@ -59,7 +59,18 @@ public class TestParquetDictionaryEncodedVectorizedReads extends TestParquetVect +@@ -59,7 +59,19 @@ public class TestParquetDictionaryEncodedVectorizedReads extends TestParquetVect @BeforeAll public static void startSpark() { @@ -2682,6 +2782,7 @@ index bc4e722bc8..c7a417e105 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2689,10 +2790,10 @@ index bc4e722bc8..c7a417e105 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java -index 0886df957d..320ac2f45e 100644 +index 0886df957d..eca0f74533 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java -@@ -57,7 +57,18 @@ public abstract class ScanTestBase extends AvroDataTest { +@@ -57,7 +57,19 @@ public abstract class ScanTestBase extends AvroDataTest { @BeforeAll public static void startSpark() { @@ -2706,6 +2807,7 @@ index 0886df957d..320ac2f45e 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2713,10 +2815,10 @@ index 0886df957d..320ac2f45e 100644 } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java -index 6b7d861364..18854ba7e8 100644 +index 6b7d861364..23419aa9b1 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java -@@ -144,7 +144,18 @@ public class TestCompressionSettings extends CatalogTestBase { +@@ -144,7 +144,19 @@ public class TestCompressionSettings extends CatalogTestBase { @BeforeAll public static void startSpark() { @@ -2730,6 +2832,7 @@ index 6b7d861364..18854ba7e8 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2737,10 +2840,10 @@ index 6b7d861364..18854ba7e8 100644 @BeforeEach diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java -index c4ba96e634..5faeec110c 100644 +index c4ba96e634..dcf72c19b6 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java -@@ -75,7 +75,18 @@ public class TestDataSourceOptions extends TestBaseWithCatalog { +@@ -75,7 +75,19 @@ public class TestDataSourceOptions extends TestBaseWithCatalog { @BeforeAll public static void startSpark() { @@ -2754,6 +2857,7 @@ index c4ba96e634..5faeec110c 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2761,10 +2865,10 @@ index c4ba96e634..5faeec110c 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java -index 348173596e..828fe716a5 100644 +index 348173596e..03156f786b 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java -@@ -110,7 +110,18 @@ public class TestFilteredScan { +@@ -110,7 +110,19 @@ public class TestFilteredScan { @BeforeAll public static void startSpark() { @@ -2778,6 +2882,7 @@ index 348173596e..828fe716a5 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2785,10 +2890,10 @@ index 348173596e..828fe716a5 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java -index 84c99a575c..79dfc3aff3 100644 +index 84c99a575c..e647ab924e 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java -@@ -93,7 +93,18 @@ public class TestForwardCompatibility { +@@ -93,7 +93,19 @@ public class TestForwardCompatibility { @BeforeAll public static void startSpark() { @@ -2802,6 +2907,7 @@ index 84c99a575c..79dfc3aff3 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2809,10 +2915,10 @@ index 84c99a575c..79dfc3aff3 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java -index 7eff93d204..aead2a6924 100644 +index 7eff93d204..2fdc87a0a8 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java -@@ -46,7 +46,18 @@ public class TestIcebergSpark { +@@ -46,7 +46,19 @@ public class TestIcebergSpark { @BeforeAll public static void startSpark() { @@ -2826,6 +2932,7 @@ index 7eff93d204..aead2a6924 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2833,10 +2940,10 @@ index 7eff93d204..aead2a6924 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java -index 9464f687b0..4850759d61 100644 +index 9464f687b0..8653560334 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java -@@ -112,7 +112,18 @@ public class TestPartitionPruning { +@@ -112,7 +112,19 @@ public class TestPartitionPruning { @BeforeAll public static void startSpark() { @@ -2850,6 +2957,7 @@ index 9464f687b0..4850759d61 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2857,10 +2965,10 @@ index 9464f687b0..4850759d61 100644 String optionKey = String.format("fs.%s.impl", CountOpenLocalFileSystem.scheme); diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java -index 5c218f21c4..dd4aeca9ef 100644 +index 5c218f21c4..2f17343d59 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java -@@ -107,7 +107,18 @@ public class TestPartitionValues { +@@ -107,7 +107,19 @@ public class TestPartitionValues { @BeforeAll public static void startSpark() { @@ -2874,6 +2982,7 @@ index 5c218f21c4..dd4aeca9ef 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2881,10 +2990,10 @@ index 5c218f21c4..dd4aeca9ef 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java -index a7334a580c..f6f034240b 100644 +index a7334a580c..68210dc8e8 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java -@@ -87,7 +87,18 @@ public class TestSnapshotSelection { +@@ -87,7 +87,19 @@ public class TestSnapshotSelection { @BeforeAll public static void startSpark() { @@ -2898,6 +3007,7 @@ index a7334a580c..f6f034240b 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2905,10 +3015,10 @@ index a7334a580c..f6f034240b 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java -index 182b1ef8f5..47e6f0bd01 100644 +index 182b1ef8f5..76e04b1472 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java -@@ -120,7 +120,18 @@ public class TestSparkDataFile { +@@ -120,7 +120,19 @@ public class TestSparkDataFile { @BeforeAll public static void startSpark() { @@ -2922,6 +3032,7 @@ index 182b1ef8f5..47e6f0bd01 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2929,10 +3040,10 @@ index 182b1ef8f5..47e6f0bd01 100644 } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java -index fb2b312bed..25b6170fd3 100644 +index fb2b312bed..846f50e0c9 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java -@@ -96,7 +96,18 @@ public class TestSparkDataWrite { +@@ -96,7 +96,19 @@ public class TestSparkDataWrite { @BeforeAll public static void startSpark() { @@ -2946,13 +3057,14 @@ index fb2b312bed..25b6170fd3 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); } @AfterEach -@@ -140,7 +151,7 @@ public class TestSparkDataWrite { +@@ -140,7 +152,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -2961,7 +3073,7 @@ index fb2b312bed..25b6170fd3 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); for (ManifestFile manifest : -@@ -210,7 +221,7 @@ public class TestSparkDataWrite { +@@ -210,7 +222,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -2970,7 +3082,7 @@ index fb2b312bed..25b6170fd3 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); } -@@ -256,7 +267,7 @@ public class TestSparkDataWrite { +@@ -256,7 +268,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -2979,7 +3091,7 @@ index fb2b312bed..25b6170fd3 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); } -@@ -309,7 +320,7 @@ public class TestSparkDataWrite { +@@ -309,7 +321,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -2988,7 +3100,7 @@ index fb2b312bed..25b6170fd3 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); } -@@ -352,7 +363,7 @@ public class TestSparkDataWrite { +@@ -352,7 +364,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -2997,7 +3109,7 @@ index fb2b312bed..25b6170fd3 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); } -@@ -392,7 +403,7 @@ public class TestSparkDataWrite { +@@ -392,7 +404,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3006,7 +3118,7 @@ index fb2b312bed..25b6170fd3 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); -@@ -458,7 +469,7 @@ public class TestSparkDataWrite { +@@ -458,7 +470,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3015,7 +3127,7 @@ index fb2b312bed..25b6170fd3 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); } -@@ -622,7 +633,7 @@ public class TestSparkDataWrite { +@@ -622,7 +634,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3024,7 +3136,7 @@ index fb2b312bed..25b6170fd3 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); -@@ -708,7 +719,7 @@ public class TestSparkDataWrite { +@@ -708,7 +720,7 @@ public class TestSparkDataWrite { // Since write and commit succeeded, the rows should be readable Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3034,10 +3146,10 @@ index fb2b312bed..25b6170fd3 100644 assertThat(actual) .describedAs("Result rows should match") diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java -index becf6a064d..f00afdf707 100644 +index becf6a064d..91451c268e 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java -@@ -83,7 +83,18 @@ public class TestSparkReadProjection extends TestReadProjection { +@@ -83,7 +83,19 @@ public class TestSparkReadProjection extends TestReadProjection { @BeforeAll public static void startSpark() { @@ -3051,6 +3163,7 @@ index becf6a064d..f00afdf707 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -3058,10 +3171,10 @@ index becf6a064d..f00afdf707 100644 ImmutableMap.of( "type", "hive", diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java -index 4f1cef5d37..e1b1716dcd 100644 +index 4f1cef5d37..4a1a602ed4 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java -@@ -136,6 +136,14 @@ public class TestSparkReaderDeletes extends DeleteReadTests { +@@ -136,6 +136,15 @@ public class TestSparkReaderDeletes extends DeleteReadTests { .config("spark.ui.liveUpdate.period", 0) .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) @@ -3071,12 +3184,13 @@ index 4f1cef5d37..e1b1716dcd 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() .getOrCreate(); -@@ -204,7 +212,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { +@@ -204,7 +213,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { } protected boolean countDeletes() { @@ -3087,10 +3201,10 @@ index 4f1cef5d37..e1b1716dcd 100644 @Override diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java -index baf7fa8f88..5ada29b4af 100644 +index baf7fa8f88..b58321b761 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java -@@ -182,6 +182,14 @@ public class TestSparkReaderWithBloomFilter { +@@ -182,6 +182,15 @@ public class TestSparkReaderWithBloomFilter { SparkSession.builder() .master("local[2]") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) @@ -3100,16 +3214,17 @@ index baf7fa8f88..5ada29b4af 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() .getOrCreate(); diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java -index c84a65cbe9..5644819222 100644 +index c84a65cbe9..e17ba73ee4 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java -@@ -67,6 +67,14 @@ public class TestStructuredStreaming { +@@ -67,6 +67,15 @@ public class TestStructuredStreaming { SparkSession.builder() .master("local[2]") .config("spark.sql.shuffle.partitions", 4) @@ -3119,16 +3234,17 @@ index c84a65cbe9..5644819222 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java -index 306444b9f2..e4c828110e 100644 +index 306444b9f2..e4082d9cd9 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java -@@ -75,7 +75,18 @@ public class TestTimestampWithoutZone extends TestBase { +@@ -75,7 +75,19 @@ public class TestTimestampWithoutZone extends TestBase { @BeforeAll public static void startSpark() { @@ -3142,6 +3258,7 @@ index 306444b9f2..e4c828110e 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -3149,10 +3266,10 @@ index 306444b9f2..e4c828110e 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java -index 841268a6be..ec29f30204 100644 +index 841268a6be..97f7187314 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java -@@ -80,7 +80,18 @@ public class TestWriteMetricsConfig { +@@ -80,7 +80,19 @@ public class TestWriteMetricsConfig { @BeforeAll public static void startSpark() { @@ -3166,6 +3283,7 @@ index 841268a6be..ec29f30204 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -3173,10 +3291,10 @@ index 841268a6be..ec29f30204 100644 } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java -index 6e09252704..2ed54a2e60 100644 +index 6e09252704..4b0f2e4d58 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java -@@ -60,6 +60,14 @@ public class TestAggregatePushDown extends CatalogTestBase { +@@ -60,6 +60,15 @@ public class TestAggregatePushDown extends CatalogTestBase { SparkSession.builder() .master("local[2]") .config("spark.sql.iceberg.aggregate_pushdown", "true") @@ -3186,6 +3304,7 @@ index 6e09252704..2ed54a2e60 100644 + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -3219,59 +3338,3 @@ index 6719c45ca9..2515454401 100644 sourceColumnName, tableName, tableName(OTHER_TABLE_NAME), -diff --git a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java -index 5a0a04edb..afe92a8b7 100644 ---- a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java -+++ b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java -@@ -38,6 +38,7 @@ import org.apache.spark.sql.catalyst.expressions.ApplyFunctionExpression; - import org.apache.spark.sql.catalyst.expressions.Expression; - import org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke; - import org.apache.spark.sql.execution.CommandResultExec; -+import org.apache.spark.sql.execution.SparkPlan; - import org.apache.spark.sql.execution.datasources.v2.V2TableWriteExec; - import org.junit.jupiter.api.AfterEach; - import org.junit.jupiter.api.BeforeEach; -@@ -313,9 +314,13 @@ public class TestSystemFunctionPushDownInRowLevelOperations extends ExtensionsTe - - private List executeAndCollectFunctionCalls(String query, Object... args) { - CommandResultExec command = (CommandResultExec) executeAndKeepPlan(query, args); -- V2TableWriteExec write = (V2TableWriteExec) command.commandPhysicalPlan(); -+ // Comet's split-operator Iceberg write plans the command as IcebergCommitExec, which is not -+ // a V2TableWriteExec; collect over the whole command plan in that case. -+ SparkPlan plan = command.commandPhysicalPlan(); -+ SparkPlan queryPlan = -+ plan instanceof V2TableWriteExec ? ((V2TableWriteExec) plan).query() : plan; - return SparkPlanUtil.collectExprs( -- write.query(), -+ queryPlan, - expr -> expr instanceof StaticInvoke || expr instanceof ApplyFunctionExpression); - } - -diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java -index 8cb92224f..777b9bb1d 100644 ---- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java -+++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java -@@ -38,6 +38,7 @@ import org.apache.spark.sql.catalyst.expressions.ApplyFunctionExpression; - import org.apache.spark.sql.catalyst.expressions.Expression; - import org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke; - import org.apache.spark.sql.execution.CommandResultExec; -+import org.apache.spark.sql.execution.SparkPlan; - import org.apache.spark.sql.execution.datasources.v2.V2TableWriteExec; - import org.junit.jupiter.api.AfterEach; - import org.junit.jupiter.api.BeforeEach; -@@ -313,9 +314,13 @@ public class TestSystemFunctionPushDownInRowLevelOperations extends ExtensionsTe - - private List executeAndCollectFunctionCalls(String query, Object... args) { - CommandResultExec command = (CommandResultExec) executeAndKeepPlan(query, args); -- V2TableWriteExec write = (V2TableWriteExec) command.commandPhysicalPlan(); -+ // Comet's split-operator Iceberg write plans the command as IcebergCommitExec, which is not -+ // a V2TableWriteExec; collect over the whole command plan in that case. -+ SparkPlan plan = command.commandPhysicalPlan(); -+ SparkPlan queryPlan = -+ plan instanceof V2TableWriteExec ? ((V2TableWriteExec) plan).query() : plan; - return SparkPlanUtil.collectExprs( -- write.query(), -+ queryPlan, - expr -> expr instanceof StaticInvoke || expr instanceof ApplyFunctionExpression); - } - diff --git a/docs/source/contributor-guide/iceberg-spark-tests.md b/docs/source/contributor-guide/iceberg-spark-tests.md index 6bfe57a4aa5..df6c168af6a 100644 --- a/docs/source/contributor-guide/iceberg-spark-tests.md +++ b/docs/source/contributor-guide/iceberg-spark-tests.md @@ -31,8 +31,13 @@ Here is an overview of the changes that the diffs make to Iceberg: uses a native Iceberg scan, these classes fail to compile and must be removed. - Configure test base classes (`TestBase`, `ExtensionsTestBase`, `ScanTestBase`, etc.) to load the Comet Spark plugin and shuffle manager +- Enable the Iceberg write split-operator plan (`spark.comet.write.iceberg.splitOperator.enabled`) alongside the + native scan in every Comet-configured session. The flag is off by default for users, so Iceberg's own suites + are the only place the split plan (`IcebergCommit -> IcebergWrite`) is exercised against Iceberg's write, + commit, and row-level-operation tests. See [#5259] [#3739]: https://github.com/apache/datafusion-comet/pull/3739 +[#5259]: https://github.com/apache/datafusion-comet/issues/5259 [apache/iceberg#15674]: https://github.com/apache/iceberg/pull/15674 ## 1. Install Comet