From a223c508e29304a2799642296ac55cde5834f176 Mon Sep 17 00:00:00 2001 From: Hongyue Zhang Date: Mon, 17 Aug 2026 09:30:27 -0700 Subject: [PATCH] Data: Remove deprecated GenericAppenderFactory and BaseFileWriterFactory Both were deprecated for removal in 1.12.0: - GenericAppenderFactory -> GenericFileWriterFactory - BaseFileWriterFactory -> RegistryBasedFileWriterFactory Neither has any remaining reference in main or test sources across all modules. TestGenericAppenderFactory is removed with the class it covered; GenericFileWriterFactory is covered by TestGenericFileWriterFactory. --- .palantir/revapi.yml | 9 + .../iceberg/data/BaseFileWriterFactory.java | 374 ------------------ .../iceberg/data/GenericAppenderFactory.java | 347 ---------------- .../iceberg/TestGenericAppenderFactory.java | 139 ------- 4 files changed, 9 insertions(+), 860 deletions(-) delete mode 100644 data/src/main/java/org/apache/iceberg/data/BaseFileWriterFactory.java delete mode 100644 data/src/main/java/org/apache/iceberg/data/GenericAppenderFactory.java delete mode 100644 data/src/test/java/org/apache/iceberg/TestGenericAppenderFactory.java diff --git a/.palantir/revapi.yml b/.palantir/revapi.yml index fd9bb06dd744..66430a5f0a5c 100644 --- a/.palantir/revapi.yml +++ b/.palantir/revapi.yml @@ -621,6 +621,15 @@ acceptedBreaks: \ org.apache.iceberg.PartitionStatsHandler::readPartitionStatsFile(org.apache.iceberg.Schema,\ \ org.apache.iceberg.io.InputFile)" justification: "Removed deprecated functionality for partition stats" + org.apache.iceberg:iceberg-data: + - code: "java.class.removed" + old: "class org.apache.iceberg.data.BaseFileWriterFactory" + justification: "Removing deprecated GenericAppenderFactory and BaseFileWriterFactory\ + \ scheduled for removal in 1.12.0" + - code: "java.class.removed" + old: "class org.apache.iceberg.data.GenericAppenderFactory" + justification: "Removing deprecated GenericAppenderFactory and BaseFileWriterFactory\ + \ scheduled for removal in 1.12.0" "1.2.0": org.apache.iceberg:iceberg-api: - code: "java.field.constantValueChanged" diff --git a/data/src/main/java/org/apache/iceberg/data/BaseFileWriterFactory.java b/data/src/main/java/org/apache/iceberg/data/BaseFileWriterFactory.java deleted file mode 100644 index f6fbf1ca6b24..000000000000 --- a/data/src/main/java/org/apache/iceberg/data/BaseFileWriterFactory.java +++ /dev/null @@ -1,374 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iceberg.data; - -import java.io.IOException; -import java.io.Serializable; -import java.io.UncheckedIOException; -import java.util.Map; -import org.apache.iceberg.FileFormat; -import org.apache.iceberg.MetricsConfig; -import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.Schema; -import org.apache.iceberg.SortOrder; -import org.apache.iceberg.StructLike; -import org.apache.iceberg.Table; -import org.apache.iceberg.avro.Avro; -import org.apache.iceberg.deletes.EqualityDeleteWriter; -import org.apache.iceberg.deletes.PositionDeleteWriter; -import org.apache.iceberg.encryption.EncryptedOutputFile; -import org.apache.iceberg.encryption.EncryptionKeyMetadata; -import org.apache.iceberg.io.DataWriter; -import org.apache.iceberg.io.FileWriterFactory; -import org.apache.iceberg.orc.ORC; -import org.apache.iceberg.parquet.Parquet; -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; - -/** - * A base writer factory to be extended by query engine integrations. - * - * @deprecated since version 1.11.0 and will be removed in 1.12.0. Use {@link - * RegistryBasedFileWriterFactory} - */ -@Deprecated -public abstract class BaseFileWriterFactory implements FileWriterFactory, Serializable { - private final Table table; - private final FileFormat dataFileFormat; - private final Schema dataSchema; - private final SortOrder dataSortOrder; - private final FileFormat deleteFileFormat; - private final int[] equalityFieldIds; - private final Schema equalityDeleteRowSchema; - private final SortOrder equalityDeleteSortOrder; - private final Schema positionDeleteRowSchema; - private final Map writerProperties; - - protected BaseFileWriterFactory( - Table table, - FileFormat dataFileFormat, - Schema dataSchema, - SortOrder dataSortOrder, - FileFormat deleteFileFormat, - int[] equalityFieldIds, - Schema equalityDeleteRowSchema, - SortOrder equalityDeleteSortOrder, - Map writerProperties) { - this.table = table; - this.dataFileFormat = dataFileFormat; - this.dataSchema = dataSchema; - this.dataSortOrder = dataSortOrder; - this.deleteFileFormat = deleteFileFormat; - this.equalityFieldIds = equalityFieldIds; - this.equalityDeleteRowSchema = equalityDeleteRowSchema; - this.equalityDeleteSortOrder = equalityDeleteSortOrder; - this.writerProperties = writerProperties; - this.positionDeleteRowSchema = null; - } - - protected BaseFileWriterFactory( - Table table, - FileFormat dataFileFormat, - Schema dataSchema, - SortOrder dataSortOrder, - FileFormat deleteFileFormat, - int[] equalityFieldIds, - Schema equalityDeleteRowSchema, - SortOrder equalityDeleteSortOrder, - Schema positionDeleteRowSchema, - Map writerProperties) { - this.table = table; - this.dataFileFormat = dataFileFormat; - this.dataSchema = dataSchema; - this.dataSortOrder = dataSortOrder; - this.deleteFileFormat = deleteFileFormat; - this.equalityFieldIds = equalityFieldIds; - this.equalityDeleteRowSchema = equalityDeleteRowSchema; - this.equalityDeleteSortOrder = equalityDeleteSortOrder; - this.positionDeleteRowSchema = positionDeleteRowSchema; - this.writerProperties = writerProperties; - } - - @Deprecated - protected BaseFileWriterFactory( - Table table, - FileFormat dataFileFormat, - Schema dataSchema, - SortOrder dataSortOrder, - FileFormat deleteFileFormat, - int[] equalityFieldIds, - Schema equalityDeleteRowSchema, - SortOrder equalityDeleteSortOrder, - Schema positionDeleteRowSchema) { - this.table = table; - this.dataFileFormat = dataFileFormat; - this.dataSchema = dataSchema; - this.dataSortOrder = dataSortOrder; - this.deleteFileFormat = deleteFileFormat; - this.equalityFieldIds = equalityFieldIds; - this.equalityDeleteRowSchema = equalityDeleteRowSchema; - this.equalityDeleteSortOrder = equalityDeleteSortOrder; - this.positionDeleteRowSchema = positionDeleteRowSchema; - this.writerProperties = ImmutableMap.of(); - } - - protected abstract void configureDataWrite(Avro.DataWriteBuilder builder); - - protected abstract void configureEqualityDelete(Avro.DeleteWriteBuilder builder); - - protected abstract void configurePositionDelete(Avro.DeleteWriteBuilder builder); - - protected abstract void configureDataWrite(Parquet.DataWriteBuilder builder); - - protected abstract void configureEqualityDelete(Parquet.DeleteWriteBuilder builder); - - protected abstract void configurePositionDelete(Parquet.DeleteWriteBuilder builder); - - protected abstract void configureDataWrite(ORC.DataWriteBuilder builder); - - protected abstract void configureEqualityDelete(ORC.DeleteWriteBuilder builder); - - protected abstract void configurePositionDelete(ORC.DeleteWriteBuilder builder); - - @Override - public DataWriter newDataWriter( - EncryptedOutputFile file, PartitionSpec spec, StructLike partition) { - EncryptionKeyMetadata keyMetadata = file.keyMetadata(); - Map properties = table == null ? ImmutableMap.of() : table.properties(); - MetricsConfig metricsConfig = - table == null ? MetricsConfig.getDefault() : MetricsConfig.forTable(table); - - try { - switch (dataFileFormat) { - case AVRO: - Avro.DataWriteBuilder avroBuilder = - Avro.writeData(file) - .schema(dataSchema) - .setAll(properties) - .setAll(writerProperties) - .metricsConfig(metricsConfig) - .withSpec(spec) - .withPartition(partition) - .withKeyMetadata(keyMetadata) - .withSortOrder(dataSortOrder) - .overwrite(); - - configureDataWrite(avroBuilder); - - return avroBuilder.build(); - - case PARQUET: - Parquet.DataWriteBuilder parquetBuilder = - Parquet.writeData(file) - .schema(dataSchema) - .setAll(properties) - .setAll(writerProperties) - .metricsConfig(metricsConfig) - .withSpec(spec) - .withPartition(partition) - .withKeyMetadata(keyMetadata) - .withSortOrder(dataSortOrder) - .overwrite(); - - configureDataWrite(parquetBuilder); - - return parquetBuilder.build(); - - case ORC: - ORC.DataWriteBuilder orcBuilder = - ORC.writeData(file) - .schema(dataSchema) - .setAll(properties) - .setAll(writerProperties) - .metricsConfig(metricsConfig) - .withSpec(spec) - .withPartition(partition) - .withKeyMetadata(keyMetadata) - .withSortOrder(dataSortOrder) - .overwrite(); - - configureDataWrite(orcBuilder); - - return orcBuilder.build(); - - default: - throw new UnsupportedOperationException( - "Unsupported data file format: " + dataFileFormat); - } - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - @Override - public EqualityDeleteWriter newEqualityDeleteWriter( - EncryptedOutputFile file, PartitionSpec spec, StructLike partition) { - EncryptionKeyMetadata keyMetadata = file.keyMetadata(); - Map properties = table == null ? ImmutableMap.of() : table.properties(); - MetricsConfig metricsConfig = - table == null ? MetricsConfig.getDefault() : MetricsConfig.forTable(table); - - try { - switch (deleteFileFormat) { - case AVRO: - Avro.DeleteWriteBuilder avroBuilder = - Avro.writeDeletes(file) - .setAll(properties) - .setAll(writerProperties) - .metricsConfig(metricsConfig) - .rowSchema(equalityDeleteRowSchema) - .equalityFieldIds(equalityFieldIds) - .withSpec(spec) - .withPartition(partition) - .withKeyMetadata(keyMetadata) - .withSortOrder(equalityDeleteSortOrder) - .overwrite(); - - configureEqualityDelete(avroBuilder); - - return avroBuilder.buildEqualityWriter(); - - case PARQUET: - Parquet.DeleteWriteBuilder parquetBuilder = - Parquet.writeDeletes(file) - .setAll(properties) - .setAll(writerProperties) - .metricsConfig(metricsConfig) - .rowSchema(equalityDeleteRowSchema) - .equalityFieldIds(equalityFieldIds) - .withSpec(spec) - .withPartition(partition) - .withKeyMetadata(keyMetadata) - .withSortOrder(equalityDeleteSortOrder) - .overwrite(); - - configureEqualityDelete(parquetBuilder); - - return parquetBuilder.buildEqualityWriter(); - - case ORC: - ORC.DeleteWriteBuilder orcBuilder = - ORC.writeDeletes(file) - .setAll(properties) - .setAll(writerProperties) - .metricsConfig(metricsConfig) - .rowSchema(equalityDeleteRowSchema) - .equalityFieldIds(equalityFieldIds) - .withSpec(spec) - .withPartition(partition) - .withKeyMetadata(keyMetadata) - .withSortOrder(equalityDeleteSortOrder) - .overwrite(); - - configureEqualityDelete(orcBuilder); - - return orcBuilder.buildEqualityWriter(); - - default: - throw new UnsupportedOperationException( - "Unsupported format for equality deletes: " + deleteFileFormat); - } - } catch (IOException e) { - throw new UncheckedIOException("Failed to create new equality delete writer", e); - } - } - - @Override - public PositionDeleteWriter newPositionDeleteWriter( - EncryptedOutputFile file, PartitionSpec spec, StructLike partition) { - EncryptionKeyMetadata keyMetadata = file.keyMetadata(); - Map properties = table == null ? ImmutableMap.of() : table.properties(); - MetricsConfig metricsConfig = MetricsConfig.forPositionDelete(); - - try { - switch (deleteFileFormat) { - case AVRO: - Avro.DeleteWriteBuilder avroBuilder = - Avro.writeDeletes(file) - .setAll(properties) - .setAll(writerProperties) - .metricsConfig(metricsConfig) - .rowSchema(positionDeleteRowSchema) - .withSpec(spec) - .withPartition(partition) - .withKeyMetadata(keyMetadata) - .overwrite(); - - configurePositionDelete(avroBuilder); - - return avroBuilder.buildPositionWriter(); - - case PARQUET: - Parquet.DeleteWriteBuilder parquetBuilder = - Parquet.writeDeletes(file) - .setAll(properties) - .setAll(writerProperties) - .metricsConfig(metricsConfig) - .rowSchema(positionDeleteRowSchema) - .withSpec(spec) - .withPartition(partition) - .withKeyMetadata(keyMetadata) - .overwrite(); - - configurePositionDelete(parquetBuilder); - - return parquetBuilder.buildPositionWriter(); - - case ORC: - ORC.DeleteWriteBuilder orcBuilder = - ORC.writeDeletes(file) - .setAll(properties) - .setAll(writerProperties) - .metricsConfig(metricsConfig) - .rowSchema(positionDeleteRowSchema) - .withSpec(spec) - .withPartition(partition) - .withKeyMetadata(keyMetadata) - .overwrite(); - - configurePositionDelete(orcBuilder); - - return orcBuilder.buildPositionWriter(); - - default: - throw new UnsupportedOperationException( - "Unsupported format for position deletes: " + deleteFileFormat); - } - - } catch (IOException e) { - throw new UncheckedIOException("Failed to create new position delete writer", e); - } - } - - protected Schema dataSchema() { - return dataSchema; - } - - protected Schema equalityDeleteRowSchema() { - return equalityDeleteRowSchema; - } - - /** - * @deprecated This method is deprecated as of version 1.11.0 and will be removed in 1.12.0. - * Position deletes that include row data are no longer supported. - */ - @Deprecated - protected Schema positionDeleteRowSchema() { - return positionDeleteRowSchema; - } -} diff --git a/data/src/main/java/org/apache/iceberg/data/GenericAppenderFactory.java b/data/src/main/java/org/apache/iceberg/data/GenericAppenderFactory.java deleted file mode 100644 index a4c62dc3f8f6..000000000000 --- a/data/src/main/java/org/apache/iceberg/data/GenericAppenderFactory.java +++ /dev/null @@ -1,347 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iceberg.data; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.util.Map; -import org.apache.iceberg.FileFormat; -import org.apache.iceberg.MetricsConfig; -import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.Schema; -import org.apache.iceberg.StructLike; -import org.apache.iceberg.Table; -import org.apache.iceberg.avro.Avro; -import org.apache.iceberg.data.avro.DataWriter; -import org.apache.iceberg.data.orc.GenericOrcWriter; -import org.apache.iceberg.data.parquet.GenericParquetWriter; -import org.apache.iceberg.deletes.EqualityDeleteWriter; -import org.apache.iceberg.deletes.PositionDeleteWriter; -import org.apache.iceberg.encryption.EncryptedOutputFile; -import org.apache.iceberg.encryption.EncryptionUtil; -import org.apache.iceberg.io.FileAppender; -import org.apache.iceberg.io.FileAppenderFactory; -import org.apache.iceberg.io.OutputFile; -import org.apache.iceberg.orc.ORC; -import org.apache.iceberg.parquet.Parquet; -import org.apache.iceberg.relocated.com.google.common.base.Preconditions; -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; -import org.apache.iceberg.relocated.com.google.common.collect.Maps; - -/** - * Factory to create a new {@link FileAppender} to write {@link Record}s. - * - * @deprecated will be removed in 1.12.0; use {@link GenericFileWriterFactory} instead. - */ -@Deprecated -public class GenericAppenderFactory implements FileAppenderFactory { - private final Table table; - private final Schema schema; - private final PartitionSpec spec; - private final int[] equalityFieldIds; - private final Schema eqDeleteRowSchema; - private final Schema posDeleteRowSchema; - private final Map config; - - public GenericAppenderFactory(Schema schema) { - this(schema, PartitionSpec.unpartitioned()); - } - - public GenericAppenderFactory(Schema schema, PartitionSpec spec) { - this(schema, spec, null, null, null); - } - - public GenericAppenderFactory( - Schema schema, - PartitionSpec spec, - int[] equalityFieldIds, - Schema eqDeleteRowSchema, - Schema posDeleteRowSchema) { - this(null, schema, spec, null, equalityFieldIds, eqDeleteRowSchema, posDeleteRowSchema); - } - - /** - * Constructor for GenericAppenderFactory. - * - * @param schema the schema of the records to write - * @param spec the partition spec of the records - * @param equalityFieldIds the field ids for equality delete - * @param eqDeleteRowSchema the schema for equality delete rows - */ - public GenericAppenderFactory( - Schema schema, PartitionSpec spec, int[] equalityFieldIds, Schema eqDeleteRowSchema) { - this(null, schema, spec, null, equalityFieldIds, eqDeleteRowSchema, null); - } - - /** - * Constructor for GenericAppenderFactory. - * - * @param table iceberg table - * @param schema the schema of the records to write - * @param spec the partition spec of the records - * @param config the configuration for the writer - * @param equalityFieldIds the field ids for equality delete - * @param eqDeleteRowSchema the schema for equality delete rows - */ - public GenericAppenderFactory( - Table table, - Schema schema, - PartitionSpec spec, - Map config, - int[] equalityFieldIds, - Schema eqDeleteRowSchema) { - this(table, schema, spec, config, equalityFieldIds, eqDeleteRowSchema, null); - } - - /** - * Constructor for GenericAppenderFactory. - * - * @param table iceberg table - * @param schema the schema of the records to write - * @param spec the partition spec of the records - * @param config the configuration for the writer - * @param equalityFieldIds the field ids for equality delete - * @param eqDeleteRowSchema the schema for equality delete rows - * @param posDeleteRowSchema the schema for position delete rows - * @deprecated This constructor is deprecated as of version 1.11.0 and will be removed in 1.12.0. - * Position deletes that include row data are no longer supported. Use {@link - * #GenericAppenderFactory(Table, Schema, PartitionSpec, Map, int[], Schema)} instead. - */ - @Deprecated - public GenericAppenderFactory( - Table table, - Schema schema, - PartitionSpec spec, - Map config, - int[] equalityFieldIds, - Schema eqDeleteRowSchema, - Schema posDeleteRowSchema) { - this.table = table; - this.config = config == null ? Maps.newHashMap() : config; - - if (table != null) { - // If the table is provided and schema and spec are not provided, derive them from the table - this.schema = schema == null ? table.schema() : schema; - this.spec = spec == null ? table.spec() : spec; - validateMetricsConfig(this.config); - } else { - this.schema = schema; - this.spec = spec; - } - - this.equalityFieldIds = equalityFieldIds; - this.eqDeleteRowSchema = eqDeleteRowSchema; - this.posDeleteRowSchema = posDeleteRowSchema; - } - - public GenericAppenderFactory set(String property, String value) { - validateMetricsConfig(ImmutableMap.of(property, value)); - config.put(property, value); - return this; - } - - public GenericAppenderFactory setAll(Map properties) { - validateMetricsConfig(properties); - config.putAll(properties); - return this; - } - - @Override - public FileAppender newAppender(OutputFile outputFile, FileFormat fileFormat) { - return newAppender(EncryptionUtil.plainAsEncryptedOutput(outputFile), fileFormat); - } - - @Override - public FileAppender newAppender( - EncryptedOutputFile encryptedOutputFile, FileFormat fileFormat) { - MetricsConfig metricsConfig = - table != null ? MetricsConfig.forTable(table) : MetricsConfig.from(config, null, null); - - try { - switch (fileFormat) { - case AVRO: - return Avro.write(encryptedOutputFile) - .schema(schema) - .createWriterFunc(DataWriter::create) - .metricsConfig(metricsConfig) - .setAll(config) - .overwrite() - .build(); - - case PARQUET: - return Parquet.write(encryptedOutputFile) - .schema(schema) - .createWriterFunc(GenericParquetWriter::create) - .setAll(config) - .metricsConfig(metricsConfig) - .overwrite() - .build(); - - case ORC: - return ORC.write(encryptedOutputFile) - .schema(schema) - .createWriterFunc(GenericOrcWriter::buildWriter) - .setAll(config) - .metricsConfig(metricsConfig) - .overwrite() - .build(); - - default: - throw new UnsupportedOperationException( - "Cannot write unknown file format: " + fileFormat); - } - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - @Override - public org.apache.iceberg.io.DataWriter newDataWriter( - EncryptedOutputFile file, FileFormat format, StructLike partition) { - return new org.apache.iceberg.io.DataWriter<>( - newAppender(file, format), - format, - file.encryptingOutputFile().location(), - spec, - partition, - file.keyMetadata()); - } - - @Override - public EqualityDeleteWriter newEqDeleteWriter( - EncryptedOutputFile file, FileFormat format, StructLike partition) { - Preconditions.checkState( - equalityFieldIds != null && equalityFieldIds.length > 0, - "Equality field ids shouldn't be null or empty when creating equality-delete writer"); - Preconditions.checkNotNull( - eqDeleteRowSchema, - "Equality delete row schema shouldn't be null when creating equality-delete writer"); - MetricsConfig metricsConfig = - table != null ? MetricsConfig.forTable(table) : MetricsConfig.from(config, null, null); - - try { - switch (format) { - case AVRO: - return Avro.writeDeletes(file) - .createWriterFunc(DataWriter::create) - .withPartition(partition) - .overwrite() - .setAll(config) - .rowSchema(eqDeleteRowSchema) - .withSpec(spec) - .withKeyMetadata(file.keyMetadata()) - .equalityFieldIds(equalityFieldIds) - .buildEqualityWriter(); - - case ORC: - return ORC.writeDeletes(file) - .createWriterFunc(GenericOrcWriter::buildWriter) - .withPartition(partition) - .overwrite() - .setAll(config) - .metricsConfig(metricsConfig) - .rowSchema(eqDeleteRowSchema) - .withSpec(spec) - .withKeyMetadata(file.keyMetadata()) - .equalityFieldIds(equalityFieldIds) - .buildEqualityWriter(); - - case PARQUET: - return Parquet.writeDeletes(file) - .createWriterFunc(GenericParquetWriter::create) - .withPartition(partition) - .overwrite() - .setAll(config) - .metricsConfig(metricsConfig) - .rowSchema(eqDeleteRowSchema) - .withSpec(spec) - .withKeyMetadata(file.keyMetadata()) - .equalityFieldIds(equalityFieldIds) - .buildEqualityWriter(); - - default: - throw new UnsupportedOperationException( - "Cannot write equality-deletes for unsupported file format: " + format); - } - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - @Override - public PositionDeleteWriter newPosDeleteWriter( - EncryptedOutputFile file, FileFormat format, StructLike partition) { - MetricsConfig metricsConfig = MetricsConfig.forPositionDelete(); - - try { - switch (format) { - case AVRO: - return Avro.writeDeletes(file) - .createWriterFunc(DataWriter::create) - .withPartition(partition) - .overwrite() - .setAll(config) - .rowSchema(posDeleteRowSchema) - .withSpec(spec) - .withKeyMetadata(file.keyMetadata()) - .buildPositionWriter(); - - case ORC: - return ORC.writeDeletes(file) - .createWriterFunc(GenericOrcWriter::buildWriter) - .withPartition(partition) - .overwrite() - .setAll(config) - .rowSchema(posDeleteRowSchema) - .withSpec(spec) - .withKeyMetadata(file.keyMetadata()) - .buildPositionWriter(); - - case PARQUET: - return Parquet.writeDeletes(file) - .createWriterFunc(GenericParquetWriter::create) - .withPartition(partition) - .overwrite() - .setAll(config) - .metricsConfig(metricsConfig) - .rowSchema(posDeleteRowSchema) - .withSpec(spec) - .withKeyMetadata(file.keyMetadata()) - .buildPositionWriter(); - - default: - throw new UnsupportedOperationException( - "Cannot write pos-deletes for unsupported file format: " + format); - } - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - private void validateMetricsConfig(Map writeConfig) { - if (table == null) { - return; - } - - if (writeConfig.keySet().stream().anyMatch(k -> k.startsWith("write.metadata.metrics."))) { - throw new IllegalArgumentException( - "Cannot set metrics properties when the table is provided, use table properties instead"); - } - } -} diff --git a/data/src/test/java/org/apache/iceberg/TestGenericAppenderFactory.java b/data/src/test/java/org/apache/iceberg/TestGenericAppenderFactory.java deleted file mode 100644 index 5d940adaec58..000000000000 --- a/data/src/test/java/org/apache/iceberg/TestGenericAppenderFactory.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iceberg; - -import static org.assertj.core.api.Assertions.assertThatNoException; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -import java.util.List; -import java.util.Map; -import org.apache.iceberg.data.GenericAppenderFactory; -import org.apache.iceberg.data.GenericRecord; -import org.apache.iceberg.data.Record; -import org.apache.iceberg.io.FileAppenderFactory; -import org.apache.iceberg.io.TestAppenderFactory; -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; -import org.apache.iceberg.relocated.com.google.common.collect.Maps; -import org.apache.iceberg.util.ArrayUtil; -import org.apache.iceberg.util.StructLikeSet; -import org.junit.jupiter.api.TestTemplate; - -public class TestGenericAppenderFactory extends TestAppenderFactory { - - private final GenericRecord gRecord = GenericRecord.create(SCHEMA); - - @Override - protected FileAppenderFactory createAppenderFactory( - List equalityFieldIds, Schema eqDeleteSchema, Schema posDeleteRowSchema) { - return new GenericAppenderFactory( - table, - table.schema(), - table.spec(), - Maps.newHashMap(), - ArrayUtil.toIntArray(equalityFieldIds), - eqDeleteSchema, - posDeleteRowSchema); - } - - @Override - protected Record createRow(Integer id, String data) { - return gRecord.copy(ImmutableMap.of("id", id, "data", data)); - } - - @Override - protected StructLikeSet expectedRowSet(Iterable records) { - StructLikeSet set = StructLikeSet.create(table.schema().asStruct()); - records.forEach(set::add); - return set; - } - - @TestTemplate - void illegalSetConfig() { - GenericAppenderFactory appenderFactory = - (GenericAppenderFactory) createAppenderFactory(null, null, null); - - assertThatThrownBy( - () -> - appenderFactory.set( - TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS, - MetricsModes.None.get().toString())) - .as("Should not allow setting metrics property if the table was provided") - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining( - "Cannot set metrics properties when the table is provided, use table properties instead"); - } - - @TestTemplate - void illegalSetAllConfigs() { - GenericAppenderFactory appenderFactory = - (GenericAppenderFactory) createAppenderFactory(null, null, null); - - Map properties = - ImmutableMap.of( - TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS, - "10", - TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "id", - MetricsModes.Full.get().toString()); - - assertThatThrownBy(() -> appenderFactory.setAll(properties)) - .as("Should not allow setting metrics property if the table was provided") - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining( - "Cannot set metrics properties when the table is provided, use table properties instead"); - } - - @TestTemplate - void setConfigExcludeMetrics() { - GenericAppenderFactory appenderFactory = - (GenericAppenderFactory) createAppenderFactory(null, null, null); - assertThatNoException().isThrownBy(() -> appenderFactory.set("key1", "value1")); - assertThatNoException() - .isThrownBy(() -> appenderFactory.setAll(ImmutableMap.of("key2", "value2"))); - } - - @TestTemplate - void setConfigWithoutTable() { - GenericAppenderFactory appenderFactory = new GenericAppenderFactory(SCHEMA); - assertThatNoException() - .isThrownBy( - () -> appenderFactory.set(TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS, "10")); - assertThatNoException() - .isThrownBy( - () -> - appenderFactory.setAll( - ImmutableMap.of(TableProperties.DEFAULT_WRITE_METRICS_MODE, "full"))); - } - - @TestTemplate - void createFactoryWithConflictConfig() { - table - .updateProperties() - .set(TableProperties.DEFAULT_WRITE_METRICS_MODE, MetricsModes.Full.get().toString()) - .commit(); - Map config = - ImmutableMap.of( - TableProperties.DEFAULT_WRITE_METRICS_MODE, MetricsModes.None.get().toString()); - - assertThatThrownBy( - () -> new GenericAppenderFactory(table, SCHEMA, SPEC, config, null, null, null)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining( - "Cannot set metrics properties when the table is provided, use table properties instead"); - } -}