From ebaeca08d324b2e0017c91458f6f4c70b4584f54 Mon Sep 17 00:00:00 2001 From: luoyuxia Date: Thu, 27 Aug 2026 07:19:58 +0800 Subject: [PATCH 1/8] [client][server][paimon] Support historical partition writes Route writes for expired partitions through internal historical targets while preserving original partition metadata across PUT_KV and PRODUCE_LOG. Tier historical KV and log records back to their original Paimon partitions, fail writes to confirmed missing targets, and safely clean fully tiered historical KV overlays with leader-epoch and offset guards. Co-Authored-By: Codex AI-Model: gpt-5 AI-Contributed/Feature: 1469/1469 AI-Contributed/UT: 2215/2215 --- .../client/utils/ClientRpcMessageUtils.java | 9 + .../write/AbstractRowLogWriteBatch.java | 12 +- .../client/write/ArrowLogWriteBatch.java | 10 +- .../client/write/CompactedLogWriteBatch.java | 3 + .../client/write/IndexedLogWriteBatch.java | 3 + .../fluss/client/write/KvWriteBatch.java | 2 + .../fluss/client/write/RecordAccumulator.java | 128 ++++- .../org/apache/fluss/client/write/Sender.java | 189 ++++++-- .../apache/fluss/client/write/WriteBatch.java | 15 + .../fluss/client/write/WriterClient.java | 81 +++- .../utils/ClientRpcMessageUtilsTest.java | 1 + .../client/write/ArrowLogWriteBatchTest.java | 3 + .../write/CompactedLogWriteBatchTest.java | 1 + .../write/IndexedLogWriteBatchTest.java | 1 + .../fluss/client/write/KvWriteBatchTest.java | 2 + .../client/write/RecordAccumulatorTest.java | 88 ++++ .../apache/fluss/client/write/SenderTest.java | 453 +++++++++++++++++- .../apache/fluss/config/ConfigOptions.java | 17 +- .../org/apache/fluss/config/TableConfig.java | 2 +- .../enumerator/TieringSourceEnumerator.java | 2 +- .../source/split/TieringSplitGenerator.java | 44 +- .../paimon/tiering/PaimonLakeCommitter.java | 5 +- .../lake/paimon/tiering/PaimonLakeWriter.java | 14 +- .../paimon/tiering/PaimonWriteResult.java | 22 +- .../tiering/PaimonWriteResultSerializer.java | 17 +- .../lake/paimon/tiering/RecordWriter.java | 40 +- .../append/AppendOnlyArrowBatchHelper.java | 30 +- .../tiering/append/AppendOnlyWriter.java | 13 +- .../tiering/mergetree/MergeTreeWriter.java | 20 +- ...se.java => HistoricalPartitionITCase.java} | 190 +++++++- .../paimon/tiering/PaimonTieringTest.java | 175 +++++++ .../rpc/entity/ProduceLogResultForBucket.java | 49 +- .../rpc/netty/client/ServerConnection.java | 41 ++ .../apache/fluss/rpc/protocol/ApiKeys.java | 3 +- .../fluss/rpc/util/CommonRpcMessageUtils.java | 12 + fluss-rpc/src/main/proto/FlussApi.proto | 4 + .../netty/client/ServerConnectionTest.java | 118 ++++- fluss-rust/crates/fluss/proto/FlussApi.proto | 4 + fluss-rust/crates/fluss/src/proto/fluss.rs | 6 + fluss-rust/crates/fluss/src/rpc/api_key.rs | 3 +- .../fluss/src/rpc/message/produce_log.rs | 1 + .../crates/fluss/src/rpc/server_connection.rs | 4 +- .../fluss/server/DynamicServerConfig.java | 2 + .../HistoricalLookupCacheConfigValidator.java | 11 +- .../entity/ProduceLogDataForBucket.java | 51 ++ .../apache/fluss/server/replica/Replica.java | 105 +++- .../fluss/server/replica/ReplicaManager.java | 85 +++- .../HistoricalPartitionManager.java | 272 ++++++++++- .../HistoricalPartitionTaskExecutor.java | 64 ++- .../fluss/server/tablet/TabletService.java | 35 +- .../server/utils/ServerRpcMessageUtils.java | 35 +- .../utils/TableDescriptorValidation.java | 4 - .../fluss/server/DynamicConfigChangeTest.java | 31 ++ .../server/replica/ReplicaManagerTest.java | 27 ++ .../HistoricalPartitionManagerTest.java | 325 +++++++++++++ .../HistoricalPartitionTaskExecutorTest.java | 31 ++ ...istoricalPartitionTableValidationTest.java | 29 +- .../utils/ServerRpcMessageUtilsTest.java | 52 ++ 58 files changed, 2814 insertions(+), 182 deletions(-) rename fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/{HistoricalPartitionLookupITCase.java => HistoricalPartitionITCase.java} (64%) create mode 100644 fluss-server/src/main/java/org/apache/fluss/server/entity/ProduceLogDataForBucket.java diff --git a/fluss-client/src/main/java/org/apache/fluss/client/utils/ClientRpcMessageUtils.java b/fluss-client/src/main/java/org/apache/fluss/client/utils/ClientRpcMessageUtils.java index 3c7512945dc..1a40afc92be 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/utils/ClientRpcMessageUtils.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/utils/ClientRpcMessageUtils.java @@ -147,6 +147,10 @@ public static ProduceLogRequest makeProduceLogRequest( if (tableBucket.getPartitionId() != null) { pbProduceLogReqForBucket.setPartitionId(tableBucket.getPartitionId()); } + if (readyBatch.writeBatch().getOriginalPartitionName() != null) { + pbProduceLogReqForBucket.setOriginalPartitionName( + readyBatch.writeBatch().getOriginalPartitionName()); + } }); return request; } @@ -202,6 +206,11 @@ public static PutKvRequest makePutKvRequest( if (tableBucket.getPartitionId() != null) { pbPutKvReqForBucket.setPartitionId(tableBucket.getPartitionId()); } + KvWriteBatch kvWriteBatch = (KvWriteBatch) readyBatch.writeBatch(); + if (kvWriteBatch.getOriginalPartitionName() != null) { + pbPutKvReqForBucket.setOriginalPartitionName( + kvWriteBatch.getOriginalPartitionName()); + } }); return request; } diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/AbstractRowLogWriteBatch.java b/fluss-client/src/main/java/org/apache/fluss/client/write/AbstractRowLogWriteBatch.java index 104f8f29e33..d9afe697600 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/AbstractRowLogWriteBatch.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/AbstractRowLogWriteBatch.java @@ -26,6 +26,8 @@ import org.apache.fluss.record.bytesview.BytesView; import org.apache.fluss.row.InternalRow; +import javax.annotation.Nullable; + import java.io.IOException; import java.util.List; @@ -49,11 +51,19 @@ protected AbstractRowLogWriteBatch( PhysicalTablePath physicalTablePath, int schemaId, WriteFormat writeFormat, + @Nullable String originalPartitionName, long createdMs, AbstractPagedOutputView outputView, MemoryLogRecordsRowBuilder recordsBuilder, String buildErrorMessage) { - super(tableId, bucketId, physicalTablePath, schemaId, writeFormat, createdMs); + super( + tableId, + bucketId, + physicalTablePath, + schemaId, + writeFormat, + originalPartitionName, + createdMs); this.outputView = outputView; this.recordsBuilder = recordsBuilder; this.buildErrorMessage = buildErrorMessage; diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/ArrowLogWriteBatch.java b/fluss-client/src/main/java/org/apache/fluss/client/write/ArrowLogWriteBatch.java index a6894b98b2e..4c4f293aa8f 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/ArrowLogWriteBatch.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/ArrowLogWriteBatch.java @@ -58,9 +58,17 @@ public ArrowLogWriteBatch( int schemaId, ArrowWriter arrowWriter, AbstractPagedOutputView outputView, + @Nullable String originalPartitionName, long createdMs, @Nullable LogRecordBatchStatisticsCollector statisticsCollector) { - super(tableId, bucketId, physicalTablePath, schemaId, WriteFormat.ARROW_LOG, createdMs); + super( + tableId, + bucketId, + physicalTablePath, + schemaId, + WriteFormat.ARROW_LOG, + originalPartitionName, + createdMs); this.outputView = outputView; this.recordsBuilder = MemoryLogRecordsArrowBuilder.builder( diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/CompactedLogWriteBatch.java b/fluss-client/src/main/java/org/apache/fluss/client/write/CompactedLogWriteBatch.java index 81bedff0fa0..1e3fc0b8dbc 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/CompactedLogWriteBatch.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/CompactedLogWriteBatch.java @@ -25,6 +25,7 @@ import org.apache.fluss.row.compacted.CompactedRow; import org.apache.fluss.rpc.messages.ProduceLogRequest; +import javax.annotation.Nullable; import javax.annotation.concurrent.NotThreadSafe; import static org.apache.fluss.utils.Preconditions.checkArgument; @@ -46,6 +47,7 @@ public CompactedLogWriteBatch( int schemaId, int writeLimit, AbstractPagedOutputView outputView, + @Nullable String originalPartitionName, long createdMs) { super( tableId, @@ -53,6 +55,7 @@ public CompactedLogWriteBatch( physicalTablePath, schemaId, WriteFormat.COMPACTED_LOG, + originalPartitionName, createdMs, outputView, MemoryLogRecordsCompactedBuilder.builder(schemaId, writeLimit, outputView, true), diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/IndexedLogWriteBatch.java b/fluss-client/src/main/java/org/apache/fluss/client/write/IndexedLogWriteBatch.java index 2bb496cbfe3..2b50402218b 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/IndexedLogWriteBatch.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/IndexedLogWriteBatch.java @@ -24,6 +24,7 @@ import org.apache.fluss.row.indexed.IndexedRow; import org.apache.fluss.rpc.messages.ProduceLogRequest; +import javax.annotation.Nullable; import javax.annotation.concurrent.NotThreadSafe; import static org.apache.fluss.utils.Preconditions.checkArgument; @@ -45,6 +46,7 @@ public IndexedLogWriteBatch( int schemaId, int writeLimit, AbstractPagedOutputView outputView, + @Nullable String originalPartitionName, long createdMs) { super( tableId, @@ -52,6 +54,7 @@ public IndexedLogWriteBatch( physicalTablePath, schemaId, WriteFormat.INDEXED_LOG, + originalPartitionName, createdMs, outputView, MemoryLogRecordsIndexedBuilder.builder(schemaId, writeLimit, outputView, true), diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/KvWriteBatch.java b/fluss-client/src/main/java/org/apache/fluss/client/write/KvWriteBatch.java index 4ca01e133b2..0a5c16db688 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/KvWriteBatch.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/KvWriteBatch.java @@ -63,6 +63,7 @@ public KvWriteBatch( AbstractPagedOutputView outputView, @Nullable int[] targetColumns, MergeMode mergeMode, + @Nullable String originalPartitionName, long createdMs) { super( tableId, @@ -70,6 +71,7 @@ public KvWriteBatch( physicalTablePath, schemaId, WriteFormat.fromKvFormat(kvFormat), + originalPartitionName, createdMs); this.outputView = outputView; this.recordsBuilder = diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java b/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java index ffa18ad9455..7546e48b8e5 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java @@ -32,6 +32,7 @@ import org.apache.fluss.metadata.PhysicalTablePath; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; import org.apache.fluss.metrics.MetricNames; import org.apache.fluss.record.LogRecordBatchStatisticsCollector; import org.apache.fluss.row.arrow.ArrowWriter; @@ -67,6 +68,7 @@ import static org.apache.fluss.record.LogRecordBatchFormat.NO_BATCH_SEQUENCE; import static org.apache.fluss.record.LogRecordBatchFormat.NO_WRITER_ID; import static org.apache.fluss.shaded.arrow.org.apache.arrow.memory.BufferAllocatorUtil.createBufferAllocator; +import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; import static org.apache.fluss.utils.Preconditions.checkNotNull; /* This file is based on source code of Apache Kafka Project (https://kafka.apache.org/), licensed by the Apache @@ -115,6 +117,9 @@ public final class RecordAccumulator { private final ConcurrentMap writeBatches = new CopyOnWriteMap<>(); + /** Tables observed by this writer with historical partition support enabled. */ + private final Set historicalPartitionEnabledTables = ConcurrentHashMap.newKeySet(); + private final IncompleteBatches incomplete; private final Map nodesDrainIndex; @@ -198,6 +203,9 @@ public RecordAppendResult append( throws Exception { PhysicalTablePath physicalTablePath = writeRecord.getPhysicalTablePath(); TableInfo tableInfo = writeRecord.getTableInfo(); + if (tableInfo.getTableConfig().isHistoricalPartitionEnabled()) { + historicalPartitionEnabledTables.add(tableInfo.getTablePath()); + } // The metadata may return null for the partition id, but it is fine to pass null here, // because we will fill the partitionId in bucketReady() before send the batch. Optional partitionIdOpt = cluster.getPartitionId(physicalTablePath); @@ -206,7 +214,9 @@ public RecordAppendResult append( physicalTablePath, k -> new BucketAndWriteBatches( - partitionIdOpt.orElse(null), tableInfo.isPartitioned())); + partitionIdOpt.orElse(null), + tableInfo.isPartitioned(), + physicalTablePath)); // We keep track of the number of appending thread to make sure we do not miss batches in // abortIncompleteBatches(). @@ -331,6 +341,49 @@ public void reEnqueue(ReadyWriteBatch readyWriteBatch) { } } + /** + * Tries to route writes for an original partition path to the given physical target. + * + *

The accumulator keeps queues keyed by {@code originalPath}, while metadata lookup, leader + * discovery, and RPC sending use {@code targetPath}. The target may therefore be either the + * original partition itself or the shared historical partition. + * + *

The first queue creation fixes the target for that original path. A later call succeeds + * only when it selects the same target; this method never moves queued or inflight batches + * between physical partitions. + * + * @return true if the target was installed or already matches, false if a different target was + * fixed previously + */ + boolean tryRouteWritesTo( + PhysicalTablePath originalPath, PhysicalTablePath targetPath, long targetPartitionId) { + BucketAndWriteBatches resolvedTarget = + new BucketAndWriteBatches(targetPartitionId, true, targetPath); + // Install the route atomically before append can create the first queue for this path. + BucketAndWriteBatches existing = writeBatches.putIfAbsent(originalPath, resolvedTarget); + if (existing == null) { + return true; + } + + // An append may already have fixed this path to a target. Keep that target and only accept + // the metadata result when it describes the same physical partition. + if (!existing.targetPath.equals(targetPath)) { + return false; + } + existing.partitionId = targetPartitionId; + return true; + } + + /** Returns whether a write target has already been chosen for this original path. */ + boolean hasWriteTarget(PhysicalTablePath originalPath) { + return writeBatches.containsKey(originalPath); + } + + /** Returns whether the target belongs to a table with historical partition support enabled. */ + boolean isHistoricalPartitionEnabled(PhysicalTablePath targetPath) { + return historicalPartitionEnabledTables.contains(targetPath.getTablePath()); + } + /** Abort all incomplete batches (whether they have been sent or not). */ public void abortAllBatches(final Exception reason) { for (WriteBatch batch : incomplete.copyAll()) { @@ -357,7 +410,8 @@ private Deque getOrCreateDeque( k -> new BucketAndWriteBatches( tableBucket.getPartitionId(), - physicalTablePath.getPartitionName() != null)); + physicalTablePath.getPartitionName() != null, + physicalTablePath)); return bucketAndWriteBatches.batches.computeIfAbsent( tableBucket.getBucket(), k -> new ArrayDeque<>()); } @@ -485,8 +539,9 @@ private long bucketReady( Cluster cluster, long nextReadyCheckDelayMs) { // first check this table has partitionId. + PhysicalTablePath targetPath = bucketAndWriteBatches.targetPath; if (bucketAndWriteBatches.isPartitionedTable && bucketAndWriteBatches.partitionId == null) { - Optional optionIdOpt = cluster.getPartitionId(physicalTablePath); + Optional optionIdOpt = cluster.getPartitionId(targetPath); if (optionIdOpt.isPresent()) { bucketAndWriteBatches.partitionId = optionIdOpt.get(); } else { @@ -495,7 +550,7 @@ private long bucketReady( physicalTablePath); // TODO: we shouldn't add unready partitions to unknownLeaderTables, // because it cases PartitionNotExistException later - unknownLeaderTables.add(physicalTablePath); + unknownLeaderTables.add(targetPath); return nextReadyCheckDelayMs; } } @@ -531,10 +586,10 @@ private long bucketReady( int bucketId = entry.getKey(); Optional tableIdOpt = cluster.getTableId(physicalTablePath.getTablePath()); if (!tableIdOpt.isPresent()) { - unknownLeaderTables.add(physicalTablePath); + unknownLeaderTables.add(targetPath); } else { TableBucket tableBucket = - cluster.getTableBucket(tableIdOpt.get(), physicalTablePath, bucketId); + cluster.getTableBucket(tableIdOpt.get(), targetPath, bucketId); // If this bucket is throttled, don't mark its node as ready. // Instead, factor the remaining throttle time into the next check delay. @@ -556,7 +611,7 @@ private long bucketReady( // This is a bucket for which leader is not known, but messages are // available to send. Note that entries are currently not removed from // batches when deque is empty. - unknownLeaderTables.add(physicalTablePath); + unknownLeaderTables.add(targetPath); } else { nextReadyCheckDelayMs = batchReady( @@ -627,6 +682,15 @@ private RecordAppendResult appendNewBatch( PreAllocatedPagedOutputView outputView = new PreAllocatedPagedOutputView(segments); int schemaId = tableInfo.getSchemaId(); WriteFormat writeFormat = writeRecord.getWriteFormat(); + BucketAndWriteBatches bucketAndWriteBatches = + checkNotNull( + writeBatches.get(physicalTablePath), + "Write batches for %s must exist.", + physicalTablePath); + String originalPartitionName = + bucketAndWriteBatches.isHistoricalWriteTarget() + ? checkNotNull(physicalTablePath.getPartitionName()) + : null; final WriteBatch batch = createWriteBatch( writeRecord, @@ -635,7 +699,8 @@ private RecordAppendResult appendNewBatch( writeFormat, physicalTablePath, outputView, - schemaId); + schemaId, + originalPartitionName); batch.tryAppend(writeRecord, callback); deque.addLast(batch); @@ -650,7 +715,8 @@ private WriteBatch createWriteBatch( WriteFormat writeFormat, PhysicalTablePath physicalTablePath, PreAllocatedPagedOutputView outputView, - int schemaId) { + int schemaId, + @Nullable String originalPartitionName) { // If the table is kv table we need to create a kv batch, otherwise we create a log batch. switch (writeFormat) { case COMPACTED_KV: @@ -665,6 +731,7 @@ private WriteBatch createWriteBatch( outputView, writeRecord.getTargetColumns(), writeRecord.getMergeMode(), + originalPartitionName, clock.milliseconds()); case ARROW_LOG: @@ -688,6 +755,7 @@ private WriteBatch createWriteBatch( tableInfo.getSchemaId(), arrowWriter, outputView, + originalPartitionName, clock.milliseconds(), statisticsCollector); @@ -699,6 +767,7 @@ private WriteBatch createWriteBatch( schemaId, outputView.getPreAllocatedSize(), outputView, + originalPartitionName, clock.milliseconds()); case INDEXED_LOG: @@ -709,6 +778,7 @@ private WriteBatch createWriteBatch( tableInfo.getSchemaId(), outputView.getPreAllocatedSize(), outputView, + originalPartitionName, clock.milliseconds()); default: @@ -1013,6 +1083,10 @@ private List getAllBucketsInCurrentNode(Integer currentNode, Clu List buckets = new ArrayList<>(); Set physicalTablePaths = cluster.getBucketLocationsByPath().keySet(); for (PhysicalTablePath path : physicalTablePaths) { + BucketAndWriteBatches bucketAndWriteBatches = writeBatches.get(path); + if (bucketAndWriteBatches != null && bucketAndWriteBatches.isHistoricalWriteTarget()) { + continue; + } List bucketsForTable = cluster.getAvailableBucketsForPhysicalTablePath(path); for (BucketLocation bucket : bucketsForTable) { @@ -1023,6 +1097,29 @@ private List getAllBucketsInCurrentNode(Integer currentNode, Clu } } } + + // Historical queues remain keyed by their original partition path. Add a location using + // that queue key while retaining the historical bucket as the RPC target. + for (Map.Entry entry : writeBatches.entrySet()) { + BucketAndWriteBatches bucketAndWriteBatches = entry.getValue(); + PhysicalTablePath originalPath = entry.getKey(); + if (!bucketAndWriteBatches.isHistoricalWriteTarget()) { + continue; + } + for (BucketLocation bucketLocation : + cluster.getAvailableBucketsForPhysicalTablePath( + bucketAndWriteBatches.targetPath)) { + if (bucketLocation.getLeader() != null + && Objects.equals(currentNode, bucketLocation.getLeader())) { + buckets.add( + new BucketLocation( + originalPath, + bucketLocation.getTableBucket(), + bucketLocation.getLeader(), + bucketLocation.getReplicas())); + } + } + } return buckets; } @@ -1162,13 +1259,24 @@ public void destroyResources() { /** Per table bucket and write batches. */ private static class BucketAndWriteBatches { public final boolean isPartitionedTable; + /** The physical partition used for metadata lookup, leader discovery, and write RPCs. */ + private final PhysicalTablePath targetPath; + public volatile @Nullable Long partitionId; // Write batches for each bucket in queue. public final Map> batches = new CopyOnWriteMap<>(); - public BucketAndWriteBatches(@Nullable Long partitionId, boolean isPartitionedTable) { + private BucketAndWriteBatches( + @Nullable Long partitionId, + boolean isPartitionedTable, + PhysicalTablePath targetPath) { this.partitionId = partitionId; this.isPartitionedTable = isPartitionedTable; + this.targetPath = targetPath; + } + + public boolean isHistoricalWriteTarget() { + return HISTORICAL_PARTITION_VALUE.equals(targetPath.getPartitionName()); } } } diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java b/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java index f4f0c6eedf5..1ecd39881ef 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java @@ -45,6 +45,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nullable; import javax.annotation.concurrent.GuardedBy; import java.util.ArrayList; @@ -52,10 +53,12 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import static org.apache.fluss.client.utils.ClientRpcMessageUtils.makeProduceLogRequest; import static org.apache.fluss.client.utils.ClientRpcMessageUtils.makePutKvRequest; +import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; import static org.apache.fluss.utils.Preconditions.checkArgument; import static org.apache.fluss.utils.Preconditions.checkNotNull; @@ -228,8 +231,8 @@ private void sendWriteData() throws Exception { // TODO: this try-catch is not needed when we don't update metadata for // unready partitions Throwable t = ExceptionUtils.stripExecutionException(e); - if (t.getCause() instanceof PartitionNotExistException) { - // ignore this exception, this is probably happen because the partition + if (t instanceof PartitionNotExistException) { + abortIfHistoricalWriteTargetMissing(readyCheckResult.unknownLeaderTables); } else { throw e; } @@ -250,10 +253,8 @@ private void sendWriteData() throws Exception { // get the list of batches prepare to send. Map> batches = accumulator.drain(clusterSnapshot, readyNodes, maxRequestSize); - if (!batches.isEmpty()) { addToInflightBatches(batches); - // TODO add logic for batch expire. sendWriteRequests(batches); @@ -410,25 +411,76 @@ private void sendWriteRequest(int destination, short acks, List } else { writeBatchByTable.forEach( (tableId, writeBatches) -> { - if (isLogBatches(writeBatches)) { - sendProduceLogRequestAndHandleResponse( - gateway, - makeProduceLogRequest( - tableId, acks, maxRequestTimeoutMs, writeBatches), - tableId, - writeBatches); - } else { - sendPutKvRequestAndHandleResponse( - gateway, - makePutKvRequest( - tableId, acks, maxRequestTimeoutMs, writeBatches), - tableId, - writeBatches); + boolean logBatches = isLogBatches(writeBatches); + for (List requestGroup : packRequestGroups(writeBatches)) { + if (logBatches) { + sendProduceLogRequestAndHandleResponse( + gateway, + makeProduceLogRequest( + tableId, acks, maxRequestTimeoutMs, requestGroup), + tableId, + requestGroup); + } else { + sendPutKvRequestAndHandleResponse( + gateway, + makePutKvRequest( + tableId, acks, maxRequestTimeoutMs, requestGroup), + tableId, + requestGroup); + } } }); } } + /** + * Splits normal and historical batches into separate requests. + * + *

Normal and historical writes cannot share a request. Both write protocols correlate + * historical responses by {@link TableBucket} and original partition name, so different + * original partitions targeting the same historical table bucket can remain in one request. + */ + private static List> packRequestGroups( + List writeBatches) { + List normalBatches = new ArrayList<>(); + List historicalBatches = new ArrayList<>(); + + for (ReadyWriteBatch readyWriteBatch : writeBatches) { + if (readyWriteBatch.writeBatch().getOriginalPartitionName() == null) { + normalBatches.add(readyWriteBatch); + } else { + historicalBatches.add(readyWriteBatch); + } + } + + List> requestGroups = new ArrayList<>(2); + if (!normalBatches.isEmpty()) { + requestGroups.add(normalBatches); + } + if (!historicalBatches.isEmpty()) { + requestGroups.add(historicalBatches); + } + return requestGroups; + } + + private static Map toBatchesByKey( + List writeBatches) { + Map recordsByKey = new HashMap<>(); + for (ReadyWriteBatch readyWriteBatch : writeBatches) { + WriteBatch writeBatch = readyWriteBatch.writeBatch(); + WriteBatchKey key = + new WriteBatchKey( + readyWriteBatch.tableBucket(), writeBatch.getOriginalPartitionName()); + ReadyWriteBatch previous = recordsByKey.put(key, readyWriteBatch); + checkArgument( + previous == null, + "A write request contains duplicate table bucket %s and original partition %s.", + readyWriteBatch.tableBucket(), + writeBatch.getOriginalPartitionName()); + } + return recordsByKey; + } + /** * Check whether the given batches are log batches. We assume all the batches are of the same * type. @@ -447,8 +499,7 @@ private void sendProduceLogRequestAndHandleResponse( ProduceLogRequest request, long tableId, List writeBatches) { - Map recordsByBucket = new HashMap<>(); - writeBatches.forEach(batch -> recordsByBucket.put(batch.tableBucket(), batch)); + Map recordsByKey = toBatchesByKey(writeBatches); long startTime = System.currentTimeMillis(); gateway.produceLog(request) .whenComplete( @@ -458,8 +509,7 @@ private void sendProduceLogRequestAndHandleResponse( if (e != null) { handleWriteRequestException(e, writeBatches); } else { - handleProduceLogResponse( - produceLogResponse, tableId, recordsByBucket); + handleProduceLogResponse(produceLogResponse, tableId, recordsByKey); } }); } @@ -469,8 +519,7 @@ private void sendPutKvRequestAndHandleResponse( PutKvRequest request, long tableId, List writeBatches) { - Map recordsByBucket = new HashMap<>(); - writeBatches.forEach(batch -> recordsByBucket.put(batch.tableBucket(), batch)); + Map recordsByKey = toBatchesByKey(writeBatches); long startTime = System.currentTimeMillis(); gateway.putKv(request) .whenComplete( @@ -480,7 +529,7 @@ private void sendPutKvRequestAndHandleResponse( if (e != null) { handleWriteRequestException(e, writeBatches); } else { - handlePutKvResponse(putKvResponse, tableId, recordsByBucket); + handlePutKvResponse(putKvResponse, tableId, recordsByKey); } }); } @@ -488,7 +537,7 @@ private void sendPutKvRequestAndHandleResponse( private void handleProduceLogResponse( ProduceLogResponse response, long tableId, - Map recordsByBucket) { + Map recordsByKey) { Set invalidMetadataTablesSet = new HashSet<>(); for (PbProduceLogRespForBucket logRespForBucket : response.getBucketsRespsList()) { TableBucket tb = @@ -498,7 +547,13 @@ private void handleProduceLogResponse( ? logRespForBucket.getPartitionId() : null, logRespForBucket.getBucketId()); - ReadyWriteBatch writeBatch = recordsByBucket.get(tb); + ReadyWriteBatch writeBatch = + recordsByKey.get( + new WriteBatchKey( + tb, + logRespForBucket.hasOriginalPartitionName() + ? logRespForBucket.getOriginalPartitionName() + : null)); if (logRespForBucket.hasErrorCode()) { Set invalidMetadataTables = handleWriteBatchException( @@ -514,7 +569,7 @@ private void handleProduceLogResponse( private void handlePutKvResponse( PutKvResponse putKvResponse, long tableId, - Map recordsByBucket) { + Map recordsByKey) { Set invalidMetadataTablesSet = new HashSet<>(); for (PbPutKvRespForBucket respForBucket : putKvResponse.getBucketsRespsList()) { TableBucket tb = @@ -528,7 +583,13 @@ private void handlePutKvResponse( accumulator.updateThrottle(tb, respForBucket.getPressure()); } - ReadyWriteBatch writeBatch = recordsByBucket.get(tb); + ReadyWriteBatch writeBatch = + recordsByKey.get( + new WriteBatchKey( + tb, + respForBucket.hasOriginalPartitionName() + ? respForBucket.getOriginalPartitionName() + : null)); if (writeBatch == null) { continue; } @@ -566,6 +627,14 @@ private Set handleWriteBatchException( ReadyWriteBatch readyWriteBatch, ApiError error) { Set invalidMetadataTables = new HashSet<>(); WriteBatch writeBatch = readyWriteBatch.writeBatch(); + // Historical queues use the original path as their accumulator key, so capture the actual + // RPC target before any retry handling. + PhysicalTablePath requestTargetPath = + writeBatch.getOriginalPartitionName() == null + ? writeBatch.physicalTablePath() + : PhysicalTablePath.of( + writeBatch.physicalTablePath().getTablePath(), + HISTORICAL_PARTITION_VALUE); if (error.exception() instanceof StorageBackpressureException) { // Hard rejection: the storage engine reached its slowdown trigger and rejected the // write. Map it to full pressure (internal hard-rejection value 1.0f) so the bucket is @@ -634,7 +703,10 @@ private Set handleWriteBatchException( readyWriteBatch.tableBucket(), error.exception()); } - invalidMetadataTables.add(writeBatch.physicalTablePath()); + // A historical batch remains keyed by its original partition path in the + // accumulator, but its RPC is sent to the internal historical partition. Invalidate + // the actual RPC target so the retry refreshes the historical bucket metadata. + invalidMetadataTables.add(requestTargetPath); } } else { LOG.warn( @@ -649,6 +721,35 @@ private Set handleWriteBatchException( return invalidMetadataTables; } + private void abortIfHistoricalWriteTargetMissing(Set unknownLeaderTables) + throws Exception { + for (PhysicalTablePath targetPath : unknownLeaderTables) { + if (!accumulator.isHistoricalPartitionEnabled(targetPath)) { + continue; + } + try { + metadataUpdater.checkAndUpdatePartitionMetadata(targetPath); + } catch (Exception e) { + Throwable t = ExceptionUtils.stripExecutionException(e); + if (t instanceof PartitionNotExistException) { + // Retrying a historical-enabled table without a leader would leave its + // batches queued indefinitely. Fail only after checking the target itself so + // ordinary writes in the bulk metadata request keep their existing behavior. + PartitionNotExistException missingTargetException = + new PartitionNotExistException( + "Write target " + + targetPath + + " for a historical-partition-enabled table no " + + "longer exists according to refreshed metadata."); + missingTargetException.initCause(t); + maybeAbortBatches(missingTargetException); + return; + } + throw e; + } + } + } + private void updateWriterMetrics(Map> batches) { batches.values() .forEach( @@ -719,4 +820,32 @@ private void awaitNextReadyCheck(long delayMs) throws InterruptedException { void destroyResources() { accumulator.destroyResources(); } + + private static final class WriteBatchKey { + private final TableBucket tableBucket; + private final @Nullable String originalPartitionName; + + private WriteBatchKey(TableBucket tableBucket, @Nullable String originalPartitionName) { + this.tableBucket = tableBucket; + this.originalPartitionName = originalPartitionName; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof WriteBatchKey)) { + return false; + } + WriteBatchKey that = (WriteBatchKey) o; + return tableBucket.equals(that.tableBucket) + && Objects.equals(originalPartitionName, that.originalPartitionName); + } + + @Override + public int hashCode() { + return Objects.hash(tableBucket, originalPartitionName); + } + } } diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/WriteBatch.java b/fluss-client/src/main/java/org/apache/fluss/client/write/WriteBatch.java index 9cbfaa6af2e..a60e8ca6645 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/WriteBatch.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/WriteBatch.java @@ -54,6 +54,14 @@ public abstract class WriteBatch { protected final List callbacks = new ArrayList<>(); private final AtomicReference finalState = new AtomicReference<>(null); private final AtomicInteger attempts = new AtomicInteger(0); + /** + * The original partition name for a batch targeting the historical system partition. + * + *

It is null for a normal write and contains the logical partition namespace for a + * historical write. + */ + private final @Nullable String originalPartitionName; + protected boolean reopened; protected int recordCount; private long drainedMs; @@ -68,12 +76,14 @@ public WriteBatch( PhysicalTablePath physicalTablePath, int schemaId, WriteFormat writeFormat, + @Nullable String originalPartitionName, long createdMs) { this.physicalTablePath = physicalTablePath; this.createdMs = createdMs; this.tableId = tableId; this.schemaId = schemaId; this.writeFormat = checkNotNull(writeFormat, "write format must be not null"); + this.originalPartitionName = originalPartitionName; this.bucketId = bucketId; this.requestFuture = new RequestFuture(); this.recordCount = 0; @@ -205,6 +215,11 @@ public PhysicalTablePath physicalTablePath() { return physicalTablePath; } + /** Returns the original partition name for a historical write, or null for a normal write. */ + public @Nullable String getOriginalPartitionName() { + return originalPartitionName; + } + public RequestFuture getRequestFuture() { return requestFuture; } diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/WriterClient.java b/fluss-client/src/main/java/org/apache/fluss/client/write/WriterClient.java index ad8c7870547..cb46c627bed 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/WriterClient.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/WriterClient.java @@ -28,10 +28,12 @@ import org.apache.fluss.config.Configuration; import org.apache.fluss.exception.FlussRuntimeException; import org.apache.fluss.exception.IllegalConfigurationException; +import org.apache.fluss.exception.PartitionNotExistException; import org.apache.fluss.metadata.PhysicalTablePath; import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.rpc.gateway.TabletServerGateway; import org.apache.fluss.rpc.metrics.ClientMetricGroup; +import org.apache.fluss.utils.AutoPartitionStrategy; import org.apache.fluss.utils.CopyOnWriteMap; import org.apache.fluss.utils.clock.SystemClock; import org.apache.fluss.utils.concurrent.ExecutorThreadFactory; @@ -42,6 +44,9 @@ import javax.annotation.concurrent.ThreadSafe; import java.time.Duration; +import java.time.Instant; +import java.time.ZonedDateTime; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.concurrent.ExecutorService; @@ -51,6 +56,9 @@ import static org.apache.fluss.config.ConfigOptions.NoKeyAssigner.ROUND_ROBIN; import static org.apache.fluss.config.ConfigOptions.NoKeyAssigner.STICKY; import static org.apache.fluss.utils.ExceptionUtils.toException; +import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; +import static org.apache.fluss.utils.PartitionUtils.generateAutoPartitionTime; +import static org.apache.fluss.utils.PartitionUtils.isPastAutoPartition; /** * A client that write records to server. @@ -70,6 +78,7 @@ public class WriterClient { private static final Logger LOG = LoggerFactory.getLogger(WriterClient.class); public static final String SENDER_THREAD_PREFIX = "fluss-write-sender"; + private static final Duration MAX_DEFAULT_TIME_ZONE_DIFFERENCE = Duration.ofHours(26); /** * {@link ConfigOptions#CLIENT_WRITER_MAX_INFLIGHT_REQUESTS_PER_BUCKET} should be less than or * equal to this value when idempotence producer enabled to ensure message ordering. @@ -194,7 +203,12 @@ private void doSend(WriteRecord record, WriteCallback callback) { PhysicalTablePath physicalTablePath = record.getPhysicalTablePath(); // Skip the call entirely on non-partitioned tables; there is no partition to create. if (tableInfo.isPartitioned()) { - dynamicPartitionCreator.checkAndCreatePartitionAsync(physicalTablePath, tableInfo); + if (mayBeExpiredHistoricalPartition(physicalTablePath, tableInfo, Instant.now())) { + resolveHistoricalWriteTarget(physicalTablePath); + } else { + dynamicPartitionCreator.checkAndCreatePartitionAsync( + physicalTablePath, tableInfo); + } } // maybe create bucket assigner. @@ -240,6 +254,71 @@ private void doSend(WriteRecord record, WriteCallback callback) { } } + static boolean mayBeExpiredHistoricalPartition( + PhysicalTablePath physicalTablePath, TableInfo tableInfo, Instant now) { + String partitionName = physicalTablePath.getPartitionName(); + AutoPartitionStrategy strategy = tableInfo.getTableConfig().getAutoPartitionStrategy(); + if (partitionName == null + || !tableInfo.getTableConfig().isHistoricalPartitionEnabled() + || strategy.numToRetain() < 0) { + return false; + } + + // The table's default time zone is not persisted. Shift the expiration boundary by the + // largest IANA time-zone difference, then apply retention in the table's partition unit. + Instant latestPotentialServerTime = now.plus(MAX_DEFAULT_TIME_ZONE_DIFFERENCE); + if (!isPastAutoPartition(partitionName, strategy, latestPotentialServerTime)) { + return false; + } + ZonedDateTime latestPotentialServerDateTime = + ZonedDateTime.ofInstant(latestPotentialServerTime, strategy.timeZone().toZoneId()); + String earliestPotentialRetainedPartition = + generateAutoPartitionTime( + latestPotentialServerDateTime, + -strategy.numToRetain(), + strategy.timeUnit(), + strategy); + return partitionName.compareTo(earliestPotentialRetainedPartition) < 0; + } + + private synchronized void resolveHistoricalWriteTarget(PhysicalTablePath originalPath) { + if (accumulator.hasWriteTarget(originalPath)) { + return; + } + + // The time check only limits metadata traffic. Invalidate a potentially stale cached route + // and authoritatively choose the target before the record enters the queue. + metadataUpdater.invalidPhysicalTableBucketAndPartitionMeta( + Collections.singleton(originalPath)); + PhysicalTablePath targetPath = originalPath; + try { + if (!metadataUpdater.checkAndUpdatePartitionMetadata(originalPath)) { + throw new FlussRuntimeException( + "Failed to resolve write target for " + originalPath + '.'); + } + } catch (PartitionNotExistException ignored) { + targetPath = + PhysicalTablePath.of(originalPath.getTablePath(), HISTORICAL_PARTITION_VALUE); + // TODO: Activate this target only after Server retirement guarantees that all accepted + // original writes have been tiered to the lake. + if (!metadataUpdater.checkAndUpdatePartitionMetadata(targetPath)) { + throw new PartitionNotExistException( + "Historical partition " + targetPath + " does not exist."); + } + } + + if (!accumulator.tryRouteWritesTo( + originalPath, targetPath, metadataUpdater.getPartitionIdOrElseThrow(targetPath))) { + throw new FlussRuntimeException( + "Cannot route writes for " + + originalPath + + " to " + + targetPath + + " because the accumulator already contains writes for a different " + + "physical target."); + } + } + private void maybeAbortBatches(Throwable t) { if (accumulator.hasIncomplete()) { LOG.error("Aborting all pending write batches due to fatal error", t); diff --git a/fluss-client/src/test/java/org/apache/fluss/client/utils/ClientRpcMessageUtilsTest.java b/fluss-client/src/test/java/org/apache/fluss/client/utils/ClientRpcMessageUtilsTest.java index 3ed17da7da5..153c3dc13f0 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/utils/ClientRpcMessageUtilsTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/utils/ClientRpcMessageUtilsTest.java @@ -140,6 +140,7 @@ private KvWriteBatch createKvWriteBatch(int bucketId, MergeMode mergeMode) throw outputView, null, mergeMode, + null, System.currentTimeMillis()); } } diff --git a/fluss-client/src/test/java/org/apache/fluss/client/write/ArrowLogWriteBatchTest.java b/fluss-client/src/test/java/org/apache/fluss/client/write/ArrowLogWriteBatchTest.java index f87f0c01a35..c9b98e1dfca 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/write/ArrowLogWriteBatchTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/write/ArrowLogWriteBatchTest.java @@ -137,6 +137,7 @@ void testAppendWithPreAllocatedMemorySegments() throws Exception { DATA1_ROW_TYPE, DEFAULT_COMPRESSION), new PreAllocatedPagedOutputView(memorySegmentList), + null, System.currentTimeMillis(), null); assertThat(arrowLogWriteBatch.pooledMemorySegments()).isEqualTo(memorySegmentList); @@ -213,6 +214,7 @@ void testArrowCompressionRatioEstimated() throws Exception { DATA1_TABLE_INFO.getSchemaId(), arrowWriter, new PreAllocatedPagedOutputView(memorySegmentList), + null, System.currentTimeMillis(), null); @@ -315,6 +317,7 @@ private ArrowLogWriteBatch createArrowLogWriteBatch(TableBucket tb, int maxSizeI DATA1_ROW_TYPE, DEFAULT_COMPRESSION), new UnmanagedPagedOutputView(128), + null, System.currentTimeMillis(), null); } diff --git a/fluss-client/src/test/java/org/apache/fluss/client/write/CompactedLogWriteBatchTest.java b/fluss-client/src/test/java/org/apache/fluss/client/write/CompactedLogWriteBatchTest.java index dc8fbce9d32..2461c2a7134 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/write/CompactedLogWriteBatchTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/write/CompactedLogWriteBatchTest.java @@ -254,6 +254,7 @@ private CompactedLogWriteBatch createLogWriteBatch( DATA1_TABLE_INFO.getSchemaId(), writeLimit, new PreAllocatedPagedOutputView(Collections.singletonList(memorySegment)), + null, System.currentTimeMillis()); } diff --git a/fluss-client/src/test/java/org/apache/fluss/client/write/IndexedLogWriteBatchTest.java b/fluss-client/src/test/java/org/apache/fluss/client/write/IndexedLogWriteBatchTest.java index 331be4209f6..a6269ad64b3 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/write/IndexedLogWriteBatchTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/write/IndexedLogWriteBatchTest.java @@ -216,6 +216,7 @@ private IndexedLogWriteBatch createLogWriteBatch( DATA1_TABLE_INFO.getSchemaId(), writeLimit, new PreAllocatedPagedOutputView(Collections.singletonList(memorySegment)), + null, System.currentTimeMillis()); } diff --git a/fluss-client/src/test/java/org/apache/fluss/client/write/KvWriteBatchTest.java b/fluss-client/src/test/java/org/apache/fluss/client/write/KvWriteBatchTest.java index 7b61976a038..b0faf06e679 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/write/KvWriteBatchTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/write/KvWriteBatchTest.java @@ -230,6 +230,7 @@ private KvWriteBatch createKvWriteBatch( outputView, null, MergeMode.DEFAULT, + null, System.currentTimeMillis()); } @@ -326,6 +327,7 @@ private KvWriteBatch createKvWriteBatchWithMergeMode(TableBucket tb, MergeMode m outputView, null, mergeMode, + null, System.currentTimeMillis()); } diff --git a/fluss-client/src/test/java/org/apache/fluss/client/write/RecordAccumulatorTest.java b/fluss-client/src/test/java/org/apache/fluss/client/write/RecordAccumulatorTest.java index acd1e4e2911..21939d16888 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/write/RecordAccumulatorTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/write/RecordAccumulatorTest.java @@ -43,6 +43,7 @@ import org.apache.fluss.row.BinaryRow; import org.apache.fluss.row.GenericRow; import org.apache.fluss.row.arrow.ArrowWriter; +import org.apache.fluss.row.encode.CompactedKeyEncoder; import org.apache.fluss.row.indexed.IndexedRow; import org.apache.fluss.rpc.GatewayClientProxy; import org.apache.fluss.rpc.RpcClient; @@ -71,14 +72,21 @@ import static org.apache.fluss.record.LogRecordBatch.CURRENT_LOG_MAGIC_VALUE; import static org.apache.fluss.record.LogRecordBatchFormat.recordBatchHeaderSize; import static org.apache.fluss.record.TestData.DATA1_PHYSICAL_TABLE_PATH; +import static org.apache.fluss.record.TestData.DATA1_PHYSICAL_TABLE_PATH_PK_PA_2024; import static org.apache.fluss.record.TestData.DATA1_ROW_TYPE; import static org.apache.fluss.record.TestData.DATA1_SCHEMA; +import static org.apache.fluss.record.TestData.DATA1_SCHEMA_PK; import static org.apache.fluss.record.TestData.DATA1_TABLE_ID; +import static org.apache.fluss.record.TestData.DATA1_TABLE_ID_PK; import static org.apache.fluss.record.TestData.DATA1_TABLE_INFO; +import static org.apache.fluss.record.TestData.DATA1_TABLE_INFO_PK; import static org.apache.fluss.record.TestData.DATA1_TABLE_PATH; +import static org.apache.fluss.record.TestData.DATA1_TABLE_PATH_PK; import static org.apache.fluss.record.TestData.DEFAULT_REMOTE_DATA_DIR; +import static org.apache.fluss.testutils.DataTestUtils.compactedRow; import static org.apache.fluss.testutils.DataTestUtils.indexedRow; import static org.apache.fluss.testutils.DataTestUtils.row; +import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -191,6 +199,41 @@ void testDrainBatches() throws Exception { verifyTableBucketInBatches(batches3, tb1, tb3); } + @Test + void testAppendAfterHistoricalTargetResolved() throws Exception { + long originalPartitionId = 11L; + long historicalPartitionId = 22L; + PhysicalTablePath originalPath = DATA1_PHYSICAL_TABLE_PATH_PK_PA_2024; + PhysicalTablePath anotherOriginalPath = PhysicalTablePath.of(DATA1_TABLE_PATH_PK, "2023"); + PhysicalTablePath historicalPath = + PhysicalTablePath.of(DATA1_TABLE_PATH_PK, HISTORICAL_PARTITION_VALUE); + TableBucket originalBucket = new TableBucket(DATA1_TABLE_ID_PK, originalPartitionId, 0); + TableBucket historicalBucket = new TableBucket(DATA1_TABLE_ID_PK, historicalPartitionId, 0); + cluster = + createPartitionedKvCluster( + originalPath, originalBucket, historicalPath, historicalBucket); + + RecordAccumulator accum = createTestRecordAccumulator(1024, 10L * 1024); + accum.tryRouteWritesTo(originalPath, historicalPath, historicalPartitionId); + accum.tryRouteWritesTo(anotherOriginalPath, historicalPath, historicalPartitionId); + accum.append(createKvRecord(originalPath), writeCallback, cluster, 0, false); + accum.append(createKvRecord(originalPath), writeCallback, cluster, 0, false); + accum.append(createKvRecord(anotherOriginalPath), writeCallback, cluster, 0, false); + + List drainedBatches = + accum.drain(cluster, Collections.singleton(node1.id()), Integer.MAX_VALUE) + .get(node1.id()); + + assertThat(drainedBatches).hasSize(2); + assertThat(drainedBatches) + .allSatisfy(batch -> assertThat(batch.tableBucket()).isEqualTo(historicalBucket)); + assertThat(drainedBatches) + .extracting(batch -> ((KvWriteBatch) batch.writeBatch()).getOriginalPartitionName()) + .containsExactlyInAnyOrder( + originalPath.getPartitionName(), anotherOriginalPath.getPartitionName()); + drainedBatches.forEach(batch -> accum.deallocate(batch.writeBatch())); + } + @Test void testDrainCompressedBatches() throws Exception { int batchSize = 10 * 1024; @@ -584,6 +627,21 @@ private WriteRecord createRecord(IndexedRow row, TableInfo tableInfo) { return WriteRecord.forIndexedAppend(tableInfo, DATA1_PHYSICAL_TABLE_PATH, row, null); } + private WriteRecord createKvRecord(PhysicalTablePath physicalTablePath) { + BinaryRow row = compactedRow(DATA1_ROW_TYPE, new Object[] {1, "a"}); + byte[] key = + new CompactedKeyEncoder(DATA1_ROW_TYPE, DATA1_SCHEMA_PK.getPrimaryKeyIndexes()) + .encodeKey(row); + return WriteRecord.forUpsert( + DATA1_TABLE_INFO_PK, + physicalTablePath, + row, + key, + key, + WriteFormat.COMPACTED_KV, + null); + } + private TableInfo withSchemaId(int schemaId) { return new TableInfo( DATA1_TABLE_INFO.getTablePath(), @@ -622,6 +680,36 @@ private Cluster updateCluster(List bucketLocations) { Collections.emptyMap()); } + private Cluster createPartitionedKvCluster( + PhysicalTablePath originalPath, + TableBucket originalBucket, + PhysicalTablePath historicalPath, + TableBucket historicalBucket) { + Map aliveTabletServersById = new HashMap<>(); + aliveTabletServersById.put(node1.id(), node1); + + Map> bucketsByPath = new HashMap<>(); + bucketsByPath.put( + originalPath, + Collections.singletonList( + new BucketLocation(originalPath, originalBucket, node1.id(), serverNodes))); + bucketsByPath.put( + historicalPath, + Collections.singletonList( + new BucketLocation( + historicalPath, historicalBucket, node1.id(), serverNodes))); + + Map partitionIdsByPath = new HashMap<>(); + partitionIdsByPath.put(originalPath, originalBucket.getPartitionId()); + partitionIdsByPath.put(historicalPath, historicalBucket.getPartitionId()); + return new Cluster( + aliveTabletServersById, + new ServerNode(0, "localhost", 89, ServerType.COORDINATOR), + bucketsByPath, + Collections.singletonMap(DATA1_TABLE_PATH_PK, DATA1_TABLE_ID_PK), + partitionIdsByPath); + } + private void delayedInterrupt(final Thread thread, final long delayMs) { Thread t = new Thread( diff --git a/fluss-client/src/test/java/org/apache/fluss/client/write/SenderTest.java b/fluss-client/src/test/java/org/apache/fluss/client/write/SenderTest.java index 3e99500c317..566e0c9e6d6 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/write/SenderTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/write/SenderTest.java @@ -19,24 +19,31 @@ import org.apache.fluss.client.metadata.TestingMetadataUpdater; import org.apache.fluss.client.metrics.TestingWriterMetricGroup; +import org.apache.fluss.cluster.BucketLocation; import org.apache.fluss.cluster.Cluster; import org.apache.fluss.cluster.ServerNode; +import org.apache.fluss.config.AutoPartitionTimeUnit; import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; import org.apache.fluss.config.MemorySize; import org.apache.fluss.exception.AuthorizationException; import org.apache.fluss.exception.NetworkException; import org.apache.fluss.exception.OutOfOrderSequenceException; +import org.apache.fluss.exception.PartitionNotExistException; import org.apache.fluss.exception.TableNotExistException; import org.apache.fluss.exception.TimeoutException; +import org.apache.fluss.metadata.DataLakeFormat; import org.apache.fluss.metadata.PhysicalTablePath; +import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.metadata.TablePath; import org.apache.fluss.record.MemoryLogRecords; import org.apache.fluss.row.BinaryRow; import org.apache.fluss.row.GenericRow; import org.apache.fluss.row.encode.CompactedKeyEncoder; +import org.apache.fluss.row.indexed.IndexedRow; import org.apache.fluss.rpc.entity.ProduceLogResultForBucket; import org.apache.fluss.rpc.entity.PutKvResultForBucket; import org.apache.fluss.rpc.messages.ApiMessage; @@ -45,7 +52,9 @@ import org.apache.fluss.rpc.messages.PutKvRequest; import org.apache.fluss.rpc.messages.PutKvResponse; import org.apache.fluss.rpc.protocol.Errors; +import org.apache.fluss.server.entity.ProduceLogDataForBucket; import org.apache.fluss.server.tablet.TestTabletServerGateway; +import org.apache.fluss.types.DataTypes; import org.apache.fluss.utils.clock.SystemClock; import org.junit.jupiter.api.AfterEach; @@ -53,10 +62,13 @@ import org.junit.jupiter.api.Test; import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Deque; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -77,12 +89,14 @@ import static org.apache.fluss.record.TestData.DATA2_TABLE_ID; import static org.apache.fluss.record.TestData.DEFAULT_REMOTE_DATA_DIR; import static org.apache.fluss.rpc.protocol.Errors.SCHEMA_NOT_EXIST; -import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getProduceLogData; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.makeProduceLogResponse; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.makePutKvResponse; +import static org.apache.fluss.server.utils.ServerRpcMessageUtils.toProduceLogDataForBuckets; import static org.apache.fluss.testutils.DataTestUtils.compactedRow; +import static org.apache.fluss.testutils.DataTestUtils.indexedRow; import static org.apache.fluss.testutils.DataTestUtils.row; import static org.apache.fluss.testutils.common.CommonTestUtils.retry; +import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; import static org.assertj.core.api.Assertions.assertThat; /** ITCase for {@link Sender}. */ @@ -115,6 +129,263 @@ public void teardown() throws Exception { sender.destroyResources(); } + @Test + void testSendsHistoricalPutWhenTargetResolvedBeforeAppend() throws Exception { + sender.destroyResources(); + String originalPartitionName = "20000101"; + TableInfo tableInfo = createHistoricalTableInfo(); + PhysicalTablePath originalPath = + PhysicalTablePath.of(tableInfo.getTablePath(), originalPartitionName); + PhysicalTablePath historicalPath = + PhysicalTablePath.of(tableInfo.getTablePath(), HISTORICAL_PARTITION_VALUE); + long historicalPartitionId = 22L; + TableBucket historicalBucket = + new TableBucket(tableInfo.getTableId(), historicalPartitionId, 0); + metadataUpdater = + new TestingMetadataUpdater( + Collections.singletonMap(tableInfo.getTablePath(), tableInfo)); + metadataUpdater.updateCluster( + partitionedCluster( + tableInfo, Collections.singletonMap(historicalPath, historicalBucket))); + sender = setupWithIdempotenceState(); + accumulator.tryRouteWritesTo(originalPath, historicalPath, historicalPartitionId); + + CompletableFuture future = + appendKvRecord(tableInfo, originalPath, 1, metadataUpdater.getCluster()); + + sender.runOnce(); + + assertThat(sender.numOfInFlightBatches(historicalBucket)).isOne(); + TestTabletServerGateway gateway = node1Gateway(); + PutKvRequest request = (PutKvRequest) gateway.getRequest(0); + assertThat(request.getBucketsReqAt(0).getPartitionId()).isEqualTo(historicalPartitionId); + assertThat(request.getBucketsReqAt(0).getOriginalPartitionName()) + .isEqualTo(originalPartitionName); + + gateway.response( + 0, createHistoricalPutKvResponse(historicalBucket, 1L, originalPartitionName)); + assertThat(future.get()).isNull(); + } + + @Test + void testPotentialExpirationUsesAutoPartitionTimeUnit() { + TableInfo tableInfo = createHistoricalTableInfo(AutoPartitionTimeUnit.HOUR, 48); + Instant now = Instant.parse("2026-08-24T00:00:00Z"); + + assertThat( + WriterClient.mayBeExpiredHistoricalPartition( + PhysicalTablePath.of(tableInfo.getTablePath(), "2026082301"), + tableInfo, + now)) + .isTrue(); + assertThat( + WriterClient.mayBeExpiredHistoricalPartition( + PhysicalTablePath.of(tableInfo.getTablePath(), "2026082302"), + tableInfo, + now)) + .isFalse(); + } + + @Test + void testFailsWriteAfterMetadataConfirmsPartitionMissing() throws Exception { + sender.destroyResources(); + TableInfo tableInfo = createHistoricalTableInfo(); + PhysicalTablePath originalPath = PhysicalTablePath.of(tableInfo.getTablePath(), "20990101"); + TableBucket originalBucket = new TableBucket(tableInfo.getTableId(), 21L, 0); + metadataUpdater = missingPartitionMetadataUpdater(tableInfo); + metadataUpdater.updateCluster( + partitionedCluster( + tableInfo, Collections.singletonMap(originalPath, originalBucket))); + sender = setupWithIdempotenceState(); + + CompletableFuture future = + appendKvRecord(tableInfo, originalPath, 1, metadataUpdater.getCluster()); + sender.runOnce(); + + TestTabletServerGateway gateway = node1Gateway(); + gateway.response( + 0, createPutKvResponse(originalBucket, Errors.UNKNOWN_TABLE_OR_BUCKET_EXCEPTION)); + assertThat(future).isNotDone(); + + sender.runOnce(); + assertThat(future.get()) + .isInstanceOf(PartitionNotExistException.class) + .hasMessageContaining(originalPath.toString()) + .hasCauseInstanceOf(PartitionNotExistException.class); + } + + @Test + void testMissingPartitionDoesNotAbortNormalWrites() throws Exception { + sender.destroyResources(); + TableInfo tableInfo = createNormalPartitionedTableInfo(); + PhysicalTablePath partitionPath = + PhysicalTablePath.of(tableInfo.getTablePath(), "20990101"); + TableBucket tableBucket = new TableBucket(tableInfo.getTableId(), 21L, 0); + metadataUpdater = missingPartitionMetadataUpdater(tableInfo); + metadataUpdater.updateCluster( + partitionedCluster( + tableInfo, Collections.singletonMap(partitionPath, tableBucket))); + sender = setupWithIdempotenceState(); + + CompletableFuture future = + appendKvRecord(tableInfo, partitionPath, 1, metadataUpdater.getCluster()); + sender.runOnce(); + + TestTabletServerGateway gateway = node1Gateway(); + gateway.response( + 0, createPutKvResponse(tableBucket, Errors.UNKNOWN_TABLE_OR_BUCKET_EXCEPTION)); + sender.runOnce(); + + assertThat(future).isNotDone(); + accumulator.abortAllBatches(new RuntimeException("Test cleanup.")); + } + + @Test + void testPackNormalAndHistoricalPutRequests() throws Exception { + sender.destroyResources(); + TableInfo tableInfo = createHistoricalTableInfo(); + PhysicalTablePath activePath = PhysicalTablePath.of(tableInfo.getTablePath(), "20990101"); + PhysicalTablePath firstOriginalPath = + PhysicalTablePath.of(tableInfo.getTablePath(), "20000101"); + PhysicalTablePath secondOriginalPath = + PhysicalTablePath.of(tableInfo.getTablePath(), "20000102"); + PhysicalTablePath historicalPath = + PhysicalTablePath.of(tableInfo.getTablePath(), HISTORICAL_PARTITION_VALUE); + TableBucket activeBucket = new TableBucket(tableInfo.getTableId(), 21L, 0); + TableBucket historicalBucket = new TableBucket(tableInfo.getTableId(), 22L, 0); + + Map tableBucketsByPath = new HashMap<>(); + tableBucketsByPath.put(activePath, activeBucket); + tableBucketsByPath.put(historicalPath, historicalBucket); + metadataUpdater = + new TestingMetadataUpdater( + Collections.singletonMap(tableInfo.getTablePath(), tableInfo)); + metadataUpdater.updateCluster(partitionedCluster(tableInfo, tableBucketsByPath)); + sender = setupWithIdempotenceState(); + + CompletableFuture activeFuture = + appendKvRecord(tableInfo, activePath, 1, metadataUpdater.getCluster()); + accumulator.tryRouteWritesTo( + firstOriginalPath, historicalPath, historicalBucket.getPartitionId()); + accumulator.tryRouteWritesTo( + secondOriginalPath, historicalPath, historicalBucket.getPartitionId()); + CompletableFuture firstHistoricalFuture = + appendKvRecord(tableInfo, firstOriginalPath, 2, metadataUpdater.getCluster()); + CompletableFuture secondHistoricalFuture = + appendKvRecord(tableInfo, secondOriginalPath, 3, metadataUpdater.getCluster()); + + sender.runOnce(); + + TestTabletServerGateway gateway = node1Gateway(); + assertThat(gateway.pendingRequestSize()).isEqualTo(2); + + PutKvRequest normalRequest = (PutKvRequest) gateway.getRequest(0); + assertThat(normalRequest.getBucketsReqsCount()).isOne(); + assertThat(normalRequest.getBucketsReqAt(0).getPartitionId()) + .isEqualTo(activeBucket.getPartitionId()); + assertThat(normalRequest.getBucketsReqAt(0).hasOriginalPartitionName()).isFalse(); + + PutKvRequest historicalRequest = (PutKvRequest) gateway.getRequest(1); + assertThat(historicalRequest.getBucketsReqsCount()).isEqualTo(2); + Set originalPartitionNames = new HashSet<>(); + for (int i = 0; i < historicalRequest.getBucketsReqsCount(); i++) { + assertThat(historicalRequest.getBucketsReqAt(i).getPartitionId()) + .isEqualTo(historicalBucket.getPartitionId()); + originalPartitionNames.add( + historicalRequest.getBucketsReqAt(i).getOriginalPartitionName()); + } + assertThat(originalPartitionNames) + .containsExactlyInAnyOrder( + firstOriginalPath.getPartitionName(), + secondOriginalPath.getPartitionName()); + + gateway.response(0, createPutKvResponse(activeBucket, 1L)); + gateway.response( + 0, + makePutKvResponse( + Arrays.asList( + PutKvResultForBucket.historicalSuccess( + historicalBucket, + 1L, + secondOriginalPath.getPartitionName()), + PutKvResultForBucket.historicalSuccess( + historicalBucket, + 1L, + firstOriginalPath.getPartitionName())))); + assertThat(activeFuture).isDone(); + assertThat(firstHistoricalFuture).isDone(); + assertThat(secondHistoricalFuture).isDone(); + assertThat(activeFuture.get()).isNull(); + assertThat(firstHistoricalFuture.get()).isNull(); + assertThat(secondHistoricalFuture.get()).isNull(); + } + + @Test + void testPackHistoricalProduceLogRequests() throws Exception { + sender.destroyResources(); + TableInfo tableInfo = createHistoricalLogTableInfo(); + PhysicalTablePath firstOriginalPath = + PhysicalTablePath.of(tableInfo.getTablePath(), "20000101"); + PhysicalTablePath secondOriginalPath = + PhysicalTablePath.of(tableInfo.getTablePath(), "20000102"); + PhysicalTablePath historicalPath = + PhysicalTablePath.of(tableInfo.getTablePath(), HISTORICAL_PARTITION_VALUE); + TableBucket historicalBucket = new TableBucket(tableInfo.getTableId(), 22L, 0); + + metadataUpdater = + new TestingMetadataUpdater( + Collections.singletonMap(tableInfo.getTablePath(), tableInfo)); + metadataUpdater.updateCluster( + partitionedCluster( + tableInfo, Collections.singletonMap(historicalPath, historicalBucket))); + sender = setupWithIdempotenceState(); + + accumulator.tryRouteWritesTo( + firstOriginalPath, historicalPath, historicalBucket.getPartitionId()); + accumulator.tryRouteWritesTo( + secondOriginalPath, historicalPath, historicalBucket.getPartitionId()); + CompletableFuture firstFuture = + appendLogRecord(tableInfo, firstOriginalPath, 1, metadataUpdater.getCluster()); + CompletableFuture secondFuture = + appendLogRecord(tableInfo, secondOriginalPath, 2, metadataUpdater.getCluster()); + + sender.runOnce(); + + TestTabletServerGateway gateway = node1Gateway(); + assertThat(gateway.pendingRequestSize()).isOne(); + ProduceLogRequest request = (ProduceLogRequest) gateway.getRequest(0); + assertThat(request.getBucketsReqsCount()).isEqualTo(2); + Set originalPartitionNames = new HashSet<>(); + for (int i = 0; i < request.getBucketsReqsCount(); i++) { + assertThat(request.getBucketsReqAt(i).getPartitionId()) + .isEqualTo(historicalBucket.getPartitionId()); + originalPartitionNames.add(request.getBucketsReqAt(i).getOriginalPartitionName()); + } + assertThat(originalPartitionNames) + .containsExactlyInAnyOrder( + firstOriginalPath.getPartitionName(), + secondOriginalPath.getPartitionName()); + + gateway.response( + 0, + makeProduceLogResponse( + Arrays.asList( + ProduceLogResultForBucket.historicalSuccess( + historicalBucket, + 1L, + 2L, + secondOriginalPath.getPartitionName()), + ProduceLogResultForBucket.historicalSuccess( + historicalBucket, + 0L, + 1L, + firstOriginalPath.getPartitionName())))); + assertThat(firstFuture).isDone(); + assertThat(secondFuture).isDone(); + assertThat(firstFuture.get()).isNull(); + assertThat(secondFuture.get()).isNull(); + } + @Test void testSimple() throws Exception { long offset = 0; @@ -1263,6 +1534,159 @@ private void resetTableInfosWith(TableInfo tableInfo) { metadataUpdater.updateTableInfos(tableInfos); } + private static TableInfo createHistoricalTableInfo() { + return createHistoricalTableInfo(AutoPartitionTimeUnit.DAY, 7); + } + + private static TableInfo createHistoricalTableInfo( + AutoPartitionTimeUnit timeUnit, int numToRetain) { + return createPartitionedKvTableInfo(timeUnit, numToRetain, true); + } + + private static TableInfo createNormalPartitionedTableInfo() { + return createPartitionedKvTableInfo(AutoPartitionTimeUnit.DAY, 7, false); + } + + private static TableInfo createPartitionedKvTableInfo( + AutoPartitionTimeUnit timeUnit, int numToRetain, boolean historicalPartitionEnabled) { + Schema schema = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("dt", DataTypes.STRING()) + .primaryKey("id", "dt") + .build(); + TableDescriptor descriptor = + TableDescriptor.builder() + .schema(schema) + .partitionedBy("dt") + .distributedBy(1) + .property(ConfigOptions.TABLE_AUTO_PARTITION_ENABLED, true) + .property(ConfigOptions.TABLE_AUTO_PARTITION_TIME_UNIT, timeUnit) + .property(ConfigOptions.TABLE_AUTO_PARTITION_NUM_RETENTION, numToRetain) + .property(ConfigOptions.TABLE_AUTO_PARTITION_TIMEZONE, "UTC") + .property(ConfigOptions.TABLE_DATALAKE_ENABLED, true) + .property(ConfigOptions.TABLE_DATALAKE_FORMAT, DataLakeFormat.PAIMON) + .property( + ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_ENABLED, + historicalPartitionEnabled) + .build(); + return TableInfo.of( + DATA1_TABLE_PATH_PK, + DATA1_TABLE_ID_PK, + 1, + descriptor, + DEFAULT_REMOTE_DATA_DIR, + 1L, + 1L); + } + + private static TestingMetadataUpdater missingPartitionMetadataUpdater(TableInfo tableInfo) { + return new TestingMetadataUpdater( + Collections.singletonMap(tableInfo.getTablePath(), tableInfo)) { + @Override + public void updatePhysicalTableMetadata(Set physicalTablePaths) { + throw new PartitionNotExistException("Partition does not exist."); + } + + @Override + public boolean checkAndUpdatePartitionMetadata(PhysicalTablePath physicalTablePath) { + throw new PartitionNotExistException("Partition does not exist."); + } + }; + } + + private static TableInfo createHistoricalLogTableInfo() { + Schema schema = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("dt", DataTypes.STRING()) + .build(); + TableDescriptor descriptor = + TableDescriptor.builder() + .schema(schema) + .partitionedBy("dt") + .distributedBy(1, "id") + .property(ConfigOptions.TABLE_AUTO_PARTITION_ENABLED, true) + .property(ConfigOptions.TABLE_DATALAKE_ENABLED, true) + .property(ConfigOptions.TABLE_DATALAKE_FORMAT, DataLakeFormat.PAIMON) + .property(ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_ENABLED, true) + .build(); + return TableInfo.of( + DATA1_TABLE_PATH, DATA1_TABLE_ID, 1, descriptor, DEFAULT_REMOTE_DATA_DIR, 1L, 1L); + } + + private static Cluster partitionedCluster( + TableInfo tableInfo, Map tableBucketsByPath) { + int[] replicas = new int[] {TestingMetadataUpdater.NODE1.id()}; + Map> bucketLocationsByPath = new HashMap<>(); + Map partitionIdsByPath = new HashMap<>(); + tableBucketsByPath.forEach( + (physicalTablePath, tableBucket) -> { + bucketLocationsByPath.put( + physicalTablePath, + Collections.singletonList( + new BucketLocation( + physicalTablePath, + tableBucket, + TestingMetadataUpdater.NODE1.id(), + replicas))); + partitionIdsByPath.put(physicalTablePath, tableBucket.getPartitionId()); + }); + return new Cluster( + Collections.singletonMap( + TestingMetadataUpdater.NODE1.id(), TestingMetadataUpdater.NODE1), + TestingMetadataUpdater.COORDINATOR, + bucketLocationsByPath, + Collections.singletonMap(tableInfo.getTablePath(), tableInfo.getTableId()), + partitionIdsByPath); + } + + private CompletableFuture appendKvRecord( + TableInfo tableInfo, PhysicalTablePath physicalTablePath, int id, Cluster cluster) + throws Exception { + BinaryRow row = + compactedRow( + tableInfo.getRowType(), + new Object[] {id, physicalTablePath.getPartitionName()}); + byte[] key = + new CompactedKeyEncoder( + tableInfo.getRowType(), + tableInfo.getSchema().getPrimaryKeyIndexes()) + .encodeKey(row); + CompletableFuture future = new CompletableFuture<>(); + accumulator.append( + WriteRecord.forUpsert( + tableInfo, + physicalTablePath, + row, + key, + key, + WriteFormat.COMPACTED_KV, + null), + (tableBucket, logEndOffset, error) -> future.complete(error), + cluster, + 0, + false); + return future; + } + + private CompletableFuture appendLogRecord( + TableInfo tableInfo, PhysicalTablePath physicalTablePath, int id, Cluster cluster) + throws Exception { + IndexedRow row = + indexedRow( + tableInfo.getRowType(), + new Object[] {id, physicalTablePath.getPartitionName()}); + CompletableFuture future = new CompletableFuture<>(); + accumulator.append( + WriteRecord.forIndexedAppend(tableInfo, physicalTablePath, row, null), + (tableBucket, logEndOffset, error) -> future.complete(error), + cluster, + 0, + false); + return future; + } + private void appendToAccumulator(TableBucket tb, GenericRow row, WriteCallback writeCallback) throws Exception { appendToAccumulator(DATA1_TABLE_INFO, tb, row, writeCallback); @@ -1308,6 +1732,11 @@ private void appendKvToAccumulator( false); } + private TestTabletServerGateway node1Gateway() { + return (TestTabletServerGateway) + metadataUpdater.newTabletServerClientForNode(TestingMetadataUpdater.NODE1.id()); + } + private ApiMessage getRequest(TableBucket tb, int index) { TestTabletServerGateway gateway = (TestTabletServerGateway) @@ -1400,6 +1829,14 @@ private PutKvResponse createPutKvResponse(TableBucket tb, long endOffset) { Collections.singletonList(new PutKvResultForBucket(tb, endOffset))); } + private PutKvResponse createHistoricalPutKvResponse( + TableBucket tb, long endOffset, String originalPartitionName) { + return makePutKvResponse( + Collections.singletonList( + PutKvResultForBucket.historicalSuccess( + tb, endOffset, originalPartitionName))); + } + private PutKvResponse createPutKvResponse(TableBucket tb, long endOffset, float pressure) { return makePutKvResponse( Collections.singletonList(new PutKvResultForBucket(tb, endOffset, pressure))); @@ -1448,14 +1885,24 @@ private IdempotenceManager createIdempotenceManager(boolean idempotenceEnabled) } private static boolean hasIdempotentRecords(TableBucket tb, ProduceLogRequest request) { - MemoryLogRecords memoryLogRecords = getProduceLogData(request).get(tb); + MemoryLogRecords memoryLogRecords = getProduceLogRecords(request, tb); return memoryLogRecords.batchIterator().next().writerId() != NO_WRITER_ID; } private static void assertBatchSequenceEquals( TableBucket tb, ProduceLogRequest request, int expectedBatchSequence) { - MemoryLogRecords memoryLogRecords = getProduceLogData(request).get(tb); + MemoryLogRecords memoryLogRecords = getProduceLogRecords(request, tb); assertThat(memoryLogRecords.batchIterator().next().batchSequence()) .isEqualTo(expectedBatchSequence); } + + private static MemoryLogRecords getProduceLogRecords( + ProduceLogRequest request, TableBucket tableBucket) { + for (ProduceLogDataForBucket bucketData : toProduceLogDataForBuckets(request)) { + if (bucketData.tableBucket().equals(tableBucket)) { + return bucketData.records(); + } + } + throw new IllegalArgumentException("No records found for table bucket " + tableBucket); + } } diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java index 19e71dfee21..b0f780e6ae8 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java @@ -439,6 +439,14 @@ public class ConfigOptions { .withDescription( "The duration after which an idle historical partition table lookuper is removed from the cache."); + public static final ConfigOption SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME = + key("server.historical-partition.kv-cleanup.idle-time") + .durationType() + .defaultValue(Duration.ofMinutes(30)) + .withDescription( + "The historical KV write idle time after which a fully tiered local overlay can be cleaned. " + + "Set to 0 to disable idle cleanup."); + public static final ConfigOption SERVER_DATA_DISK_WRITE_LIMIT_RATIO = key("server.data-disk.write-limit-ratio") .doubleType() @@ -1924,11 +1932,12 @@ public class ConfigOptions { .booleanType() .defaultValue(false) .withDescription( - "Whether to enable historical partition lookup for the table. " + "Whether to enable historical partition access for the table. " + "When enabled, the coordinator creates and retains a system partition " - + "for routing lookups of expired partitions to lake storage. " - + "Currently, this option only supports auto-partitioned Paimon primary " - + "key tables with a single partition key. Disabled by default. " + + "for routing writes to expired partitions and, for primary-key tables, " + + "lookups of expired partitions to lake storage. Currently, this option " + + "only supports auto-partitioned Paimon tables with a single partition " + + "key. Disabled by default. " + "After changing this option, restart existing lookup jobs that need " + "to look up historical partition data so that their clients load the " + "updated table configuration."); diff --git a/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java b/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java index ea046e3b11d..d1b41983768 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java @@ -129,7 +129,7 @@ public boolean isDataLakeEnabled() { return config.get(ConfigOptions.TABLE_DATALAKE_ENABLED); } - /** Whether historical partition lookup is enabled. */ + /** Whether historical partition access is enabled. */ public boolean isHistoricalPartitionEnabled() { return config.get(ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_ENABLED); } diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/enumerator/TieringSourceEnumerator.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/enumerator/TieringSourceEnumerator.java index f1bf98527af..eb27397c9c6 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/enumerator/TieringSourceEnumerator.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/enumerator/TieringSourceEnumerator.java @@ -158,7 +158,7 @@ public void start() { this.coordinatorGateway = GatewayClientProxy.createGatewayProxy( metadataUpdater::getCoordinatorServer, rpcClient, CoordinatorGateway.class); - this.splitGenerator = new TieringSplitGenerator(flussAdmin); + this.splitGenerator = new TieringSplitGenerator(flussAdmin, metadataUpdater); LOG.info("Starting register Tiering Service to Fluss Coordinator..."); try { diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/split/TieringSplitGenerator.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/split/TieringSplitGenerator.java index a4b2638f309..fc2cbd9a03d 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/split/TieringSplitGenerator.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/split/TieringSplitGenerator.java @@ -22,8 +22,10 @@ import org.apache.fluss.client.initializer.OffsetsInitializer.BucketOffsetsRetriever; import org.apache.fluss.client.metadata.KvSnapshots; import org.apache.fluss.client.metadata.LakeSnapshot; +import org.apache.fluss.client.metadata.MetadataUpdater; import org.apache.fluss.exception.LakeTableSnapshotNotExistException; import org.apache.fluss.metadata.PartitionInfo; +import org.apache.fluss.metadata.PhysicalTablePath; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.metadata.TablePath; @@ -36,6 +38,7 @@ import javax.annotation.Nullable; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; @@ -43,6 +46,7 @@ import java.util.stream.IntStream; import static org.apache.fluss.client.table.scanner.log.LogScanner.EARLIEST_OFFSET; +import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; import static org.apache.fluss.utils.Preconditions.checkState; /** A generator for lake splits. */ @@ -51,9 +55,11 @@ public class TieringSplitGenerator { private static final Logger LOG = LoggerFactory.getLogger(TieringSplitGenerator.class); private final Admin flussAdmin; + private final MetadataUpdater metadataUpdater; - public TieringSplitGenerator(Admin flussAdmin) { + public TieringSplitGenerator(Admin flussAdmin, MetadataUpdater metadataUpdater) { this.flussAdmin = flussAdmin; + this.metadataUpdater = metadataUpdater; } public List generateTableSplits(TableInfo tableInfo) throws Exception { @@ -92,6 +98,21 @@ public List generateTableSplits(TableInfo tableInfo) throws Except Collectors.toMap( PartitionInfo::getPartitionId, PartitionInfo::getPartitionName)); + if (tableInfo.getTableConfig().isHistoricalPartitionEnabled()) { + // The internal historical partition is intentionally omitted from + // listPartitionInfos(), but tiering must consume it to synchronize historical + // writes to the lake table. Resolve it explicitly and include it in the splits. + PhysicalTablePath historicalPath = + PhysicalTablePath.of(tablePath, HISTORICAL_PARTITION_VALUE); + // Partition metadata is decoded using the tableId-to-path mapping already present + // in the Cluster, so initialize the table metadata before requesting the internal + // partition directly. + metadataUpdater.checkAndUpdateTableMetadata(Collections.singleton(tablePath)); + metadataUpdater.checkAndUpdatePartitionMetadata(historicalPath); + partitionNameById.put( + metadataUpdater.getPartitionIdOrElseThrow(historicalPath), + HISTORICAL_PARTITION_VALUE); + } return generatePartitionTableSplit( tableInfo, partitionNameById, bucketOffsetsRetriever, lakeSnapshotInfo); @@ -112,6 +133,7 @@ private List generatePartitionTableSplit( for (Map.Entry partitionNameByIdEntry : partitionNameById.entrySet()) { long partitionId = partitionNameByIdEntry.getKey(); String partitionName = partitionNameByIdEntry.getValue(); + boolean historicalPartition = HISTORICAL_PARTITION_VALUE.equals(partitionName); Map latestBucketsOffset = bucketOffsetsRetriever.latestOffsets( partitionName, @@ -119,7 +141,7 @@ private List generatePartitionTableSplit( .boxed() .collect(Collectors.toList())); KvSnapshots latestKvSnapshots = null; - if (tableInfo.hasPrimaryKey()) { + if (tableInfo.hasPrimaryKey() && !historicalPartition) { // get the table partition latest kv snapshot info try { latestKvSnapshots = @@ -134,6 +156,8 @@ private List generatePartitionTableSplit( ExceptionUtils.stripCompletionException(e)); } } + // Historical KV replicas do not create regular KV snapshots. Their lake snapshot is + // the durable base, so tier them from the retained WAL like log tables. splits.addAll( generateTableSplit( @@ -142,7 +166,8 @@ private List generatePartitionTableSplit( partitionName, lakeSnapshotInfo, latestKvSnapshots, - latestBucketsOffset)); + latestBucketsOffset, + historicalPartition)); } return splits; } @@ -172,7 +197,13 @@ private List generateNonPartitionedTableSplit( } return generateTableSplit( - tableInfo, null, null, lakeSnapshotInfo, latestKvSnapshots, latestBucketsOffset); + tableInfo, + null, + null, + lakeSnapshotInfo, + latestKvSnapshots, + latestBucketsOffset, + false); } private List generateTableSplit( @@ -181,10 +212,11 @@ private List generateTableSplit( @Nullable String partitionName, @Nullable LakeSnapshot lakeSnapshotInfo, @Nullable KvSnapshots latestKvSnapshots, - Map latestBucketsOffset) { + Map latestBucketsOffset, + boolean historicalPartition) { List splits = new ArrayList<>(); - if (tableInfo.hasPrimaryKey()) { + if (tableInfo.hasPrimaryKey() && !historicalPartition) { // it's primary key table checkState(latestKvSnapshots != null); for (int bucket = 0; bucket < tableInfo.getNumBuckets(); bucket++) { diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeCommitter.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeCommitter.java index c23cc373cda..7b16280eaf8 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeCommitter.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeCommitter.java @@ -35,6 +35,7 @@ import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.TableSnapshot; import org.apache.paimon.table.sink.CommitCallback; +import org.apache.paimon.table.sink.CommitMessage; import org.apache.paimon.table.sink.TableCommitImpl; import org.apache.paimon.utils.SnapshotManager; import org.slf4j.Logger; @@ -91,7 +92,9 @@ public PaimonCommittable toCommittable(List paimonWriteResult throws IOException { ManifestCommittable committable = new ManifestCommittable(COMMIT_IDENTIFIER); for (PaimonWriteResult paimonWriteResult : paimonWriteResults) { - committable.addFileCommittable(paimonWriteResult.commitMessage()); + for (CommitMessage commitMessage : paimonWriteResult.commitMessages()) { + committable.addFileCommittable(commitMessage); + } } return new PaimonCommittable(committable); } diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeWriter.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeWriter.java index 9b82403b6ce..a4bb0d44a41 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeWriter.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeWriter.java @@ -32,7 +32,6 @@ import org.apache.paimon.CoreOptions; import org.apache.paimon.catalog.Catalog; import org.apache.paimon.table.FileStoreTable; -import org.apache.paimon.table.sink.CommitMessage; import java.io.IOException; import java.util.Collections; @@ -40,6 +39,7 @@ import java.util.Map; import static org.apache.fluss.lake.paimon.utils.PaimonConversions.toPaimon; +import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; /** Implementation of {@link LakeWriter} for Paimon. */ public class PaimonLakeWriter implements LakeWriter, SupportsRecordBatchWrite { @@ -58,6 +58,8 @@ public PaimonLakeWriter( List partitionKeys = fileStoreTable.partitionKeys(); RowType flussRowType = writerInitContext.tableInfo().getRowType(); + boolean historicalPartition = + HISTORICAL_PARTITION_VALUE.equals(writerInitContext.partition()); // FIP-27: detect whether the target Paimon table is a clean table (only user columns) or a // legacy table (carrying the three Fluss system columns). Writers emit system columns only @@ -72,7 +74,8 @@ public PaimonLakeWriter( writerInitContext.partition(), partitionKeys, flussRowType, - paimonIncludingSystemColumns) + paimonIncludingSystemColumns, + historicalPartition) : new MergeTreeWriter( fileStoreTable, writerInitContext.tableBucket(), @@ -80,7 +83,8 @@ public PaimonLakeWriter( partitionKeys, flussRowType, writerInitContext.ioTmpDirs(), - paimonIncludingSystemColumns); + paimonIncludingSystemColumns, + historicalPartition); } @Override @@ -113,13 +117,11 @@ public void write(RecordBatch recordBatch) throws IOException { @Override public PaimonWriteResult complete() throws IOException { - CommitMessage commitMessage; try { - commitMessage = recordWriter.complete(); + return new PaimonWriteResult(recordWriter.complete()); } catch (Exception e) { throw new IOException("Failed to complete Paimon write.", e); } - return new PaimonWriteResult(commitMessage); } @Override diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonWriteResult.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonWriteResult.java index 70575c00e16..b2d40fbb25f 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonWriteResult.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonWriteResult.java @@ -20,19 +20,29 @@ import org.apache.paimon.table.sink.CommitMessage; import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; /** The write result of Paimon lake writer to pass to committer to commit. */ -public class PaimonWriteResult implements Serializable { +public final class PaimonWriteResult implements Serializable { private static final long serialVersionUID = 1L; - private final CommitMessage commitMessage; + private final List commitMessages; - public PaimonWriteResult(CommitMessage commitMessage) { - this.commitMessage = commitMessage; + /** Creates a write result containing all commit messages produced by one lake writer. */ + public PaimonWriteResult(List commitMessages) { + checkNotNull(commitMessages, "commitMessages must not be null"); + checkArgument(!commitMessages.isEmpty(), "commitMessages must not be empty"); + this.commitMessages = Collections.unmodifiableList(new ArrayList<>(commitMessages)); } - public CommitMessage commitMessage() { - return commitMessage; + /** Returns all commit messages produced by the lake writer. */ + public List commitMessages() { + return commitMessages; } } diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonWriteResultSerializer.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonWriteResultSerializer.java index 7efb3d37548..3ffecc9855d 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonWriteResultSerializer.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonWriteResultSerializer.java @@ -19,10 +19,14 @@ import org.apache.fluss.lake.serializer.SimpleVersionedSerializer; +import org.apache.paimon.io.DataInputViewStreamWrapper; +import org.apache.paimon.io.DataOutputSerializer; import org.apache.paimon.table.sink.CommitMessage; import org.apache.paimon.table.sink.CommitMessageSerializer; +import java.io.ByteArrayInputStream; import java.io.IOException; +import java.util.List; /** The {@link SimpleVersionedSerializer} for {@link PaimonWriteResult}. */ public class PaimonWriteResultSerializer implements SimpleVersionedSerializer { @@ -38,8 +42,9 @@ public int getVersion() { @Override public byte[] serialize(PaimonWriteResult paimonWriteResult) throws IOException { - CommitMessage commitMessage = paimonWriteResult.commitMessage(); - return messageSer.serialize(commitMessage); + DataOutputSerializer output = new DataOutputSerializer(64); + messageSer.serializeList(paimonWriteResult.commitMessages(), output); + return output.getCopyOfBuffer(); } @Override @@ -52,7 +57,11 @@ public PaimonWriteResult deserialize(int version, byte[] serialized) throws IOEx + version + "."); } - CommitMessage commitMessage = messageSer.deserialize(messageSer.getVersion(), serialized); - return new PaimonWriteResult(commitMessage); + try (DataInputViewStreamWrapper input = + new DataInputViewStreamWrapper(new ByteArrayInputStream(serialized))) { + List commitMessages = + messageSer.deserializeList(messageSer.getVersion(), input); + return new PaimonWriteResult(commitMessages); + } } } diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/RecordWriter.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/RecordWriter.java index 2260d553bc9..6e762a96141 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/RecordWriter.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/RecordWriter.java @@ -31,6 +31,7 @@ import java.util.List; import static org.apache.fluss.lake.paimon.utils.PaimonConversions.toPaimonPartition; +import static org.apache.fluss.utils.Preconditions.checkNotNull; import static org.apache.fluss.utils.Preconditions.checkState; /** A base interface to write {@link LogRecord} to Paimon. */ @@ -40,7 +41,8 @@ public abstract class RecordWriter implements AutoCloseable { protected final RowType tableRowType; protected final int bucket; protected final List partitionKeys; - protected final BinaryRow partition; + protected final boolean historicalPartition; + protected final @Nullable BinaryRow fixedPartition; protected final FlussRecordAsPaimonRow flussRecordAsPaimonRow; public RecordWriter( @@ -50,17 +52,21 @@ public RecordWriter( @Nullable String partition, List partitionKeys, org.apache.fluss.types.RowType flussRowType, - boolean paimonIncludingSystemColumns) { + boolean paimonIncludingSystemColumns, + boolean historicalPartition) { this.tableWrite = tableWrite; this.tableRowType = tableRowType; this.bucket = tableBucket.getBucket(); this.partitionKeys = partitionKeys; - if (partition == null || partitionKeys.isEmpty()) { + this.historicalPartition = historicalPartition; + if (historicalPartition) { + this.fixedPartition = null; + } else if (partition == null || partitionKeys.isEmpty()) { // non-partitioned table - this.partition = BinaryRow.EMPTY_ROW; + this.fixedPartition = BinaryRow.EMPTY_ROW; } else { // eagerly resolve BinaryRow partition from partition name string - this.partition = resolvePartition(partition, partitionKeys, flussRowType); + this.fixedPartition = resolvePartition(partition, partitionKeys, flussRowType); } this.flussRecordAsPaimonRow = new FlussRecordAsPaimonRow( @@ -69,19 +75,31 @@ public RecordWriter( public abstract void write(LogRecord record) throws Exception; - CommitMessage complete() throws Exception { + List complete() throws Exception { List commitMessages = tableWrite.prepareCommit(); - checkState( - commitMessages.size() == 1, - "The size of CommitMessage must be 1, but got %s.", - commitMessages); - return commitMessages.get(0); + // A normal writer targets one fixed partition, while a historical writer may write to + // multiple original partitions and therefore produce multiple commit messages. + if (!historicalPartition) { + checkState( + commitMessages.size() == 1, + "The size of CommitMessage must be 1, but got %s.", + commitMessages); + } + return commitMessages; } public void close() throws Exception { tableWrite.close(); } + /** Sets the current Fluss record and returns the Paimon partition it should be written to. */ + protected BinaryRow prepareRecordAndGetPartition(LogRecord record) { + flussRecordAsPaimonRow.setFlussRecord(record); + return historicalPartition + ? tableWrite.getPartition(flussRecordAsPaimonRow) + : checkNotNull(fixedPartition); + } + /** * Resolves a Paimon {@link BinaryRow} partition from the partition name string by parsing each * partition value to its typed Fluss representation, constructing a synthetic row, and diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyArrowBatchHelper.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyArrowBatchHelper.java index ac3b75cad3b..83518179ddf 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyArrowBatchHelper.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyArrowBatchHelper.java @@ -44,6 +44,8 @@ import java.util.ArrayList; import java.util.List; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + /** * Helper class that encapsulates Arrow-dependent batch writing logic for append-only tables. * @@ -106,7 +108,11 @@ class AppendOnlyArrowBatchHelper implements AutoCloseable { * system columns (__bucket, __offset, __timestamp) and uses Paimon's {@link ArrowBundleRecords} * for efficient batch writing. */ - void writeArrowBatch(ArrowBatchData arrowBatchData, BinaryRow partition) throws Exception { + void writeArrowBatch( + ArrowBatchData arrowBatchData, + @Nullable BinaryRow fixedPartition, + boolean historicalPartition) + throws Exception { int writtenBucket = bucket; if (fileStoreTable.store().bucketMode() == BucketMode.BUCKET_UNAWARE) { writtenBucket = 0; @@ -119,7 +125,7 @@ void writeArrowBatch(ArrowBatchData arrowBatchData, BinaryRow partition) throws // the Paimon table schema. Write it directly without enriching system columns. ArrowBundleRecords cleanRecords = new ArrowBundleRecords(originalRoot, tableRowType, CASE_SENSITIVE); - tableWrite.writeBundle(partition, writtenBucket, cleanRecords); + writeArrowBundle(cleanRecords, fixedPartition, historicalPartition, writtenBucket); return; } @@ -133,7 +139,25 @@ void writeArrowBatch(ArrowBatchData arrowBatchData, BinaryRow partition) throws ArrowBundleRecords arrowBundleRecords = new ArrowBundleRecords(enrichedRoot, tableRowType, CASE_SENSITIVE); - tableWrite.writeBundle(partition, writtenBucket, arrowBundleRecords); + writeArrowBundle(arrowBundleRecords, fixedPartition, historicalPartition, writtenBucket); + } + + private void writeArrowBundle( + ArrowBundleRecords arrowBundleRecords, + @Nullable BinaryRow fixedPartition, + boolean historicalPartition, + int writtenBucket) + throws Exception { + if (historicalPartition) { + // writeBundle accepts one fixed partition, but a historical batch may contain rows + // from multiple original partitions. + for (InternalRow row : arrowBundleRecords) { + BinaryRow partition = tableWrite.getPartition(row); + tableWrite.getWrite().write(partition, writtenBucket, row); + } + } else { + tableWrite.writeBundle(checkNotNull(fixedPartition), writtenBucket, arrowBundleRecords); + } } /** diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyWriter.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyWriter.java index 23f61a33171..a6d4f4292fd 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyWriter.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyWriter.java @@ -23,6 +23,7 @@ import org.apache.fluss.record.LogRecord; import org.apache.fluss.types.RowType; +import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.table.BucketMode; import org.apache.paimon.table.FileStoreTable; @@ -54,7 +55,8 @@ public AppendOnlyWriter( @Nullable String partition, List partitionKeys, RowType flussRowType, - boolean paimonIncludingSystemColumns) { + boolean paimonIncludingSystemColumns, + boolean historicalPartition) { //noinspection unchecked super( (TableWriteImpl) @@ -65,14 +67,15 @@ public AppendOnlyWriter( partition, partitionKeys, flussRowType, - paimonIncludingSystemColumns); + paimonIncludingSystemColumns, + historicalPartition); this.fileStoreTable = fileStoreTable; this.paimonIncludingSystemColumns = paimonIncludingSystemColumns; } @Override public void write(LogRecord record) throws Exception { - flussRecordAsPaimonRow.setFlussRecord(record); + BinaryRow targetPartition = prepareRecordAndGetPartition(record); // hacky, call internal method tableWrite.getWrite() to support // to write to given partition, otherwise, it'll always extract a partition from Paimon row @@ -82,7 +85,7 @@ public void write(LogRecord record) throws Exception { if (fileStoreTable.store().bucketMode() == BucketMode.BUCKET_UNAWARE) { writtenBucket = 0; } - tableWrite.getWrite().write(partition, writtenBucket, flussRecordAsPaimonRow); + tableWrite.getWrite().write(targetPartition, writtenBucket, flussRecordAsPaimonRow); } /** @@ -104,7 +107,7 @@ public void writeArrowBatch(ArrowBatchData arrowBatchData) throws Exception { } else { helper = (AppendOnlyArrowBatchHelper) arrowBatchHelper; } - helper.writeArrowBatch(arrowBatchData, partition); + helper.writeArrowBatch(arrowBatchData, fixedPartition, historicalPartition); } @Override diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/mergetree/MergeTreeWriter.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/mergetree/MergeTreeWriter.java index 37aeef7afe6..7d48c850e9a 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/mergetree/MergeTreeWriter.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/mergetree/MergeTreeWriter.java @@ -23,6 +23,7 @@ import org.apache.fluss.types.RowType; import org.apache.paimon.KeyValue; +import org.apache.paimon.data.BinaryRow; import org.apache.paimon.disk.IOManager; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.sink.RowKeyExtractor; @@ -58,7 +59,8 @@ public MergeTreeWriter( partitionKeys, flussRowType, (String[]) null, - paimonIncludingSystemColumns); + paimonIncludingSystemColumns, + false); } public MergeTreeWriter( @@ -68,7 +70,8 @@ public MergeTreeWriter( List partitionKeys, RowType flussRowType, @Nullable String[] ioTmpDirs, - boolean paimonIncludingSystemColumns) { + boolean paimonIncludingSystemColumns, + boolean historicalPartition) { this( fileStoreTable, createIOManager(ioTmpDirs), @@ -76,7 +79,8 @@ public MergeTreeWriter( partition, partitionKeys, flussRowType, - paimonIncludingSystemColumns); + paimonIncludingSystemColumns, + historicalPartition); } MergeTreeWriter( @@ -86,7 +90,8 @@ public MergeTreeWriter( @Nullable String partition, List partitionKeys, RowType flussRowType, - boolean paimonIncludingSystemColumns) { + boolean paimonIncludingSystemColumns, + boolean historicalPartition) { super( createTableWrite(fileStoreTable, ioManager), fileStoreTable.rowType(), @@ -94,7 +99,8 @@ public MergeTreeWriter( partition, partitionKeys, flussRowType, - paimonIncludingSystemColumns); + paimonIncludingSystemColumns, + historicalPartition); this.rowKeyExtractor = fileStoreTable.createRowKeyExtractor(); this.ioManager = ioManager; } @@ -128,7 +134,7 @@ public void close() throws Exception { @Override public void write(LogRecord record) throws Exception { - flussRecordAsPaimonRow.setFlussRecord(record); + BinaryRow targetPartition = prepareRecordAndGetPartition(record); rowKeyExtractor.setRecord(flussRecordAsPaimonRow); keyValue.replace( @@ -139,6 +145,6 @@ public void write(LogRecord record) throws Exception { // hacky, call internal method tableWrite.getWrite() to support // to write to given partition, otherwise, it'll always extract a partition from Paimon row // which may be costly - tableWrite.getWrite().write(partition, bucket, keyValue); + tableWrite.getWrite().write(targetPartition, bucket, keyValue); } } diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/HistoricalPartitionLookupITCase.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/HistoricalPartitionITCase.java similarity index 64% rename from fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/HistoricalPartitionLookupITCase.java rename to fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/HistoricalPartitionITCase.java index 091c72bc21b..a135d520162 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/HistoricalPartitionLookupITCase.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/HistoricalPartitionITCase.java @@ -38,12 +38,15 @@ import org.apache.fluss.types.DataTypes; import org.apache.flink.core.execution.JobClient; +import org.apache.paimon.utils.CloseableIterator; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; import java.time.Duration; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -56,8 +59,8 @@ import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; import static org.assertj.core.api.Assertions.assertThat; -/** End-to-end IT case for looking up expired Fluss partitions from Paimon. */ -class HistoricalPartitionLookupITCase extends FlinkPaimonTieringTestBase { +/** End-to-end IT case for historical partition writes, tiering, recovery, and lookup. */ +class HistoricalPartitionITCase extends FlinkPaimonTieringTestBase { private static final String EXPIRED_PARTITION_NAME = "20240101"; private static final String SECOND_EXPIRED_PARTITION_NAME = "20240102"; @@ -76,6 +79,87 @@ protected static void beforeAll() { FlinkPaimonTieringTestBase.beforeAll(FLUSS_CLUSTER_EXTENSION.getClientConfig()); } + @Test + void testWriteAndTierHistoricalKvToPaimon() throws Exception { + TablePath tablePath = TablePath.of(DEFAULT_DB, "historical_write_tiering"); + Schema schema = partitionedPkSchema(true); + long tableId = + createTable( + tablePath, + partitionedPkDescriptor(schema, true, EXPIRED_PARTITION_RETENTION)); + + try { + long historicalPartitionId = waitUntilHistoricalPartitionReady(tablePath, tableId); + + InternalRow expectedRow = dataRow(true, 1, "unused", "Alice"); + assertThat(admin.listPartitionInfos(tablePath).get()) + .noneMatch(p -> EXPIRED_PARTITION_NAME.equals(p.getPartitionName())); + writeRows(tablePath, Collections.singletonList(expectedRow), false); + // Historical writes must not recreate the expired original partition. + assertThat(admin.listPartitionInfos(tablePath).get()) + .noneMatch(p -> EXPIRED_PARTITION_NAME.equals(p.getPartitionName())); + + TableBucket historicalBucket = new TableBucket(tableId, historicalPartitionId, 0); + assertThat(getLeaderReplica(historicalBucket).getLocalLogEndOffset()).isEqualTo(1); + JobClient jobClient = buildTieringJob(execEnv); + try { + assertReplicaStatus(historicalBucket, 1); + checkFlussOffsetsInSnapshot( + tablePath, Collections.singletonMap(historicalBucket, 1L)); + assertThat(readPaimonRows(tablePath)) + .containsExactly("1|" + EXPIRED_PARTITION_NAME + "|Alice"); + } finally { + jobClient.cancel().get(); + } + + restartLeaderAndVerifyLookup(tablePath, historicalBucket, schema, expectedRow); + } finally { + dropTable(tablePath); + } + } + + @Test + void testWriteAndTierHistoricalLogToPaimon() throws Exception { + TablePath tablePath = TablePath.of(DEFAULT_DB, "historical_log_write_tiering"); + Schema schema = partitionedLogSchema(); + long tableId = createTable(tablePath, partitionedLogDescriptor(schema)); + + try { + long historicalPartitionId = waitUntilHistoricalPartitionReady(tablePath, tableId); + + List expectedRows = + Arrays.asList( + row(1, EXPIRED_PARTITION_NAME, "Alice"), + row(2, SECOND_EXPIRED_PARTITION_NAME, "Bob")); + writeRows(tablePath, expectedRows, true); + assertThat(admin.listPartitionInfos(tablePath).get()) + .noneMatch( + partitionInfo -> + EXPIRED_PARTITION_NAME.equals(partitionInfo.getPartitionName()) + || SECOND_EXPIRED_PARTITION_NAME.equals( + partitionInfo.getPartitionName())); + + TableBucket historicalBucket = new TableBucket(tableId, historicalPartitionId, 0); + assertThat(getLeaderReplica(historicalBucket).getLocalLogEndOffset()).isEqualTo(2); + + JobClient jobClient = buildTieringJob(execEnv); + try { + assertReplicaStatus(historicalBucket, 2); + checkFlussOffsetsInSnapshot( + tablePath, Collections.singletonMap(historicalBucket, 2L)); + + assertThat(readPaimonRows(tablePath)) + .containsExactlyInAnyOrder( + "1|" + EXPIRED_PARTITION_NAME + "|Alice", + "2|" + SECOND_EXPIRED_PARTITION_NAME + "|Bob"); + } finally { + jobClient.cancel().get(); + } + } finally { + dropTable(tablePath); + } + } + @ParameterizedTest(name = "defaultBucketKey={0}") @ValueSource(booleans = {true, false}) void testLookupExpiredPartitionFromPaimon(boolean defaultBucketKey) throws Exception { @@ -86,7 +170,7 @@ void testLookupExpiredPartitionFromPaimon(boolean defaultBucketKey) throws Excep ? "historical_lookup_default_bucket" : "historical_lookup_bucket_subset"); Schema oldSchema = partitionedPkSchema(defaultBucketKey); - long tableId = createTable(tablePath, partitionedPkDescriptor(oldSchema)); + long tableId = createTable(tablePath, partitionedPkDescriptor(oldSchema, false)); // Enable historical lookup through ALTER TABLE to cover dynamic creation of the // coordinator-owned historical system partition. @@ -232,6 +316,59 @@ protected FlussClusterExtension getFlussClusterExtension() { return FLUSS_CLUSTER_EXTENSION; } + private static long waitUntilHistoricalPartitionReady(TablePath tablePath, long tableId) + throws Exception { + Optional historicalPartition = + FLUSS_CLUSTER_EXTENSION + .getZooKeeperClient() + .getPartition(tablePath, HISTORICAL_PARTITION_VALUE); + assertThat(historicalPartition).isPresent(); + long partitionId = historicalPartition.get().getPartitionId(); + FLUSS_CLUSTER_EXTENSION.waitUntilTablePartitionReady(tableId, partitionId); + return partitionId; + } + + private List readPaimonRows(TablePath tablePath) throws Exception { + List actualRows = new ArrayList<>(); + try (CloseableIterator rows = + getPaimonRowCloseableIterator(tablePath)) { + while (rows.hasNext()) { + org.apache.paimon.data.InternalRow row = rows.next(); + actualRows.add(row.getInt(0) + "|" + row.getString(1) + "|" + row.getString(2)); + } + } + return actualRows; + } + + private void restartLeaderAndVerifyLookup( + TablePath tablePath, + TableBucket historicalBucket, + Schema schema, + InternalRow expectedRow) + throws Exception { + int tabletServerId = FLUSS_CLUSTER_EXTENSION.waitAndGetLeader(historicalBucket); + FLUSS_CLUSTER_EXTENSION.stopTabletServer(tabletServerId); + try { + FLUSS_CLUSTER_EXTENSION.startTabletServer(tabletServerId); + FLUSS_CLUSTER_EXTENSION.waitAndGetLeaderReplica(historicalBucket); + + try (Connection connection = ConnectionFactory.createConnection(clientConf); + Table table = connection.getTable(tablePath)) { + InternalRow actualRow = + table.newLookup() + .createLookuper() + .lookup(lookupKey(true, 1, "unused")) + .get() + .getSingletonRow(); + assertThatRow(actualRow).withSchema(schema.getRowType()).isEqualTo(expectedRow); + } + } finally { + if (FLUSS_CLUSTER_EXTENSION.getTabletServerById(tabletServerId) == null) { + FLUSS_CLUSTER_EXTENSION.startTabletServer(tabletServerId); + } + } + } + private static Schema partitionedPkSchema(boolean defaultBucketKey) { if (defaultBucketKey) { return Schema.newBuilder() @@ -250,6 +387,14 @@ private static Schema partitionedPkSchema(boolean defaultBucketKey) { .build(); } + private static Schema partitionedLogSchema() { + return Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("dt", DataTypes.STRING()) + .column("name", DataTypes.STRING()) + .build(); + } + private static Schema evolvedPartitionedPkSchema(boolean defaultBucketKey) { if (defaultBucketKey) { return Schema.newBuilder() @@ -270,11 +415,41 @@ private static Schema evolvedPartitionedPkSchema(boolean defaultBucketKey) { .build(); } - private static TableDescriptor partitionedPkDescriptor(Schema schema) { + private static TableDescriptor partitionedPkDescriptor( + Schema schema, boolean historicalPartitionEnabled) { + return partitionedPkDescriptor( + schema, historicalPartitionEnabled, INITIAL_PARTITION_RETENTION); + } + + private static TableDescriptor partitionedPkDescriptor( + Schema schema, boolean historicalPartitionEnabled, int partitionRetention) { + TableDescriptor.Builder builder = + TableDescriptor.builder() + .schema(schema) + // This is the default bucket key for (id, dt), and a strict subset of the + // physical primary key for (id, sub_id, dt). + .distributedBy(1, "id") + .partitionedBy("dt") + .property(ConfigOptions.TABLE_AUTO_PARTITION_ENABLED, true) + .property(ConfigOptions.TABLE_AUTO_PARTITION_KEY, "dt") + .property( + ConfigOptions.TABLE_AUTO_PARTITION_TIME_UNIT, + AutoPartitionTimeUnit.DAY) + .property( + ConfigOptions.TABLE_AUTO_PARTITION_NUM_RETENTION, + partitionRetention) + .property(ConfigOptions.TABLE_AUTO_PARTITION_TIMEZONE, "UTC") + .property(ConfigOptions.TABLE_DATALAKE_ENABLED, true) + .property(ConfigOptions.TABLE_DATALAKE_FRESHNESS, Duration.ofMillis(500)); + if (historicalPartitionEnabled) { + builder.property(ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_ENABLED, true); + } + return builder.build(); + } + + private static TableDescriptor partitionedLogDescriptor(Schema schema) { return TableDescriptor.builder() .schema(schema) - // This is the default bucket key for (id, dt), and a strict subset of the physical - // primary key for (id, sub_id, dt). .distributedBy(1, "id") .partitionedBy("dt") .property(ConfigOptions.TABLE_AUTO_PARTITION_ENABLED, true) @@ -282,10 +457,11 @@ private static TableDescriptor partitionedPkDescriptor(Schema schema) { .property(ConfigOptions.TABLE_AUTO_PARTITION_TIME_UNIT, AutoPartitionTimeUnit.DAY) .property( ConfigOptions.TABLE_AUTO_PARTITION_NUM_RETENTION, - INITIAL_PARTITION_RETENTION) + EXPIRED_PARTITION_RETENTION) .property(ConfigOptions.TABLE_AUTO_PARTITION_TIMEZONE, "UTC") .property(ConfigOptions.TABLE_DATALAKE_ENABLED, true) .property(ConfigOptions.TABLE_DATALAKE_FRESHNESS, Duration.ofMillis(500)) + .property(ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_ENABLED, true) .build(); } diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringTest.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringTest.java index a433af06c82..e689778cf12 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringTest.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringTest.java @@ -17,25 +17,34 @@ package org.apache.fluss.lake.paimon.tiering; +import org.apache.fluss.config.AutoPartitionTimeUnit; import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; +import org.apache.fluss.lake.batch.ArrowRecordBatch; import org.apache.fluss.lake.committer.CommittedLakeSnapshot; import org.apache.fluss.lake.committer.CommitterInitContext; import org.apache.fluss.lake.committer.LakeCommitter; import org.apache.fluss.lake.serializer.SimpleVersionedSerializer; import org.apache.fluss.lake.writer.LakeWriter; +import org.apache.fluss.lake.writer.SupportsRecordBatchWrite; import org.apache.fluss.lake.writer.WriterInitContext; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.record.ArrowBatchData; import org.apache.fluss.record.ChangeType; import org.apache.fluss.record.GenericRecord; import org.apache.fluss.record.LogRecord; import org.apache.fluss.row.BinaryString; import org.apache.fluss.row.GenericRow; +import org.apache.fluss.utils.UnshadedArrowReadUtils; import org.apache.fluss.utils.types.Tuple2; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.paimon.CoreOptions; import org.apache.paimon.catalog.Catalog; import org.apache.paimon.catalog.CatalogContext; @@ -58,11 +67,13 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; import javax.annotation.Nullable; import java.io.File; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -82,6 +93,7 @@ import static org.apache.fluss.record.ChangeType.UPDATE_AFTER; import static org.apache.fluss.record.ChangeType.UPDATE_BEFORE; import static org.apache.fluss.record.TestData.DEFAULT_REMOTE_DATA_DIR; +import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; import static org.assertj.core.api.Assertions.assertThat; /** The UT for tiering to Paimon via {@link PaimonLakeTieringFactory}. */ @@ -212,6 +224,76 @@ void testTieringWriteTable(boolean isPrimaryKeyTable, boolean isPartitioned) thr } } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testHistoricalPartitionTiering(boolean isPrimaryKeyTable) throws Exception { + TablePath tablePath = + TablePath.of( + "paimon", "test_historical_" + (isPrimaryKeyTable ? "primary_key" : "log")); + TableInfo tableInfo = createHistoricalTable(tablePath, isPrimaryKeyTable); + long timestamp = 1_000L; + List records = + Arrays.asList( + historicalRecord( + 0L, timestamp, 1, "partition-1", "20240101", isPrimaryKeyTable), + historicalRecord( + 1L, timestamp, 1, "partition-2", "20240102", isPrimaryKeyTable)); + + PaimonWriteResult writeResult; + try (LakeWriter lakeWriter = + createLakeWriter(tablePath, 0, HISTORICAL_PARTITION_VALUE, 1L, tableInfo)) { + for (LogRecord record : records) { + lakeWriter.write(record); + } + writeResult = lakeWriter.complete(); + } + + assertThat(writeResult.commitMessages()).hasSize(2); + SimpleVersionedSerializer serializer = + paimonLakeTieringFactory.getWriteResultSerializer(); + assertThat(serializer.getVersion()).isEqualTo(1); + byte[] serialized = serializer.serialize(writeResult); + writeResult = serializer.deserialize(serializer.getVersion(), serialized); + assertThat(writeResult.commitMessages()).hasSize(2); + + commitWriteResults(tablePath, tableInfo, Collections.singletonList(writeResult)); + verifyHistoricalRecords(tablePath, isPrimaryKeyTable, records); + } + + @Test + void testHistoricalArrowBatchTiering() throws Exception { + TablePath tablePath = TablePath.of("paimon", "test_historical_arrow"); + TableInfo tableInfo = createHistoricalTable(tablePath, false); + long baseOffset = 10L; + long timestamp = 1_000L; + List records = + Arrays.asList( + historicalRecord( + baseOffset, timestamp, 1, "partition-1", "20240101", false), + historicalRecord( + baseOffset + 1, timestamp, 2, "partition-2", "20240102", false)); + + PaimonWriteResult writeResult; + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE); + LakeWriter lakeWriter = + createLakeWriter(tablePath, 0, HISTORICAL_PARTITION_VALUE, 1L, tableInfo)) { + VectorSchemaRoot root = + VectorSchemaRoot.create( + UnshadedArrowReadUtils.toArrowSchema(tableInfo.getRowType()), + allocator); + try (ArrowRecordBatch arrowRecordBatch = + new ArrowRecordBatch(new ArrowBatchData(root, baseOffset, timestamp, 1))) { + writeArrowRows(root, records); + ((SupportsRecordBatchWrite) lakeWriter).write(arrowRecordBatch); + } + writeResult = lakeWriter.complete(); + } + + assertThat(writeResult.commitMessages()).hasSize(2); + commitWriteResults(tablePath, tableInfo, Collections.singletonList(writeResult)); + verifyHistoricalRecords(tablePath, false, records); + } + @Test void testEmptyCommitCreatesSnapshot() throws Exception { TablePath tablePath = TablePath.of("paimon", "test_empty_commit"); @@ -593,6 +675,23 @@ private void verifyLogTableRecordsThreePartition( actualRecords.close(); } + private void verifyHistoricalRecords( + TablePath tablePath, boolean isPrimaryKeyTable, List records) + throws Exception { + List partitions = Arrays.asList("20240101", "20240102"); + assertThat(paimonCatalog.listPartitions(toPaimon(tablePath))) + .extracting(partition -> partition.spec().get("c3")) + .containsExactlyInAnyOrderElementsOf(partitions); + for (int i = 0; i < partitions.size(); i++) { + String partition = partitions.get(i); + verifyTableRecords( + getPaimonRows(tablePath, partition, isPrimaryKeyTable, 0), + Collections.singletonList(records.get(i)), + 0, + partition); + } + } + private void verifyTableRecords( CloseableIterator actualRecords, List expectRecords, @@ -737,6 +836,36 @@ private GenericRecord toRecord(long offset, GenericRow row, ChangeType changeTyp return new GenericRecord(offset, System.currentTimeMillis(), changeType, row); } + private LogRecord historicalRecord( + long offset, + long timestamp, + int key, + String value, + String partition, + boolean isPrimaryKeyTable) { + GenericRow row = new GenericRow(3); + row.setField(0, key); + row.setField(1, BinaryString.fromString(value)); + row.setField(2, BinaryString.fromString(partition)); + return new GenericRecord( + offset, timestamp, isPrimaryKeyTable ? INSERT : ChangeType.APPEND_ONLY, row); + } + + private void writeArrowRows(VectorSchemaRoot root, List records) { + root.allocateNew(); + IntVector keyVector = (IntVector) root.getVector("c1"); + VarCharVector valueVector = (VarCharVector) root.getVector("c2"); + VarCharVector partitionVector = (VarCharVector) root.getVector("c3"); + for (int i = 0; i < records.size(); i++) { + org.apache.fluss.row.InternalRow row = records.get(i).getRow(); + keyVector.setSafe(i, row.getInt(0)); + valueVector.setSafe(i, row.getString(1).toString().getBytes(StandardCharsets.UTF_8)); + partitionVector.setSafe( + i, row.getString(2).toString().getBytes(StandardCharsets.UTF_8)); + } + root.setRowCount(records.size()); + } + private CloseableIterator getPaimonRows( TablePath tablePath, @Nullable String partition, boolean isPrimaryKeyTable, int bucket) throws Exception { @@ -911,6 +1040,52 @@ private void createTable( doCreatePaimonTable(tablePath, builder); } + private TableInfo createHistoricalTable(TablePath tablePath, boolean isPrimaryKeyTable) + throws Exception { + createTable( + tablePath, + isPrimaryKeyTable, + true, + isPrimaryKeyTable ? 1 : null, + Collections.emptyMap()); + + org.apache.fluss.metadata.Schema.Builder schemaBuilder = + org.apache.fluss.metadata.Schema.newBuilder() + .column("c1", org.apache.fluss.types.DataTypes.INT()) + .column("c2", org.apache.fluss.types.DataTypes.STRING()) + .column("c3", org.apache.fluss.types.DataTypes.STRING()); + if (isPrimaryKeyTable) { + schemaBuilder.primaryKey("c1", "c3"); + } + TableDescriptor descriptor = + TableDescriptor.builder() + .schema(schemaBuilder.build()) + .partitionedBy("c3") + .distributedBy(1) + .property(ConfigOptions.TABLE_DATALAKE_ENABLED, true) + .property(ConfigOptions.TABLE_AUTO_PARTITION_ENABLED, true) + .property(ConfigOptions.TABLE_AUTO_PARTITION_KEY, "c3") + .property( + ConfigOptions.TABLE_AUTO_PARTITION_TIME_UNIT, + AutoPartitionTimeUnit.DAY) + .build(); + return TableInfo.of(tablePath, 0, 1, descriptor, DEFAULT_REMOTE_DATA_DIR, 1L, 1L); + } + + private void commitWriteResults( + TablePath tablePath, TableInfo tableInfo, List writeResults) + throws Exception { + try (LakeCommitter committer = + createLakeCommitter(tablePath, tableInfo, new Configuration())) { + PaimonCommittable committable = committer.toCommittable(writeResults); + assertThat( + committer + .commit(committable, Collections.emptyMap()) + .getCommittedSnapshotId()) + .isOne(); + } + } + private void createMultiPartitionTable(TablePath tablePath) throws Exception { Schema.Builder builder = Schema.newBuilder() diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/ProduceLogResultForBucket.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/ProduceLogResultForBucket.java index 37efad6c646..98812e97447 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/ProduceLogResultForBucket.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/ProduceLogResultForBucket.java @@ -23,35 +23,68 @@ import org.apache.fluss.rpc.protocol.ApiError; import org.apache.fluss.rpc.protocol.Errors; +import javax.annotation.Nullable; + +import java.util.Objects; + /** Result of {@link ProduceLogRequest} for each table bucket. */ @Internal public class ProduceLogResultForBucket extends WriteResultForBucket { private final long baseOffset; + private final @Nullable String originalPartitionName; public ProduceLogResultForBucket(TableBucket tableBucket, long baseOffset, long endOffset) { - this(tableBucket, baseOffset, endOffset, ApiError.NONE); + this(tableBucket, baseOffset, endOffset, ApiError.NONE, null); } public ProduceLogResultForBucket(TableBucket tableBucket, ApiError error) { - this(tableBucket, -1L, -1L, error); + this(tableBucket, -1L, -1L, error, null); + } + + public static ProduceLogResultForBucket historicalSuccess( + TableBucket tableBucket, + long baseOffset, + long endOffset, + String originalPartitionName) { + return new ProduceLogResultForBucket( + tableBucket, baseOffset, endOffset, ApiError.NONE, originalPartitionName); + } + + public static ProduceLogResultForBucket historicalFailure( + TableBucket tableBucket, ApiError error, String originalPartitionName) { + return new ProduceLogResultForBucket(tableBucket, -1L, -1L, error, originalPartitionName); } private ProduceLogResultForBucket( - TableBucket tableBucket, long baseOffset, long endOffset, ApiError error) { + TableBucket tableBucket, + long baseOffset, + long endOffset, + ApiError error, + @Nullable String originalPartitionName) { super(tableBucket, endOffset, error); this.baseOffset = baseOffset; + this.originalPartitionName = originalPartitionName; } public long getBaseOffset() { return baseOffset; } + /** Returns the original partition name for a historical write, or null for a normal write. */ + public @Nullable String getOriginalPartitionName() { + return originalPartitionName; + } + @Override public T copy(Errors newError) { //noinspection unchecked return (T) new ProduceLogResultForBucket( - tableBucket, baseOffset, getWriteLogEndOffset(), newError.toApiError()); + tableBucket, + baseOffset, + getWriteLogEndOffset(), + newError.toApiError(), + originalPartitionName); } @Override @@ -66,6 +99,12 @@ public boolean equals(Object o) { return false; } ProduceLogResultForBucket that = (ProduceLogResultForBucket) o; - return baseOffset == that.baseOffset; + return baseOffset == that.baseOffset + && Objects.equals(originalPartitionName, that.originalPartitionName); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), baseOffset, originalPartitionName); } } diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/client/ServerConnection.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/client/ServerConnection.java index 46ec0806110..f28cf1e9b76 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/client/ServerConnection.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/client/ServerConnection.java @@ -25,11 +25,14 @@ import org.apache.fluss.exception.InvalidServerTypeException; import org.apache.fluss.exception.NetworkException; import org.apache.fluss.exception.RetriableAuthenticationException; +import org.apache.fluss.exception.UnsupportedVersionException; import org.apache.fluss.rpc.messages.ApiMessage; import org.apache.fluss.rpc.messages.ApiVersionsRequest; import org.apache.fluss.rpc.messages.ApiVersionsResponse; import org.apache.fluss.rpc.messages.AuthenticateRequest; import org.apache.fluss.rpc.messages.AuthenticateResponse; +import org.apache.fluss.rpc.messages.ProduceLogRequest; +import org.apache.fluss.rpc.messages.PutKvRequest; import org.apache.fluss.rpc.metrics.ClientMetricGroup; import org.apache.fluss.rpc.metrics.ConnectionMetrics; import org.apache.fluss.rpc.protocol.ApiKeys; @@ -62,12 +65,16 @@ import java.util.concurrent.TimeUnit; import java.util.function.BiConsumer; +import static org.apache.fluss.rpc.util.CommonRpcMessageUtils.hasHistoricalProduce; +import static org.apache.fluss.rpc.util.CommonRpcMessageUtils.hasHistoricalPut; import static org.apache.fluss.utils.IOUtils.closeQuietly; /** Connection to a Netty server used by the {@link NettyClient}. */ @ThreadSafe final class ServerConnection { private static final Logger LOG = LoggerFactory.getLogger(ServerConnection.class); + private static final short HISTORICAL_PRODUCE_LOG_MIN_VERSION = 1; + private static final short HISTORICAL_PUT_KV_MIN_VERSION = 3; private final ServerNode node; @@ -309,6 +316,7 @@ private CompletableFuture doSend( if (serverApiVersions != null) { try { version = serverApiVersions.highestAvailableVersion(apiKey); + validateVersionCompatibility(apiKey, version, rawRequest); } catch (Exception e) { responseFuture.completeExceptionally(e); return responseFuture; @@ -361,6 +369,39 @@ private CompletableFuture doSend( } } + private void validateVersionCompatibility( + ApiKeys apiKey, short version, ApiMessage rawRequest) { + if (apiKey == ApiKeys.PRODUCE_LOG && version < HISTORICAL_PRODUCE_LOG_MIN_VERSION) { + ProduceLogRequest produceLogRequest = (ProduceLogRequest) rawRequest; + if (hasHistoricalProduce(produceLogRequest)) { + throw new UnsupportedVersionException( + "Historical partition writes require PRODUCE_LOG version " + + HISTORICAL_PRODUCE_LOG_MIN_VERSION + + " or newer, but server " + + node + + " negotiated version " + + version + + '.'); + } + } + + if (apiKey != ApiKeys.PUT_KV || version >= HISTORICAL_PUT_KV_MIN_VERSION) { + return; + } + + PutKvRequest putKvRequest = (PutKvRequest) rawRequest; + if (hasHistoricalPut(putKvRequest)) { + throw new UnsupportedVersionException( + "Historical partition writes require PUT_KV version " + + HISTORICAL_PUT_KV_MIN_VERSION + + " or newer, but server " + + node + + " negotiated version " + + version + + '.'); + } + } + private void handleApiVersionsResponse(ApiMessage response, Throwable cause) { if (cause != null) { close(cause); diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/ApiKeys.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/ApiKeys.java index baf4256650e..ffdd1978de9 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/ApiKeys.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/ApiKeys.java @@ -42,7 +42,8 @@ public enum ApiKeys { GET_TABLE_SCHEMA(1011, 0, 0, PUBLIC), GET_METADATA(1012, 0, 0, PUBLIC), UPDATE_METADATA(1013, 0, 0, PRIVATE), - PRODUCE_LOG(1014, 0, 0, PUBLIC), + // Version 1: Supports original_partition_name in requests and responses for historical writes. + PRODUCE_LOG(1014, 0, 1, PUBLIC), FETCH_LOG(1015, 0, 0, PUBLIC), // Version 0: Uses lake's encoder for primary key encoding (legacy behavior). diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/util/CommonRpcMessageUtils.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/util/CommonRpcMessageUtils.java index 219b51087a4..befd3fc7f3e 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/util/CommonRpcMessageUtils.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/util/CommonRpcMessageUtils.java @@ -34,6 +34,7 @@ import org.apache.fluss.rpc.messages.PbPartitionSpec; import org.apache.fluss.rpc.messages.PbRemoteLogFetchInfo; import org.apache.fluss.rpc.messages.PbRemoteLogSegment; +import org.apache.fluss.rpc.messages.ProduceLogRequest; import org.apache.fluss.rpc.messages.PutKvRequest; import org.apache.fluss.rpc.protocol.ApiError; import org.apache.fluss.security.acl.AccessControlEntry; @@ -83,6 +84,17 @@ public static boolean hasHistoricalPut(PutKvRequest putKvRequest) { && putKvRequest.getBucketsReqAt(0).hasOriginalPartitionName(); } + /** + * Returns whether the produce-log request is for historical partition writes. + * + *

Normal and historical write buckets cannot be mixed in the same request, so the first + * bucket determines the request type. + */ + public static boolean hasHistoricalProduce(ProduceLogRequest produceLogRequest) { + return produceLogRequest.getBucketsReqsCount() > 0 + && produceLogRequest.getBucketsReqAt(0).hasOriginalPartitionName(); + } + public static List toPbAclInfos(Collection aclBindings) { return aclBindings.stream() .map(CommonRpcMessageUtils::toPbAclInfo) diff --git a/fluss-rpc/src/main/proto/FlussApi.proto b/fluss-rpc/src/main/proto/FlussApi.proto index 9d5e05fbdae..1ae6b1f126b 100644 --- a/fluss-rpc/src/main/proto/FlussApi.proto +++ b/fluss-rpc/src/main/proto/FlussApi.proto @@ -886,6 +886,8 @@ message PbProduceLogReqForBucket { optional int64 partition_id = 1; required int32 bucket_id = 2; required bytes records = 3; + // The original partition name for a historical write; unset for a normal write. + optional string original_partition_name = 4; } message PbProduceLogRespForBucket { @@ -894,6 +896,8 @@ message PbProduceLogRespForBucket { optional int32 error_code = 3; optional string error_message = 4; optional int64 base_offset = 5; + // The original partition name echoed from a historical write request; unset for a normal write. + optional string original_partition_name = 6; } message PbFetchLogReqForTable { diff --git a/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/client/ServerConnectionTest.java b/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/client/ServerConnectionTest.java index 4368872ec73..541c6f50ee2 100644 --- a/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/client/ServerConnectionTest.java +++ b/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/client/ServerConnectionTest.java @@ -23,6 +23,7 @@ import org.apache.fluss.config.Configuration; import org.apache.fluss.exception.DisconnectException; import org.apache.fluss.exception.InvalidServerTypeException; +import org.apache.fluss.exception.UnsupportedVersionException; import org.apache.fluss.metrics.Gauge; import org.apache.fluss.metrics.Metric; import org.apache.fluss.metrics.MetricType; @@ -33,11 +34,18 @@ import org.apache.fluss.rpc.TestingGatewayService; import org.apache.fluss.rpc.TestingTabletGatewayService; import org.apache.fluss.rpc.messages.ApiMessage; +import org.apache.fluss.rpc.messages.ApiVersionsRequest; +import org.apache.fluss.rpc.messages.ApiVersionsResponse; import org.apache.fluss.rpc.messages.GetTableSchemaRequest; import org.apache.fluss.rpc.messages.ListDatabasesRequest; import org.apache.fluss.rpc.messages.LookupRequest; +import org.apache.fluss.rpc.messages.PbApiVersion; import org.apache.fluss.rpc.messages.PbLookupReqForBucket; import org.apache.fluss.rpc.messages.PbTablePath; +import org.apache.fluss.rpc.messages.ProduceLogRequest; +import org.apache.fluss.rpc.messages.ProduceLogResponse; +import org.apache.fluss.rpc.messages.PutKvRequest; +import org.apache.fluss.rpc.messages.PutKvResponse; import org.apache.fluss.rpc.metrics.ClientMetricGroup; import org.apache.fluss.rpc.metrics.TestingClientMetricGroup; import org.apache.fluss.rpc.netty.client.ServerConnection.ConnectionState; @@ -62,6 +70,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import static org.apache.fluss.metrics.MetricNames.CLIENT_BYTES_IN_RATE_AVG; import static org.apache.fluss.metrics.MetricNames.CLIENT_BYTES_IN_RATE_TOTAL; @@ -239,7 +248,80 @@ public ChannelFuture connect(String host, int port) { .isInstanceOf(DisconnectException.class); } + @Test + void testRejectHistoricalWritesForOldServer() throws Exception { + nettyServer.close(); + OldWriteGatewayService oldGatewayService = new OldWriteGatewayService(); + buildNettyServer(oldGatewayService); + + ServerConnection connection = + new ServerConnection( + bootstrap, + serverNode, + TestingClientMetricGroup.newInstance(), + clientAuthenticator, + (con, ignore) -> {}); + try { + assertThat(connection.send(ApiKeys.PUT_KV, putKvRequest(null)).get()) + .isInstanceOf(PutKvResponse.class); + assertThat(oldGatewayService.putKvRequests).hasValue(1); + + assertThatThrownBy( + () -> + connection + .send(ApiKeys.PUT_KV, putKvRequest("dt=20260823")) + .get()) + .rootCause() + .isInstanceOf(UnsupportedVersionException.class) + .hasMessageContaining("require PUT_KV version 3 or newer") + .hasMessageContaining("negotiated version 2"); + assertThat(oldGatewayService.putKvRequests).hasValue(1); + + assertThat(connection.send(ApiKeys.PRODUCE_LOG, produceLogRequest(null)).get()) + .isInstanceOf(ProduceLogResponse.class); + assertThat(oldGatewayService.produceLogRequests).hasValue(1); + + assertThatThrownBy( + () -> + connection + .send( + ApiKeys.PRODUCE_LOG, + produceLogRequest("dt=20260823")) + .get()) + .rootCause() + .isInstanceOf(UnsupportedVersionException.class) + .hasMessageContaining("require PRODUCE_LOG version 1 or newer") + .hasMessageContaining("negotiated version 0"); + assertThat(oldGatewayService.produceLogRequests).hasValue(1); + } finally { + connection.close().get(); + } + } + + private static PutKvRequest putKvRequest(String originalPartitionName) { + PutKvRequest request = new PutKvRequest().setTableId(1L).setAcks(1).setTimeoutMs(10_000); + request.addBucketsReq().setBucketId(0).setRecords(new byte[0]); + if (originalPartitionName != null) { + request.getBucketsReqAt(0).setOriginalPartitionName(originalPartitionName); + } + return request; + } + + private static ProduceLogRequest produceLogRequest(String originalPartitionName) { + ProduceLogRequest request = + new ProduceLogRequest().setTableId(1L).setAcks(1).setTimeoutMs(10_000); + request.addBucketsReq().setBucketId(0).setRecords(new byte[0]); + if (originalPartitionName != null) { + request.getBucketsReqAt(0).setOriginalPartitionName(originalPartitionName); + } + return request; + } + private void buildNettyServer() throws Exception { + buildNettyServer(new TestingTabletGatewayService()); + } + + private void buildNettyServer(TestingGatewayService gatewayService) throws Exception { try (NetUtils.Port availablePort = getAvailablePort(); NetUtils.Port availablePort2 = getAvailablePort()) { serverNode = @@ -248,7 +330,7 @@ private void buildNettyServer() throws Exception { serverNode2 = new ServerNode( 2, "localhost", availablePort2.getPort(), ServerType.TABLET_SERVER); - service = new TestingTabletGatewayService(); + service = gatewayService; MetricGroup metricGroup = NOPMetricsGroup.newInstance(); nettyServer = new NettyServer( @@ -263,6 +345,40 @@ private void buildNettyServer() throws Exception { } } + private static class OldWriteGatewayService extends TestingTabletGatewayService { + + private final AtomicInteger putKvRequests = new AtomicInteger(); + private final AtomicInteger produceLogRequests = new AtomicInteger(); + + @Override + public CompletableFuture apiVersions(ApiVersionsRequest request) { + return super.apiVersions(request) + .thenApply( + response -> { + for (PbApiVersion apiVersion : response.getApiVersionsList()) { + if (apiVersion.getApiKey() == ApiKeys.PUT_KV.id) { + apiVersion.setMaxVersion(2); + } else if (apiVersion.getApiKey() == ApiKeys.PRODUCE_LOG.id) { + apiVersion.setMaxVersion(0); + } + } + return response; + }); + } + + @Override + public CompletableFuture putKv(PutKvRequest request) { + putKvRequests.incrementAndGet(); + return CompletableFuture.completedFuture(new PutKvResponse()); + } + + @Override + public CompletableFuture produceLog(ProduceLogRequest request) { + produceLogRequests.incrementAndGet(); + return CompletableFuture.completedFuture(new ProduceLogResponse()); + } + } + private static class MockMetricRegistry extends NOPMetricRegistry { Map registeredMetrics = new HashMap<>(); diff --git a/fluss-rust/crates/fluss/proto/FlussApi.proto b/fluss-rust/crates/fluss/proto/FlussApi.proto index 9d5e05fbdae..1ae6b1f126b 100644 --- a/fluss-rust/crates/fluss/proto/FlussApi.proto +++ b/fluss-rust/crates/fluss/proto/FlussApi.proto @@ -886,6 +886,8 @@ message PbProduceLogReqForBucket { optional int64 partition_id = 1; required int32 bucket_id = 2; required bytes records = 3; + // The original partition name for a historical write; unset for a normal write. + optional string original_partition_name = 4; } message PbProduceLogRespForBucket { @@ -894,6 +896,8 @@ message PbProduceLogRespForBucket { optional int32 error_code = 3; optional string error_message = 4; optional int64 base_offset = 5; + // The original partition name echoed from a historical write request; unset for a normal write. + optional string original_partition_name = 6; } message PbFetchLogReqForTable { diff --git a/fluss-rust/crates/fluss/src/proto/fluss.rs b/fluss-rust/crates/fluss/src/proto/fluss.rs index 95e7045c57e..e29dbaea874 100644 --- a/fluss-rust/crates/fluss/src/proto/fluss.rs +++ b/fluss-rust/crates/fluss/src/proto/fluss.rs @@ -1165,6 +1165,9 @@ pub struct PbProduceLogReqForBucket { pub bucket_id: i32, #[prost(bytes = "bytes", required, tag = "3")] pub records: ::prost::bytes::Bytes, + /// The original partition name for a historical write; unset for a normal write. + #[prost(string, optional, tag = "4")] + pub original_partition_name: ::core::option::Option<::prost::alloc::string::String>, } #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct PbProduceLogRespForBucket { @@ -1178,6 +1181,9 @@ pub struct PbProduceLogRespForBucket { pub error_message: ::core::option::Option<::prost::alloc::string::String>, #[prost(int64, optional, tag = "5")] pub base_offset: ::core::option::Option, + /// The original partition name echoed from a historical write request; unset for a normal write. + #[prost(string, optional, tag = "6")] + pub original_partition_name: ::core::option::Option<::prost::alloc::string::String>, } #[derive(Clone, PartialEq, ::prost::Message)] pub struct PbFetchLogReqForTable { diff --git a/fluss-rust/crates/fluss/src/rpc/api_key.rs b/fluss-rust/crates/fluss/src/rpc/api_key.rs index e9720a4e2b6..1341d1e6180 100644 --- a/fluss-rust/crates/fluss/src/rpc/api_key.rs +++ b/fluss-rust/crates/fluss/src/rpc/api_key.rs @@ -93,7 +93,6 @@ impl ApiKey { | ApiKey::TableExists | ApiKey::GetTableSchema | ApiKey::MetaData - | ApiKey::ProduceLog | ApiKey::FetchLog | ApiKey::ListOffsets | ApiKey::GetLatestKvSnapshots @@ -129,6 +128,8 @@ impl ApiKey { | ApiKey::GetClusterHealth | ApiKey::ListRemoteLogManifests | ApiKey::ListKvSnapshots => Some(ApiVersionRange::new(ApiVersion(0), ApiVersion(0))), + // ProduceLog v1 adds historical partition context to requests and responses. + ApiKey::ProduceLog => Some(ApiVersionRange::new(ApiVersion(0), ApiVersion(1))), // PutKv v2 adds the storage backpressure error code; v3 adds historical partition // context to requests and responses. ApiKey::PutKv => Some(ApiVersionRange::new(ApiVersion(0), ApiVersion(3))), diff --git a/fluss-rust/crates/fluss/src/rpc/message/produce_log.rs b/fluss-rust/crates/fluss/src/rpc/message/produce_log.rs index de9dce118d2..041e0adde01 100644 --- a/fluss-rust/crates/fluss/src/rpc/message/produce_log.rs +++ b/fluss-rust/crates/fluss/src/rpc/message/produce_log.rs @@ -49,6 +49,7 @@ impl ProduceLogRequest { partition_id: ready_batch.table_bucket.partition_id(), bucket_id: ready_batch.table_bucket.bucket_id(), records: ready_batch.write_batch.build()?, + original_partition_name: None, }) } diff --git a/fluss-rust/crates/fluss/src/rpc/server_connection.rs b/fluss-rust/crates/fluss/src/rpc/server_connection.rs index a8b36cbecc2..66825ab265a 100644 --- a/fluss-rust/crates/fluss/src/rpc/server_connection.rs +++ b/fluss-rust/crates/fluss/src/rpc/server_connection.rs @@ -1196,7 +1196,7 @@ mod tests { min_version: 0, max_version: 3, }, - // ProduceLog: server v0..v2, client v0 only → negotiated v0 + // ProduceLog: server v0..v2, client v0..v1 → negotiated v1 PbApiVersion { api_key: 1014, min_version: 0, @@ -1238,7 +1238,7 @@ mod tests { negotiated .highest_available_version(ApiKey::ProduceLog) .unwrap(), - ApiVersion(0) + ApiVersion(1) ); // Disjoint range → error diff --git a/fluss-server/src/main/java/org/apache/fluss/server/DynamicServerConfig.java b/fluss-server/src/main/java/org/apache/fluss/server/DynamicServerConfig.java index 98d721e152e..5f6e860dbe4 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/DynamicServerConfig.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/DynamicServerConfig.java @@ -56,6 +56,7 @@ import static org.apache.fluss.config.ConfigOptions.REMOTE_DATA_DIRS_WEIGHTS; import static org.apache.fluss.config.ConfigOptions.SERVER_DATA_DISK_WRITE_LIMIT_RATIO; import static org.apache.fluss.config.ConfigOptions.SERVER_DATA_DISK_WRITE_RECOVER_RATIO; +import static org.apache.fluss.config.ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME; import static org.apache.fluss.config.ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUPER_CACHE_EXPIRE_AFTER_ACCESS; import static org.apache.fluss.config.ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO; import static org.apache.fluss.config.ConfigOptions.SERVER_SASL_CREDENTIALS; @@ -84,6 +85,7 @@ class DynamicServerConfig { KV_SNAPSHOT_INTERVAL.key(), SERVER_DATA_DISK_WRITE_RECOVER_RATIO.key(), SERVER_DATA_DISK_WRITE_LIMIT_RATIO.key(), + SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME.key(), SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO.key(), SERVER_HISTORICAL_PARTITION_LOOKUPER_CACHE_EXPIRE_AFTER_ACCESS.key(), // Config options for remote.data.dirs diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/HistoricalLookupCacheConfigValidator.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/HistoricalLookupCacheConfigValidator.java index c9d763f410a..337b9542d45 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/HistoricalLookupCacheConfigValidator.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/HistoricalLookupCacheConfigValidator.java @@ -24,11 +24,20 @@ import java.time.Duration; -/** Validates dynamic historical lookup cache settings. */ +/** Validates dynamic historical partition settings used outside the coordinator. */ final class HistoricalLookupCacheConfigValidator implements ServerReconfigurable { @Override public void validate(Configuration newConfig) throws ConfigException { + Duration newCleanupIdleTime = + newConfig.get(ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME); + if (newCleanupIdleTime.isNegative()) { + throw new ConfigException( + String.format( + "Invalid configuration for %s, it must not be negative.", + ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME.key())); + } + double newMaxRatio = newConfig.get( ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/entity/ProduceLogDataForBucket.java b/fluss-server/src/main/java/org/apache/fluss/server/entity/ProduceLogDataForBucket.java new file mode 100644 index 00000000000..50897fea1dc --- /dev/null +++ b/fluss-server/src/main/java/org/apache/fluss/server/entity/ProduceLogDataForBucket.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.entity; + +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.record.MemoryLogRecords; + +import javax.annotation.Nullable; + +/** Log records for one normal or historical partition bucket. */ +public class ProduceLogDataForBucket { + private final TableBucket tableBucket; + private final MemoryLogRecords records; + private final @Nullable String originalPartitionName; + + public ProduceLogDataForBucket( + TableBucket tableBucket, + MemoryLogRecords records, + @Nullable String originalPartitionName) { + this.tableBucket = tableBucket; + this.records = records; + this.originalPartitionName = originalPartitionName; + } + + public TableBucket tableBucket() { + return tableBucket; + } + + public MemoryLogRecords records() { + return records; + } + + public @Nullable String originalPartitionName() { + return originalPartitionName; + } +} diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java index ee28567756f..0f578468a23 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java @@ -222,6 +222,8 @@ public final class Replica { private volatile @Nullable KvTablet kvTablet; private volatile @Nullable CloseableRegistry closeableRegistryForKv; private @Nullable PeriodicSnapshotManager kvSnapshotManager; + // The lake log end offset used as the durable base of the current historical KV overlay. + private volatile long historicalKvBaseOffset = -1L; /** * Server-wide {@link ScannerManager}. Active sessions for this bucket are closed in {@link @@ -380,6 +382,58 @@ public long getLakeLogEndOffset() { return logTablet.getLakeLogEndOffset(); } + /** Returns whether the lake and local log end offsets match for historical KV cleanup. */ + public boolean isHistoricalKvCleanupReady() { + return inReadLock( + leaderIsrUpdateLock, + () -> { + long localLogEndOffset = logTablet.localLogEndOffset(); + return isHistoricalKvCleanupReady(localLogEndOffset); + }); + } + + /** + * Drops and recreates a fully tiered historical KV overlay if leadership and offsets still + * match. + * + * @param expectedLeaderEpoch leader epoch captured when cleanup was scheduled + * @param logEndOffset matching lake and local log end offset that triggered cleanup + * @param prepareLakeLookup marks the covering lake snapshot before the overlay is dropped + * @return whether the overlay was cleaned + */ + public boolean cleanupHistoricalKv( + int expectedLeaderEpoch, long logEndOffset, Runnable prepareLakeLookup) { + checkNotNull(prepareLakeLookup, "prepareLakeLookup must not be null"); + return inWriteLock( + leaderIsrUpdateLock, + () -> { + long localLogEndOffset = logTablet.localLogEndOffset(); + if (leaderEpoch != expectedLeaderEpoch + || localLogEndOffset != logEndOffset + || !isHistoricalKvCleanupReady(localLogEndOffset)) { + return false; + } + + LOG.info( + "Cleaning historical KV overlay for {} at local log end offset {} " + + "covered by lake log end offset {}.", + tableBucket, + localLogEndOffset, + logTablet.getLakeLogEndOffset()); + try { + // A lookup started after the rebuilt empty overlay is published must open + // a lake view that covers the state removed by this cleanup. + prepareLakeLookup.run(); + dropKv(); + createHistoricalKvAfterCleanup(); + return true; + } catch (RuntimeException e) { + fatalErrorHandler.onFatalError(e); + throw e; + } + }); + } + public boolean isDataLakeEnabled() { return getTableConfig().isDataLakeEnabled(); } @@ -725,6 +779,16 @@ private void logTableConfigChanges(TableInfo oldTableInfo, TableInfo newTableInf } } + private boolean isHistoricalKvCleanupReady(long localLogEndOffset) { + return isLeader() + && isHistoricalPartition() + && isKvTable() + && kvTablet != null + && historicalKvBaseOffset >= 0L + && historicalKvBaseOffset < localLogEndOffset + && logTablet.getLakeLogEndOffset() == localLogEndOffset; + } + private void createKv() { try { // create a closeable registry for the closable related to kv @@ -754,10 +818,7 @@ private void createKv() { } // A historical KV tablet is a disposable overlay over the lake snapshot. It is recovered // by replaying WAL from the lake log end offset and does not create its own KV snapshots. - if (isHistoricalPartition()) { - // TODO: Clean up historical KV state after the corresponding WAL is fully tiered to - // lake storage. - } else { + if (!isHistoricalPartition()) { startPeriodicKvSnapshot(snapshotUsed.orElse(null)); } } @@ -777,6 +838,26 @@ private void dropKv() { kvManager.dropKv(tableBucket); kvTablet = null; } + historicalKvBaseOffset = -1L; + } + + private void createHistoricalKvAfterCleanup() { + checkState(isHistoricalPartition(), "Only a historical KV overlay can be cleaned."); + try { + closeableRegistryForKv = new CloseableRegistry(); + closeableRegistry.registerCloseable(closeableRegistryForKv); + initKvTablet(); + } catch (Exception e) { + try { + dropKv(); + } catch (Exception cleanupError) { + e.addSuppressed(cleanupError); + } + throw new KvStorageException( + String.format( + "Failed to recreate historical KV overlay for bucket %s.", tableBucket), + e); + } } private void mayFlushKv(long newHighWatermark) { @@ -898,6 +979,9 @@ private Optional initKvTablet() { logTablet.updateMinRetainOffset(restoreStartOffset); recoverKvTablet(restoreStartOffset, rowCount, autoIncIDRange); + if (isHistoricalPartition()) { + historicalKvBaseOffset = restoreStartOffset; + } } catch (Exception e) { throw new KvStorageException( String.format( @@ -1044,8 +1128,10 @@ private void recoverKvTablet( private long historicalRecoveryStartOffset() { long lakeLogEndOffset = logTablet.getLakeLogEndOffset(); + long localLogEndOffset = logTablet.localLogEndOffset(); long logStartOffset = logTablet.logStartOffset(); - long recoveryStartOffset = lakeLogEndOffset >= 0 ? lakeLogEndOffset : 0L; + long recoveryStartOffset = + lakeLogEndOffset >= 0 ? Math.min(lakeLogEndOffset, localLogEndOffset) : 0L; checkState( recoveryStartOffset >= logStartOffset, "Cannot recover historical KV state: recovery start offset %s is before the " @@ -1161,9 +1247,14 @@ public LogAppendInfo appendRecordsToLeader(MemoryLogRecords memoryLogRecords, in "Leader not local for bucket %s on tabletServer %d", tableBucket, localTabletServerId)); } - if (isHistoricalPartition()) { + // Historical primary-key writes must go through PUT_KV so the server can + // preserve the original partition namespace and consult the lake on a local + // miss. Append-only records already contain their partition columns, so a log + // table can append them directly to its historical system partition. + if (isHistoricalPartition() && isKvTable()) { throw new InvalidPartitionException( - "Normal write request must not target a historical partition."); + "Produce-log request must not target the historical partition of " + + "a primary-key table."); } validateInSyncReplicaSize(requiredAcks); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java index b3357019230..67fc81a3350 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java @@ -82,6 +82,7 @@ import org.apache.fluss.server.entity.NotifyLeaderAndIsrData; import org.apache.fluss.server.entity.NotifyLeaderAndIsrResultForBucket; import org.apache.fluss.server.entity.NotifyRemoteLogOffsetsData; +import org.apache.fluss.server.entity.ProduceLogDataForBucket; import org.apache.fluss.server.entity.PutKvDataForBucket; import org.apache.fluss.server.entity.StopReplicaData; import org.apache.fluss.server.entity.StopReplicaResultForBucket; @@ -373,7 +374,8 @@ public ReplicaManager( localDiskManager, dataDir, dataDirVolumeBytes, - scheduler); + scheduler, + clock); registerMetrics(); } @@ -417,6 +419,7 @@ public int getCoordinatorEpoch() { public void validate(Configuration newConfig) throws ConfigException { // Type validation is already handled by DynamicServerConfig. // Here we only do basic sanity checks. + historicalPartitionManager.validate(newConfig); int newMinInSyncReplicas = newConfig.get(ConfigOptions.LOG_REPLICA_MIN_IN_SYNC_REPLICAS_NUMBER); if (newMinInSyncReplicas <= 0) { @@ -688,6 +691,51 @@ public void appendRecordsToLog( timeoutMs, requiredAcks, entriesPerBucket.size(), appendResult, responseCallback); } + /** Appends historical log batches while preserving each original partition in the response. */ + public void appendHistoricalRecordsToLog( + int timeoutMs, + int requiredAcks, + Collection entriesPerBucket, + @Nullable UserContext userContext, + Consumer> responseCallback) { + if (entriesPerBucket.isEmpty()) { + responseCallback.accept(Collections.emptyList()); + return; + } + + List results = Collections.synchronizedList(new ArrayList<>()); + AtomicInteger remaining = new AtomicInteger(entriesPerBucket.size()); + for (ProduceLogDataForBucket bucketData : entriesPerBucket) { + String originalPartitionName = + checkNotNull( + bucketData.originalPartitionName(), + "originalPartitionName must not be null"); + appendRecordsToLog( + timeoutMs, + requiredAcks, + Collections.singletonMap(bucketData.tableBucket(), bucketData.records()), + userContext, + bucketResults -> { + ProduceLogResultForBucket result = bucketResults.get(0); + ProduceLogResultForBucket historicalResult = + result.failed() + ? ProduceLogResultForBucket.historicalFailure( + result.getTableBucket(), + result.getError(), + originalPartitionName) + : ProduceLogResultForBucket.historicalSuccess( + result.getTableBucket(), + result.getBaseOffset(), + result.getWriteLogEndOffset(), + originalPartitionName); + results.add(historicalResult); + if (remaining.decrementAndGet() == 0) { + responseCallback.accept(new ArrayList<>(results)); + } + }); + } + } + /** * Fetch records from a replica. Currently, we will return the fetched records immediately. * @@ -1350,7 +1398,8 @@ public void notifyLakeTableOffset( lakeBucketOffsets.entrySet()) { TableBucket tb = lakeBucketOffsetEntry.getKey(); LakeBucketOffset lakeBucketOffset = lakeBucketOffsetEntry.getValue(); - LogTablet logTablet = getReplicaOrException(tb).getLogTablet(); + Replica replica = getReplicaOrException(tb); + LogTablet logTablet = replica.getLogTablet(); logTablet.updateLakeTableSnapshotId(lakeBucketOffset.getSnapshotId()); lakeBucketOffset @@ -1359,7 +1408,19 @@ public void notifyLakeTableOffset( lakeBucketOffset .getLogEndOffset() - .ifPresent(logTablet::updateLakeLogEndOffset); + .ifPresent( + lakeLogEndOffset -> { + logTablet.updateLakeLogEndOffset(lakeLogEndOffset); + if (replica.isHistoricalPartition() + && replica.isKvTable()) { + // Only an explicit log-end-offset notification can + // make historical cleanup eligible. + historicalPartitionManager.onLakeProgress( + replica, + lakeBucketOffset.getSnapshotId(), + lakeLogEndOffset); + } + }); lakeBucketOffset .getMaxTimestamp() @@ -1402,7 +1463,13 @@ private void makeLeaders( if (replica.isDataLakeEnabled()) { updateWithLakeTableSnapshot(replica); } + int previousLeaderEpoch = replica.getLeaderEpoch(); replica.makeLeader(data); + if (replica.isHistoricalPartition() + && replica.isKvTable() + && previousLeaderEpoch != replica.getLeaderEpoch()) { + historicalPartitionManager.onLeaderActivated(replica); + } // start the remote log tiering tasks for leaders remoteLogManager.startLogTiering(replica); @@ -1425,6 +1492,9 @@ private void updateWithLakeTableSnapshot(Replica replica) throws Exception { LakeTableSnapshot lakeTableSnapshot = optLakeTableSnapshot.get(); long snapshotId = optLakeTableSnapshot.get().getSnapshotId(); replica.getLogTablet().updateLakeTableSnapshotId(snapshotId); + lakeTableSnapshot + .getLogEndOffset(tb) + .ifPresent(replica.getLogTablet()::updateLakeLogEndOffset); if (replica.isHistoricalPartition()) { // The historical overlay will be rebuilt from this snapshot's lake offset. // Refresh a cached lookuper before it becomes the fallback for data omitted @@ -1432,9 +1502,6 @@ private void updateWithLakeTableSnapshot(Replica replica) throws Exception { historicalPartitionManager.requireLakeSnapshot( replica.getTableBucket().getTableId(), snapshotId); } - lakeTableSnapshot - .getLogEndOffset(tb) - .ifPresent(replica.getLogTablet()::updateLakeLogEndOffset); } } catch (Exception e) { if (replica.isHistoricalPartition()) { @@ -1476,6 +1543,9 @@ private void makeFollowers( replicasBecomeFollower.add(replica); scannerManager.closeScannersForBucket(tb); } + if (replica.isHistoricalPartition()) { + historicalPartitionManager.onReplicaStopped(tb); + } // stop the remote log tiering tasks for followers remoteLogManager.stopLogTiering(replica); result.put(tb, new NotifyLeaderAndIsrResultForBucket(tb)); @@ -2278,6 +2348,9 @@ private StopReplicaResultForBucket stopReplica( HostedReplica replica = getReplica(tb); if (replica instanceof OnlineReplica) { Replica replicaToDelete = ((OnlineReplica) replica).getReplica(); + if (replicaToDelete.isHistoricalPartition()) { + historicalPartitionManager.onReplicaStopped(tb); + } if (deleteLocal) { if (allReplicas.remove(tb) != null) { serverMetricGroup.removeTableBucketMetricGroup( diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java index 13ec002e0df..62a66d61137 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java @@ -19,7 +19,9 @@ import org.apache.fluss.annotation.Internal; import org.apache.fluss.annotation.VisibleForTesting; +import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; +import org.apache.fluss.exception.ConfigException; import org.apache.fluss.exception.HistoricalPartitionThrottledException; import org.apache.fluss.exception.InvalidPartitionException; import org.apache.fluss.lake.lakestorage.LakeTableLookuper; @@ -43,11 +45,17 @@ import org.apache.fluss.server.storage.LocalDiskManager; import org.apache.fluss.utils.ByteArraySlice; import org.apache.fluss.utils.ByteArrayWrapper; +import org.apache.fluss.utils.clock.Clock; +import org.apache.fluss.utils.clock.SystemClock; import org.apache.fluss.utils.concurrent.Scheduler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import javax.annotation.Nullable; import java.io.File; +import java.time.Duration; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -55,15 +63,29 @@ import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicBoolean; +import static org.apache.fluss.utils.Preconditions.checkArgument; import static org.apache.fluss.utils.Preconditions.checkNotNull; /** Coordinates lookup, write, and lifecycle operations for historical partitions. */ @Internal public final class HistoricalPartitionManager implements AutoCloseable { + private static final Logger LOG = LoggerFactory.getLogger(HistoricalPartitionManager.class); + private static final long MAX_HISTORICAL_KV_SIZE_BYTES = 5L * 1024 * 1024 * 1024; + private final HistoricalPartitionTaskExecutor taskExecutor; private final HistoricalLakeLookupManager lakeLookupManager; + private final Clock clock; + private volatile long cleanupIdleTimeMs; + private final long maxHistoricalKvSizeBytes; + // Per-physical-bucket state for coordinating historical write admission and overlay cleanup. + // The state is replaced when a new leader epoch is activated and removed when the local + // replica stops. + private final ConcurrentMap historicalWriteStates; /** Creates a historical partition manager from the tablet server dependencies. */ public HistoricalPartitionManager( @@ -72,8 +94,10 @@ public HistoricalPartitionManager( LocalDiskManager localDiskManager, File dataDir, long dataDirVolumeBytes, - Scheduler scheduler) { + Scheduler scheduler, + Clock clock) { this( + conf, new HistoricalPartitionTaskExecutor(conf), new HistoricalLakeLookupManager( conf, @@ -81,16 +105,45 @@ public HistoricalPartitionManager( localDiskManager, dataDir, dataDirVolumeBytes, - scheduler)); + scheduler), + clock, + MAX_HISTORICAL_KV_SIZE_BYTES); } @VisibleForTesting HistoricalPartitionManager( HistoricalPartitionTaskExecutor taskExecutor, HistoricalLakeLookupManager lakeLookupManager) { + this( + new Configuration(), + taskExecutor, + lakeLookupManager, + SystemClock.getInstance(), + MAX_HISTORICAL_KV_SIZE_BYTES); + } + + @VisibleForTesting + HistoricalPartitionManager( + Configuration conf, + HistoricalPartitionTaskExecutor taskExecutor, + HistoricalLakeLookupManager lakeLookupManager, + Clock clock, + long maxHistoricalKvSizeBytes) { + Duration cleanupIdleTime = + conf.get(ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME); + checkArgument( + !cleanupIdleTime.isNegative(), + "%s must not be negative.", + ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME.key()); + checkArgument( + maxHistoricalKvSizeBytes > 0L, "maxHistoricalKvSizeBytes must be greater than 0."); this.taskExecutor = checkNotNull(taskExecutor, "taskExecutor must not be null"); this.lakeLookupManager = checkNotNull(lakeLookupManager, "lakeLookupManager must not be null"); + this.clock = checkNotNull(clock, "clock must not be null"); + this.cleanupIdleTimeMs = cleanupIdleTime.toMillis(); + this.maxHistoricalKvSizeBytes = maxHistoricalKvSizeBytes; + this.historicalWriteStates = new ConcurrentHashMap<>(); } /** Starts the resources used by historical partition operations. */ @@ -98,6 +151,41 @@ public void startup(Scheduler scheduler) { lakeLookupManager.startup(scheduler); } + /** Starts tracking cleanup activity for a newly activated historical KV leader. */ + public void onLeaderActivated(Replica replica) { + historicalWriteStates.put( + replica.getTableBucket(), new HistoricalWriteState(clock.milliseconds())); + } + + /** Stops tracking cleanup activity for a replica that is no longer a local leader. */ + public void onReplicaStopped(TableBucket tableBucket) { + historicalWriteStates.remove(tableBucket); + } + + /** Records new lake progress and schedules any cleanup that it makes eligible. */ + public void onLakeProgress(Replica replica, long lakeSnapshotId, long lakeLogEndOffset) { + if (!replica.isLeader() || !replica.isKvTable()) { + return; + } + int expectedLeaderEpoch = replica.getLeaderEpoch(); + long localLogEndOffset = replica.getLocalLogEndOffset(); + if (lakeLogEndOffset != localLogEndOffset) { + return; + } + HistoricalWriteState state = historicalWriteStateFor(replica); + boolean maxSizeReached = state.maxSizeReached.get(); + // Lake progress is the cleanup trigger. The ordered cleanup task rechecks the latest write + // time when it actually runs, so a write accepted after this notification cancels an idle + // cleanup without being overtaken by it. + scheduleCleanup( + replica, + state, + maxSizeReached, + lakeSnapshotId, + expectedLeaderEpoch, + lakeLogEndOffset); + } + /** Looks up historical keys from the local overlay and then lake storage. */ public CompletableFuture lookup( Replica replica, @@ -119,7 +207,8 @@ public CompletableFuture lookup( + tableBucket + " (original partition " + lookupData.originalPartitionName() - + ").")))); + + ") because the historical request " + + "queue is full.")))); } catch (RuntimeException e) { return CompletableFuture.completedFuture( new LookupResultForBucket( @@ -141,6 +230,20 @@ public CompletableFuture putKv( checkNotNull( putData.originalPartitionName(), "originalPartitionName must not be null"); + HistoricalWriteState state = historicalWriteStateFor(replica); + if (state.maxSizeReached.get()) { + return CompletableFuture.completedFuture( + maxSizeThrottledResult( + putData, originalPartitionName, maxHistoricalKvSizeBytes)); + } + long liveSstSize = replica.logicalStorageKvSize(); + if (liveSstSize >= maxHistoricalKvSizeBytes) { + markMaxSizeReached(replica, state); + return CompletableFuture.completedFuture( + maxSizeThrottledResult( + putData, originalPartitionName, maxHistoricalKvSizeBytes)); + } + state.lastHistoricalWriteMs = clock.milliseconds(); return taskExecutor.submitOrdered( putData.tableBucket(), () -> { @@ -152,6 +255,7 @@ public CompletableFuture putKv( targetColumns, mergeMode, requiredAcks); + state.lastHistoricalWriteMs = clock.milliseconds(); return PutKvResultForBucket.historicalSuccess( putData.tableBucket(), appendInfo.lastOffset() + 1, @@ -163,17 +267,7 @@ public CompletableFuture putKv( originalPartitionName); } }, - () -> - PutKvResultForBucket.historicalFailure( - putData.tableBucket(), - ApiError.fromThrowable( - new HistoricalPartitionThrottledException( - "Historical write is throttled for " - + putData.tableBucket() - + " (original partition " - + originalPartitionName - + ").")), - originalPartitionName)); + () -> requestLimitThrottledResult(putData, originalPartitionName)); } catch (RuntimeException e) { return CompletableFuture.completedFuture( PutKvResultForBucket.historicalFailure( @@ -183,9 +277,33 @@ public CompletableFuture putKv( } } - /** Applies dynamic historical lookup configuration changes. */ + /** Validates dynamic historical partition configuration changes. */ + public void validate(Configuration newConf) throws ConfigException { + Duration newCleanupIdleTime = + newConf.get(ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME); + if (newCleanupIdleTime.isNegative()) { + throw new ConfigException( + String.format( + "Invalid configuration for %s, it must not be negative.", + ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME.key())); + } + } + + /** Applies dynamic historical partition configuration changes. */ public void reconfigure(Configuration newConf) { lakeLookupManager.reconfigure(newConf); + long newCleanupIdleTimeMs = + newConf.get(ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME) + .toMillis(); + if (newCleanupIdleTimeMs == cleanupIdleTimeMs) { + return; + } + long oldCleanupIdleTimeMs = cleanupIdleTimeMs; + cleanupIdleTimeMs = newCleanupIdleTimeMs; + LOG.info( + "Historical KV cleanup idle time reconfigured: {} ms -> {} ms.", + oldCleanupIdleTimeMs, + newCleanupIdleTimeMs); } /** Invalidates the cached lake lookuper for the given table. */ @@ -286,8 +404,117 @@ LogAppendInfo processPut( requiredAcks); } + private void markMaxSizeReached(Replica replica, HistoricalWriteState state) { + if (state.maxSizeReached.compareAndSet(false, true)) { + LOG.warn( + "Pausing historical writes for {} because its live SST size reached the " + + "maximum size {} bytes.", + replica.getTableBucket(), + maxHistoricalKvSizeBytes); + } + } + + private HistoricalWriteState historicalWriteStateFor(Replica replica) { + return historicalWriteStates.computeIfAbsent( + replica.getTableBucket(), + ignored -> new HistoricalWriteState(clock.milliseconds())); + } + + private void scheduleCleanup( + Replica replica, + HistoricalWriteState state, + boolean maxSizeReached, + long lakeSnapshotId, + int expectedLeaderEpoch, + long logEndOffset) { + CompletableFuture cleanupFuture; + cleanupFuture = + taskExecutor.submitOrderedMaintenance( + replica.getTableBucket(), + () -> + runCleanup( + replica, + state, + maxSizeReached, + lakeSnapshotId, + expectedLeaderEpoch, + logEndOffset)); + cleanupFuture.whenComplete( + (ignored, error) -> { + if (error != null) { + LOG.error( + "Historical KV cleanup failed for {}.", + replica.getTableBucket(), + error); + } + }); + } + + private void runCleanup( + Replica replica, + HistoricalWriteState state, + boolean maxSizeReached, + long lakeSnapshotId, + int expectedLeaderEpoch, + long logEndOffset) { + long now = clock.milliseconds(); + if (historicalWriteStates.get(replica.getTableBucket()) != state + || expectedLeaderEpoch != replica.getLeaderEpoch() + || (!maxSizeReached + && (cleanupIdleTimeMs <= 0L + || now < state.lastHistoricalWriteMs + || now - state.lastHistoricalWriteMs < cleanupIdleTimeMs))) { + return; + } + + if (replica.cleanupHistoricalKv( + expectedLeaderEpoch, + logEndOffset, + () -> requireLakeSnapshot(replica.getTableBucket().getTableId(), lakeSnapshotId))) { + state.maxSizeReached.set(false); + LOG.info( + "Cleaned {} historical KV overlay for {}.", + maxSizeReached ? "max-size-triggered" : "idle-triggered", + replica.getTableBucket()); + } + } + + private static PutKvResultForBucket requestLimitThrottledResult( + PutKvDataForBucket putData, String originalPartitionName) { + return PutKvResultForBucket.historicalFailure( + putData.tableBucket(), + ApiError.fromThrowable( + new HistoricalPartitionThrottledException( + "Historical write is throttled for " + + putData.tableBucket() + + " (original partition " + + originalPartitionName + + ") because the historical request queue is full.")), + originalPartitionName); + } + + private static PutKvResultForBucket maxSizeThrottledResult( + PutKvDataForBucket putData, String originalPartitionName, long maxHistoricalKvSize) { + return PutKvResultForBucket.historicalFailure( + putData.tableBucket(), + ApiError.fromThrowable( + new HistoricalPartitionThrottledException( + "Historical write is throttled for " + + putData.tableBucket() + + " (original partition " + + originalPartitionName + + ") because its historical KV overlay reached the live " + + "SST maximum size of " + + maxHistoricalKvSize + + " bytes. New writes are paused until lake tiering " + + "covers all previously accepted writes and the local " + + "overlay cleanup completes.")), + originalPartitionName); + } + @Override public void close() { + historicalWriteStates.clear(); taskExecutor.close(); lakeLookupManager.close(); } @@ -352,4 +579,19 @@ private LookupResultForBucket lookupInternal( tableBucket, originalPartitionName, ApiError.fromThrowable(e)); } } + + /** Per-bucket historical write activity and maximum-size state. */ + private static final class HistoricalWriteState { + // Latched when the live SST size reaches the maximum. It is cleared only after a cleanup + // covered by lake progress succeeds, so transient RocksDB size changes cannot resume + // writes prematurely. + private final AtomicBoolean maxSizeReached = new AtomicBoolean(); + + // Updated when a write is admitted and again when it completes successfully. + private volatile long lastHistoricalWriteMs; + + private HistoricalWriteState(long lastHistoricalWriteMs) { + this.lastHistoricalWriteMs = lastHistoricalWriteMs; + } + } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionTaskExecutor.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionTaskExecutor.java index 9bf66848fad..763f2cd1807 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionTaskExecutor.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionTaskExecutor.java @@ -141,24 +141,47 @@ public CompletableFuture submitOrdered( return CompletableFuture.completedFuture(throttledResult.get()); } - CompletableFuture future; - CompletableFuture tail; try { - synchronized (orderedTasksLock) { - CompletableFuture previousTail = orderedTaskTails.get(orderingKey); - if (previousTail == null) { - future = CompletableFuture.supplyAsync(task, executor); - } else { - future = previousTail.thenApplyAsync(ignored -> task.get(), executor); - } - // Convert success or failure into a normal completion used only for sequencing. - tail = future.handle((ignored, error) -> null); - orderedTaskTails.put(orderingKey, tail); - } + return enqueueOrdered(orderingKey, task, true); } catch (RuntimeException e) { requestPermits.release(); throw e; } + } + + /** + * Submits an internal maintenance task after all accepted tasks with the same ordering key. + * + *

Maintenance work is not a client request and therefore does not consume a request permit. + */ + public CompletableFuture submitOrderedMaintenance( + Object orderingKey, Runnable maintenanceTask) { + checkNotNull(orderingKey, "orderingKey must not be null."); + checkNotNull(maintenanceTask, "maintenanceTask must not be null."); + return enqueueOrdered( + orderingKey, + () -> { + maintenanceTask.run(); + return null; + }, + false); + } + + private CompletableFuture enqueueOrdered( + Object orderingKey, Supplier task, boolean releaseRequestPermit) { + CompletableFuture future; + CompletableFuture tail; + synchronized (orderedTasksLock) { + CompletableFuture previousTail = orderedTaskTails.get(orderingKey); + if (previousTail == null) { + future = CompletableFuture.supplyAsync(task, executor); + } else { + future = previousTail.thenApplyAsync(ignored -> task.get(), executor); + } + // Convert success or failure into a normal completion used only for sequencing. + tail = future.handle((ignored, error) -> null); + orderedTaskTails.put(orderingKey, tail); + } CompletableFuture currentTail = tail; tail.whenComplete( @@ -167,17 +190,24 @@ public CompletableFuture submitOrdered( orderedTaskTails.remove(orderingKey, currentTail); } }); - return trackAcceptedRequest(future); + return trackAcceptedRequest(future, releaseRequestPermit); } private CompletableFuture trackAcceptedRequest(CompletableFuture future) { + return trackAcceptedRequest(future, true); + } + + private CompletableFuture trackAcceptedRequest( + CompletableFuture future, boolean releaseRequestPermit) { pendingRequests.add(future); future.whenComplete( (ignored, error) -> { - // Release the permit exactly once when the accepted task reaches a terminal - // state, including exceptional completion and cancellation. pendingRequests.remove(future); - requestPermits.release(); + if (releaseRequestPermit) { + // Release the permit exactly once when the accepted request reaches a + // terminal state, including exceptional completion and cancellation. + requestPermits.release(); + } }); return future; } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java b/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java index bd3ef49b35a..c3468b7a941 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java @@ -36,6 +36,7 @@ import org.apache.fluss.rpc.entity.FetchLogResultForBucket; import org.apache.fluss.rpc.entity.LookupResultForBucket; import org.apache.fluss.rpc.entity.PrefixLookupResultForBucket; +import org.apache.fluss.rpc.entity.ProduceLogResultForBucket; import org.apache.fluss.rpc.entity.ResultForBucket; import org.apache.fluss.rpc.gateway.CoordinatorGateway; import org.apache.fluss.rpc.gateway.TabletServerGateway; @@ -92,6 +93,7 @@ import org.apache.fluss.server.entity.NotifyLakeTableOffsetData; import org.apache.fluss.server.entity.NotifyLeaderAndIsrData; import org.apache.fluss.server.entity.NotifyRemoteLogOffsetsData; +import org.apache.fluss.server.entity.ProduceLogDataForBucket; import org.apache.fluss.server.entity.PutKvDataForBucket; import org.apache.fluss.server.entity.StopReplicaData; import org.apache.fluss.server.entity.UserContext; @@ -125,6 +127,7 @@ import java.util.stream.Collectors; import static org.apache.fluss.rpc.util.CommonRpcMessageUtils.hasHistoricalLookup; +import static org.apache.fluss.rpc.util.CommonRpcMessageUtils.hasHistoricalProduce; import static org.apache.fluss.rpc.util.CommonRpcMessageUtils.hasHistoricalPut; import static org.apache.fluss.security.acl.OperationType.DESCRIBE; import static org.apache.fluss.security.acl.OperationType.READ; @@ -137,7 +140,6 @@ import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getNotifyLeaderAndIsrRequestData; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getNotifyRemoteLogOffsetsData; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getNotifySnapshotOffsetData; -import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getProduceLogData; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getStopReplicaData; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getTableFilterInfoMap; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getTableStatsRequestData; @@ -157,6 +159,7 @@ import static org.apache.fluss.server.utils.ServerRpcMessageUtils.toHistoricalLookupData; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.toLookupData; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.toPrefixLookupData; +import static org.apache.fluss.server.utils.ServerRpcMessageUtils.toProduceLogDataForBuckets; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.toPutKvDataForBuckets; /** An RPC Gateway service for tablet server. */ @@ -216,13 +219,29 @@ public void shutdown() {} public CompletableFuture produceLog(ProduceLogRequest request) { authorizeTable(WRITE, request.getTableId()); CompletableFuture response = new CompletableFuture<>(); - Map produceLogData = getProduceLogData(request); - replicaManager.appendRecordsToLog( - request.getTimeoutMs(), - request.getAcks(), - produceLogData, - new UserContext(currentSession().getPrincipal()), - bucketResponseMap -> response.complete(makeProduceLogResponse(bucketResponseMap))); + List produceLogData = toProduceLogDataForBuckets(request); + UserContext userContext = new UserContext(currentSession().getPrincipal()); + Consumer> responseCallback = + results -> response.complete(makeProduceLogResponse(results)); + if (hasHistoricalProduce(request)) { + replicaManager.appendHistoricalRecordsToLog( + request.getTimeoutMs(), + request.getAcks(), + produceLogData, + userContext, + responseCallback); + } else { + Map recordsByBucket = new HashMap<>(); + for (ProduceLogDataForBucket bucketData : produceLogData) { + recordsByBucket.put(bucketData.tableBucket(), bucketData.records()); + } + replicaManager.appendRecordsToLog( + request.getTimeoutMs(), + request.getAcks(), + recordsByBucket, + userContext, + responseCallback); + } return response; } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java index 02297ead2dd..58a35b392b0 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java @@ -187,6 +187,7 @@ import org.apache.fluss.server.entity.NotifyLeaderAndIsrData; import org.apache.fluss.server.entity.NotifyLeaderAndIsrResultForBucket; import org.apache.fluss.server.entity.NotifyRemoteLogOffsetsData; +import org.apache.fluss.server.entity.ProduceLogDataForBucket; import org.apache.fluss.server.entity.PutKvDataForBucket; import org.apache.fluss.server.entity.StopReplicaData; import org.apache.fluss.server.entity.StopReplicaResultForBucket; @@ -231,6 +232,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.apache.fluss.rpc.util.CommonRpcMessageUtils.hasHistoricalProduce; import static org.apache.fluss.rpc.util.CommonRpcMessageUtils.toByteBuffer; import static org.apache.fluss.rpc.util.CommonRpcMessageUtils.toPbAclInfo; import static org.apache.fluss.utils.Preconditions.checkNotNull; @@ -892,12 +894,20 @@ public static StopReplicaResponse makeStopReplicaResponse( return stopReplicaResponse; } - public static Map getProduceLogData( + /** Converts produce-log requests while preserving their historical partition context. */ + public static List toProduceLogDataForBuckets( ProduceLogRequest produceRequest) { long tableId = produceRequest.getTableId(); - Map produceEntryData = new HashMap<>(); + List produceLogData = + new ArrayList<>(produceRequest.getBucketsReqsCount()); + Map> originalPartitionsByBucket = new HashMap<>(); + boolean historicalWriteRequest = hasHistoricalProduce(produceRequest); for (PbProduceLogReqForBucket produceLogReqForBucket : produceRequest.getBucketsReqsList()) { + if (produceLogReqForBucket.hasOriginalPartitionName() != historicalWriteRequest) { + throw new IllegalArgumentException( + "Normal and historical writes cannot be mixed in the same request."); + } ByteBuffer recordBuffer = toByteBuffer(produceLogReqForBucket.getRecordsSlice()); MemoryLogRecords logRecords = MemoryLogRecords.pointToByteBuffer(recordBuffer); TableBucket tb = @@ -907,9 +917,23 @@ public static Map getProduceLogData( ? produceLogReqForBucket.getPartitionId() : null, produceLogReqForBucket.getBucketId()); - produceEntryData.put(tb, logRecords); + String originalPartitionName = + produceLogReqForBucket.hasOriginalPartitionName() + ? produceLogReqForBucket.getOriginalPartitionName() + : null; + Set originalPartitions = + originalPartitionsByBucket.computeIfAbsent(tb, ignored -> new HashSet<>()); + if (!originalPartitions.add(originalPartitionName)) { + throw new IllegalArgumentException( + "A ProduceLog request contains duplicate table bucket " + + tb + + " and original partition " + + originalPartitionName + + '.'); + } + produceLogData.add(new ProduceLogDataForBucket(tb, logRecords, originalPartitionName)); } - return produceEntryData; + return produceLogData; } public static ProduceLogResponse makeProduceLogResponse( @@ -923,6 +947,9 @@ public static ProduceLogResponse makeProduceLogResponse( if (tableBucket.getPartitionId() != null) { producedBucket.setPartitionId(tableBucket.getPartitionId()); } + if (bucketResult.getOriginalPartitionName() != null) { + producedBucket.setOriginalPartitionName(bucketResult.getOriginalPartitionName()); + } if (bucketResult.failed()) { producedBucket.setError( diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java index 8ba8204a54c..0bd9ab6f522 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java @@ -214,10 +214,6 @@ private static void checkHistoricalPartition( DataLakeFormat.PAIMON, dataLakeFormat.get())); } - if (!tableDescriptor.hasPrimaryKey()) { - unmetRequirements.add("the table must define a primary key"); - } - int partitionKeyCount = tableDescriptor.getPartitionKeys().size(); if (partitionKeyCount != 1) { unmetRequirements.add( diff --git a/fluss-server/src/test/java/org/apache/fluss/server/DynamicConfigChangeTest.java b/fluss-server/src/test/java/org/apache/fluss/server/DynamicConfigChangeTest.java index 5bc4a1fe8ce..adc6a68c63a 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/DynamicConfigChangeTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/DynamicConfigChangeTest.java @@ -191,6 +191,37 @@ void testAlterLakehouseConfigs() throws Exception { } } + @Test + void testAlterHistoricalKvCleanupIdleTime() throws Exception { + DynamicConfigManager dynamicConfigManager = createManager(new Configuration()); + AtomicReference cleanupIdleTime = new AtomicReference<>(); + dynamicConfigManager.register( + new ServerReconfigurable() { + @Override + public void validate(Configuration newConfig) throws ConfigException {} + + @Override + public void reconfigure(Configuration newConfig) { + cleanupIdleTime.set( + newConfig.get( + ConfigOptions + .SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME)); + } + }); + dynamicConfigManager.startup(); + + alterConfig( + dynamicConfigManager, + ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME.key(), + "5min"); + + assertThat(cleanupIdleTime.get()).isEqualTo(Duration.ofMinutes(5)); + assertThat(zookeeperClient.fetchEntityConfig()) + .containsEntry( + ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME.key(), + "5min"); + } + @Test void testOverrideConfigs() throws Exception { Configuration configuration = new Configuration(); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerTest.java index db4a88bc085..3364aa61b08 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerTest.java @@ -67,6 +67,7 @@ import org.apache.fluss.server.entity.FetchReqInfo; import org.apache.fluss.server.entity.NotifyLeaderAndIsrData; import org.apache.fluss.server.entity.NotifyLeaderAndIsrResultForBucket; +import org.apache.fluss.server.entity.ProduceLogDataForBucket; import org.apache.fluss.server.entity.StopReplicaData; import org.apache.fluss.server.entity.StopReplicaResultForBucket; import org.apache.fluss.server.kv.KvTablet; @@ -238,6 +239,32 @@ tb, genMemoryLogRecordsByObject(DATA1)), "Unknown table or bucket: TableBucket{tableId=10001, bucket=0}"))); } + @Test + void testProduceHistoricalLogBatchesToSameTableBucket() throws Exception { + replicaManager.getDiskUsageMonitor().update(0.10); + TableBucket tableBucket = new TableBucket(DATA1_TABLE_ID, 1); + makeLogTableAsLeader(tableBucket.getBucket()); + + CompletableFuture> future = new CompletableFuture<>(); + replicaManager.appendHistoricalRecordsToLog( + 20_000, + 1, + Arrays.asList( + new ProduceLogDataForBucket( + tableBucket, genMemoryLogRecordsByObject(DATA1), "dt=2025-01-01"), + new ProduceLogDataForBucket( + tableBucket, genMemoryLogRecordsByObject(DATA1), "dt=2025-01-02")), + null, + future::complete); + + assertThat(future.get()) + .containsExactlyInAnyOrder( + ProduceLogResultForBucket.historicalSuccess( + tableBucket, 0L, 10L, "dt=2025-01-01"), + ProduceLogResultForBucket.historicalSuccess( + tableBucket, 10L, 20L, "dt=2025-01-02")); + } + @Test void testFetchLog() throws Exception { SchemaGetter schemaGetter = diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java index b18f96b3a7a..eea8578ff2a 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java @@ -22,6 +22,7 @@ import org.apache.fluss.config.AutoPartitionTimeUnit; import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; +import org.apache.fluss.exception.ConfigException; import org.apache.fluss.exception.HistoricalPartitionThrottledException; import org.apache.fluss.exception.InvalidPartitionException; import org.apache.fluss.lake.lakestorage.LakeTableLookuper; @@ -83,6 +84,7 @@ import com.github.benmanes.caffeine.cache.Scheduler; import com.github.benmanes.caffeine.cache.Ticker; import org.junit.jupiter.api.Test; +import org.rocksdb.FlushOptions; import javax.annotation.Nullable; @@ -123,6 +125,10 @@ class HistoricalPartitionManagerTest extends ReplicaTestBase { private static final String ANOTHER_ORIGINAL_PARTITION = "20240108"; private static final String HISTORICAL_PARTITION = HISTORICAL_PARTITION_VALUE; private static final TableBucket TABLE_BUCKET = new TableBucket(TABLE_ID, PARTITION_ID, 0); + private static final RowType HISTORICAL_KEY_TYPE = + DataTypes.ROW( + new DataField("id", DataTypes.INT()), + new DataField("region", DataTypes.STRING())); @Test void testResolvesMultipleLakeMissesWithoutPrewriteRollback() throws Exception { @@ -728,6 +734,291 @@ void testRecoversHistoricalOverlayFromLakeCommitOffset() throws Exception { } } + @Test + void testCleansFullyTieredHistoricalOverlayAfterWriteIdleTime() throws Exception { + TableInfo tableInfo = registerHistoricalTableAndBecomeLeader(); + Replica replica = replicaManager.getReplicaOrException(TABLE_BUCKET); + KvTablet originalKvTablet = replica.getKvTablet(); + assertThat(originalKvTablet).isNotNull(); + + Configuration cleanupConf = lookupConfiguration(); + cleanupConf.set( + ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME, + Duration.ofMinutes(1)); + ManuallyTriggeredScheduledExecutorService executor = + new ManuallyTriggeredScheduledExecutorService(); + TestingHistoricalLakeLookupManager lakeLookupManager = + new TestingHistoricalLakeLookupManager(cleanupConf); + HistoricalPartitionManager historicalPartitionManager = + createCleanupManager( + cleanupConf, executor, lakeLookupManager, replica, Long.MAX_VALUE); + + byte[] primaryKey = new CompactedKeyEncoder(HISTORICAL_KEY_TYPE).encodeKey(row(1, "us")); + Object[] valueObjects = new Object[] {1, "us", ORIGINAL_PARTITION, "v1"}; + byte[] lakeValue = + ValueEncoder.encodeValue( + (short) tableInfo.getSchemaId(), + compactedRow(tableInfo.getRowType(), valueObjects)); + lakeLookupManager.putLakeValue(ORIGINAL_PARTITION, lakeValue); + + try { + CompletableFuture putFuture = + putHistoricalRecords( + historicalPartitionManager, + replica, + batch( + HISTORICAL_KEY_TYPE, + tableInfo.getRowType(), + Tuple2.of(new Object[] {1, "us"}, valueObjects))); + executor.triggerAll(); + assertThat(putFuture.get(10, TimeUnit.SECONDS).failed()).isFalse(); + flushAndWait(originalKvTablet, Long.MAX_VALUE); + + manualClock.advanceTime(Duration.ofMinutes(1)); + // The idle policy cannot clean until lake progress covers the local WAL. + assertThat(executor.numQueuedRunnables()).isZero(); + assertThat(replica.getKvTablet()).isSameAs(originalKvTablet); + + long tieredOffset = replica.getLocalLogEndOffset(); + historicalPartitionManager.onLakeProgress(replica, 9L, tieredOffset - 1); + assertThat(executor.numQueuedRunnables()).isZero(); + + replica.getLogTablet().updateLakeLogEndOffset(tieredOffset); + historicalPartitionManager.onLakeProgress(replica, 10L, tieredOffset); + assertThat(executor.numQueuedRunnables()).isOne(); + assertThat(lakeLookupManager.requiredLakeSnapshotIds).isEmpty(); + + // A write admitted after idle cleanup was scheduled must either cancel that cleanup + // or run against the newly created overlay. It must never be lost during the reset. + CompletableFuture laterPut = + putHistoricalRecords( + historicalPartitionManager, + replica, + batch( + HISTORICAL_KEY_TYPE, + tableInfo.getRowType(), + Tuple2.of(new Object[] {1, "us"}, valueObjects))); + executor.triggerAll(); + assertThat(laterPut.get(10, TimeUnit.SECONDS).failed()).isFalse(); + assertThat(replica.getKvTablet()).isSameAs(originalKvTablet); + assertThat(lakeLookupManager.requiredLakeSnapshotIds).isEmpty(); + + flushAndWait(originalKvTablet, Long.MAX_VALUE); + manualClock.advanceTime(Duration.ofMinutes(1)); + long latestTieredOffset = replica.getLocalLogEndOffset(); + replica.getLogTablet().updateLakeLogEndOffset(latestTieredOffset); + + Configuration longerIdleTimeConf = new Configuration(cleanupConf); + longerIdleTimeConf.set( + ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME, + Duration.ofMinutes(2)); + historicalPartitionManager.validate(longerIdleTimeConf); + historicalPartitionManager.reconfigure(longerIdleTimeConf); + historicalPartitionManager.onLakeProgress(replica, 11L, latestTieredOffset); + assertThat(lakeLookupManager.requiredLakeSnapshotIds).isEmpty(); + executor.triggerAll(); + assertThat(replica.getKvTablet()).isSameAs(originalKvTablet); + + Configuration shorterIdleTimeConf = new Configuration(cleanupConf); + historicalPartitionManager.validate(shorterIdleTimeConf); + historicalPartitionManager.reconfigure(shorterIdleTimeConf); + historicalPartitionManager.onLakeProgress(replica, 12L, latestTieredOffset); + executor.triggerAll(); + + KvTablet cleanedKvTablet = replica.getKvTablet(); + assertThat(cleanedKvTablet).isNotNull().isNotSameAs(originalKvTablet); + assertThat(lakeLookupManager.requiredLakeSnapshotIds).containsExactly(12L); + assertThat(cleanedKvTablet.getRocksDBKv().limitScan(10)).isEmpty(); + assertThat(cleanedKvTablet.lookupHistoricalLocal(ORIGINAL_PARTITION, primaryKey)) + .isEqualTo(KvStateLookupResult.notFound()); + + CompletableFuture lookupFuture = + historicalPartitionManager.lookup( + replica, + new LookupDataForBucket( + TABLE_BUCKET, + Collections.singletonList(primaryKey), + ORIGINAL_PARTITION), + (lookupTimeNanos, lookupFileDownloaded) -> {}); + executor.triggerAll(); + assertThat(lookupFuture.get(10, TimeUnit.SECONDS).lookupValues()) + .containsExactly(lakeValue); + } finally { + historicalPartitionManager.close(); + } + } + + @Test + void testRejectsNegativeCleanupIdleTimeDuringReconfiguration() { + Configuration cleanupConf = lookupConfiguration(); + ManuallyTriggeredScheduledExecutorService executor = + new ManuallyTriggeredScheduledExecutorService(); + HistoricalPartitionManager historicalPartitionManager = + new HistoricalPartitionManager( + cleanupConf, + new HistoricalPartitionTaskExecutor(cleanupConf, executor), + new TestingHistoricalLakeLookupManager(cleanupConf), + manualClock, + Long.MAX_VALUE); + Configuration invalidConf = new Configuration(cleanupConf); + invalidConf.set( + ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME, + Duration.ofMillis(-1)); + + try { + assertThatThrownBy(() -> historicalPartitionManager.validate(invalidConf)) + .isInstanceOf(ConfigException.class) + .hasMessageContaining( + ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME.key(), + "must not be negative"); + } finally { + historicalPartitionManager.close(); + } + } + + @Test + void testCleanupRequiresLakeAndLocalOffsetsToMatch() throws Exception { + TableInfo tableInfo = registerHistoricalTableAndBecomeLeader(); + Replica replica = replicaManager.getReplicaOrException(TABLE_BUCKET); + KvTablet originalKvTablet = replica.getKvTablet(); + assertThat(originalKvTablet).isNotNull(); + + Configuration cleanupConf = lookupConfiguration(); + cleanupConf.set( + ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME, + Duration.ofMinutes(1)); + ManuallyTriggeredScheduledExecutorService executor = + new ManuallyTriggeredScheduledExecutorService(); + TestingHistoricalLakeLookupManager lakeLookupManager = + new TestingHistoricalLakeLookupManager(cleanupConf); + HistoricalPartitionManager historicalPartitionManager = + createCleanupManager( + cleanupConf, executor, lakeLookupManager, replica, Long.MAX_VALUE); + + KvRecordBatch records = + batch( + HISTORICAL_KEY_TYPE, + tableInfo.getRowType(), + Tuple2.of( + new Object[] {1, "us"}, + new Object[] {1, "us", ORIGINAL_PARTITION, "v1"})); + + try { + CompletableFuture firstPut = + putHistoricalRecords(historicalPartitionManager, replica, records); + executor.triggerAll(); + assertThat(firstPut.get(10, TimeUnit.SECONDS).failed()).isFalse(); + + long firstTieredOffset = replica.getLocalLogEndOffset(); + replica.getLogTablet().updateLakeLogEndOffset(firstTieredOffset); + historicalPartitionManager.onLakeProgress(replica, 10L, firstTieredOffset); + executor.triggerAll(); + assertThat(replica.getKvTablet()).isSameAs(originalKvTablet); + assertThat(lakeLookupManager.requiredLakeSnapshotIds).isEmpty(); + + CompletableFuture secondPut = + putHistoricalRecords(historicalPartitionManager, replica, records); + manualClock.advanceTime(Duration.ofMinutes(1)); + historicalPartitionManager.onLakeProgress(replica, 10L, firstTieredOffset); + + // The second write runs before the cleanup that captured snapshot 10 / first offset. + executor.trigger(); + assertThat(secondPut.get(10, TimeUnit.SECONDS).failed()).isFalse(); + long secondTieredOffset = replica.getLocalLogEndOffset(); + assertThat(secondTieredOffset).isGreaterThan(firstTieredOffset); + + manualClock.advanceTime(Duration.ofMinutes(1)); + // A lake offset beyond the local end is inconsistent and must not schedule cleanup. + historicalPartitionManager.onLakeProgress(replica, 11L, secondTieredOffset + 1); + executor.trigger(); + + // Snapshot 10 does not cover the second write, so its queued cleanup must be skipped. + assertThat(replica.getKvTablet()).isSameAs(originalKvTablet); + assertThat(lakeLookupManager.requiredLakeSnapshotIds).isEmpty(); + assertThat(executor.numQueuedRunnables()).isZero(); + + replica.getLogTablet().updateLakeLogEndOffset(secondTieredOffset); + historicalPartitionManager.onLakeProgress(replica, 12L, secondTieredOffset); + executor.triggerAll(); + assertThat(replica.getKvTablet()).isNotNull().isNotSameAs(originalKvTablet); + assertThat(lakeLookupManager.requiredLakeSnapshotIds).containsExactly(12L); + } finally { + historicalPartitionManager.close(); + } + } + + @Test + void testMaxSizeBlocksWritesUntilOverlayIsTieredAndCleaned() throws Exception { + TableInfo tableInfo = registerHistoricalTableAndBecomeLeader(); + Replica replica = replicaManager.getReplicaOrException(TABLE_BUCKET); + KvTablet originalKvTablet = replica.getKvTablet(); + assertThat(originalKvTablet).isNotNull(); + + Configuration cleanupConf = lookupConfiguration(); + cleanupConf.set( + ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME, Duration.ZERO); + ManuallyTriggeredScheduledExecutorService executor = + new ManuallyTriggeredScheduledExecutorService(); + HistoricalPartitionManager historicalPartitionManager = + createCleanupManager( + cleanupConf, + executor, + new TestingHistoricalLakeLookupManager(cleanupConf), + replica, + 1L); + + KvRecordBatch firstBatch = + batch( + HISTORICAL_KEY_TYPE, + tableInfo.getRowType(), + Tuple2.of( + new Object[] {1, "us"}, + new Object[] {1, "us", ORIGINAL_PARTITION, "v1"})); + KvRecordBatch secondBatch = + batch( + HISTORICAL_KEY_TYPE, + tableInfo.getRowType(), + Tuple2.of( + new Object[] {2, "eu"}, + new Object[] {2, "eu", ORIGINAL_PARTITION, "v2"})); + + try { + CompletableFuture firstPut = + putHistoricalRecords(historicalPartitionManager, replica, firstBatch); + executor.triggerAll(); + assertThat(firstPut.get(10, TimeUnit.SECONDS).failed()).isFalse(); + flushAndWait(originalKvTablet, Long.MAX_VALUE); + try (FlushOptions flushOptions = new FlushOptions().setWaitForFlush(true)) { + originalKvTablet.getRocksDBKv().getDb().flush(flushOptions); + } + assertThat(originalKvTablet.liveSstFilesSize()).isPositive(); + + PutKvResultForBucket blockedWrite = + putHistoricalRecords(historicalPartitionManager, replica, secondBatch) + .get(10, TimeUnit.SECONDS); + assertThat(blockedWrite.getError().error()) + .isEqualTo(Errors.HISTORICAL_PARTITION_THROTTLED); + assertThat(blockedWrite.getError().message()) + .contains( + "reached the live SST maximum size of 1 bytes", + "lake tiering covers all previously accepted writes"); + assertThat(executor.numQueuedRunnables()).isZero(); + + long tieredOffset = replica.getLocalLogEndOffset(); + replica.getLogTablet().updateLakeLogEndOffset(tieredOffset); + historicalPartitionManager.onLakeProgress(replica, 11L, tieredOffset); + executor.triggerAll(); + assertThat(replica.getKvTablet()).isNotNull().isNotSameAs(originalKvTablet); + + CompletableFuture resumedWrite = + putHistoricalRecords(historicalPartitionManager, replica, secondBatch); + executor.triggerAll(); + assertThat(resumedWrite.get(10, TimeUnit.SECONDS).failed()).isFalse(); + } finally { + historicalPartitionManager.close(); + } + } + @Test void testHistoricalLookupThrottledWhenPermitsExhausted() throws Exception { registerHistoricalTableAndBecomeLeader(); @@ -936,6 +1227,33 @@ private static void await(CountDownLatch latch) { } } + private HistoricalPartitionManager createCleanupManager( + Configuration configuration, + ManuallyTriggeredScheduledExecutorService executor, + TestingHistoricalLakeLookupManager lakeLookupManager, + Replica replica, + long maxHistoricalKvSizeBytes) { + HistoricalPartitionManager manager = + new HistoricalPartitionManager( + configuration, + new HistoricalPartitionTaskExecutor(configuration, executor), + lakeLookupManager, + manualClock, + maxHistoricalKvSizeBytes); + manager.onLeaderActivated(replica); + return manager; + } + + private static CompletableFuture putHistoricalRecords( + HistoricalPartitionManager manager, Replica replica, KvRecordBatch records) { + return manager.putKv( + replica, + new PutKvDataForBucket(TABLE_BUCKET, records, ORIGINAL_PARTITION), + null, + MergeMode.DEFAULT, + 1); + } + @SafeVarargs private static KvRecordBatch batch( RowType keyType, RowType rowType, Tuple2... keyAndValues) @@ -971,6 +1289,7 @@ private final class TestingHistoricalLakeLookupManager extends HistoricalLakeLoo private final AtomicInteger lookupCount = new AtomicInteger(); private final AtomicInteger lookupBatchCount = new AtomicInteger(); private final Map lakeValuesByPartition = new HashMap<>(); + private final List requiredLakeSnapshotIds = new ArrayList<>(); private volatile @Nullable Runnable lookupHook; private TestingHistoricalLakeLookupManager(Configuration configuration) { @@ -992,6 +1311,12 @@ private void setLookupHook(Runnable lookupHook) { this.lookupHook = lookupHook; } + @Override + void requireLakeSnapshot(long tableId, long snapshotId) { + requiredLakeSnapshotIds.add(snapshotId); + super.requireLakeSnapshot(tableId, snapshotId); + } + @Override List lookup( LookupDataForBucket lookupData, diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionTaskExecutorTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionTaskExecutorTest.java index 98574e7d9fd..4293fc3d0df 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionTaskExecutorTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionTaskExecutorTest.java @@ -148,6 +148,37 @@ void testOrderedTaskContinuesAfterPreviousFailure() throws Exception { assertThat(taskExecutor.numInflightRequests()).isZero(); } + @Test + void testMaintenanceTaskKeepsOrderWithoutConsumingRequestPermit() throws Exception { + ManualExecutor executor = new ManualExecutor(); + HistoricalPartitionTaskExecutor taskExecutor = + new HistoricalPartitionTaskExecutor(configuration(1), executor); + List executionOrder = new ArrayList<>(); + + CompletableFuture write = + taskExecutor.submitOrdered( + "bucket", + () -> { + executionOrder.add("write"); + return "written"; + }, + () -> "throttled"); + CompletableFuture maintenance = + taskExecutor.submitOrderedMaintenance( + "bucket", () -> executionOrder.add("maintenance")); + + assertThat(taskExecutor.numInflightRequests()).isOne(); + assertThat(taskExecutor.submitOrdered("another-bucket", () -> "written", () -> "throttled")) + .isCompletedWithValue("throttled"); + + executor.runNext(); + executor.runNext(); + assertThat(write).isCompletedWithValue("written"); + assertThat(maintenance).isDone(); + assertThat(executionOrder).containsExactly("write", "maintenance"); + assertThat(taskExecutor.numInflightRequests()).isZero(); + } + @Test void testRejectNonPositiveRequestLimit() { assertThatThrownBy( diff --git a/fluss-server/src/test/java/org/apache/fluss/server/utils/HistoricalPartitionTableValidationTest.java b/fluss-server/src/test/java/org/apache/fluss/server/utils/HistoricalPartitionTableValidationTest.java index cb427c23ec8..7fa4a90a1d6 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/utils/HistoricalPartitionTableValidationTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/utils/HistoricalPartitionTableValidationTest.java @@ -26,6 +26,7 @@ import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThatCode; import static org.assertj.core.api.Assertions.assertThatThrownBy; class HistoricalPartitionTableValidationTest { @@ -54,7 +55,6 @@ void testReportsAllUnmetHistoricalPartitionRequirements() { + "'table.datalake.enabled' must be set to true; " + "'table.datalake.format' must be set to 'paimon' " + "(currently not set); " - + "the table must define a primary key; " + "the table must define exactly one partition key (found 0)."); // Case 2: Aggregate requirements before related validators can report only one failure. @@ -80,7 +80,32 @@ void testReportsAllUnmetHistoricalPartitionRequirements() { "'table.datalake.historical-partition.enabled' has unmet requirements: " + "'table.datalake.format' must be set to 'paimon' " + "(currently 'iceberg'); " - + "the table must define a primary key; " + "the table must define exactly one partition key (found 0)."); } + + @Test + void testAllowsHistoricalPartitionForLogTable() { + TableDescriptor logTableDescriptor = + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("dt", DataTypes.STRING()) + .build()) + .partitionedBy("dt") + .distributedBy(1, "id") + .property(ConfigOptions.TABLE_REPLICATION_FACTOR, 1) + .property(ConfigOptions.TABLE_AUTO_PARTITION_ENABLED, true) + .property(ConfigOptions.TABLE_AUTO_PARTITION_KEY, "dt") + .property(ConfigOptions.TABLE_DATALAKE_ENABLED, true) + .property(ConfigOptions.TABLE_DATALAKE_FORMAT, DataLakeFormat.PAIMON) + .property(ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_ENABLED, true) + .build(); + + assertThatCode( + () -> + TableDescriptorValidation.validateTableDescriptor( + logTableDescriptor, 100, DataLakeFormat.PAIMON)) + .doesNotThrowAnyException(); + } } diff --git a/fluss-server/src/test/java/org/apache/fluss/server/utils/ServerRpcMessageUtilsTest.java b/fluss-server/src/test/java/org/apache/fluss/server/utils/ServerRpcMessageUtilsTest.java index 9f318eded4c..c5ed0c48a6b 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/utils/ServerRpcMessageUtilsTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/utils/ServerRpcMessageUtilsTest.java @@ -21,12 +21,18 @@ import org.apache.fluss.record.KvRecordBatch; import org.apache.fluss.row.encode.KvValueLayout; import org.apache.fluss.rpc.entity.LookupResultForBucket; +import org.apache.fluss.rpc.entity.ProduceLogResultForBucket; import org.apache.fluss.rpc.entity.PutKvResultForBucket; import org.apache.fluss.rpc.messages.LookupResponse; +import org.apache.fluss.rpc.messages.PbProduceLogReqForBucket; +import org.apache.fluss.rpc.messages.PbProduceLogRespForBucket; import org.apache.fluss.rpc.messages.PbPutKvReqForBucket; import org.apache.fluss.rpc.messages.PbPutKvRespForBucket; +import org.apache.fluss.rpc.messages.ProduceLogRequest; +import org.apache.fluss.rpc.messages.ProduceLogResponse; import org.apache.fluss.rpc.messages.PutKvRequest; import org.apache.fluss.rpc.messages.PutKvResponse; +import org.apache.fluss.server.entity.ProduceLogDataForBucket; import org.apache.fluss.server.entity.PutKvDataForBucket; import org.junit.jupiter.api.Test; @@ -64,6 +70,52 @@ void testLookupResponseSupportsMixedValueLayouts() { .containsExactly(new byte[] {1, 0, 11, 12}, new byte[] {1, 0, 21, 22}); } + @Test + void testHistoricalProduceLogRequestAndResponsePreserveOriginalPartitions() { + long tableId = 1L; + long partitionId = 2L; + PbProduceLogReqForBucket firstBucketRequest = + new PbProduceLogReqForBucket() + .setPartitionId(partitionId) + .setBucketId(0) + .setRecords(new byte[0]) + .setOriginalPartitionName("dt=2025-01-01"); + ProduceLogRequest request = + new ProduceLogRequest().setTableId(tableId).setAcks(1).setTimeoutMs(10_000); + request.addAllBucketsReqs( + Arrays.asList( + firstBucketRequest, + new PbProduceLogReqForBucket() + .copyFrom(firstBucketRequest) + .setOriginalPartitionName("dt=2025-01-02"))); + + TableBucket tableBucket = new TableBucket(tableId, partitionId, 0); + List decoded = + ServerRpcMessageUtils.toProduceLogDataForBuckets(request); + assertThat(decoded) + .extracting(ProduceLogDataForBucket::tableBucket) + .containsOnly(tableBucket); + assertThat(decoded) + .extracting(ProduceLogDataForBucket::originalPartitionName) + .containsExactly("dt=2025-01-01", "dt=2025-01-02"); + + ProduceLogResponse response = + ServerRpcMessageUtils.makeProduceLogResponse( + Arrays.asList( + ProduceLogResultForBucket.historicalSuccess( + tableBucket, 0L, 1L, "dt=2025-01-01"), + ProduceLogResultForBucket.historicalSuccess( + tableBucket, 1L, 2L, "dt=2025-01-02"))); + assertThat(response.getBucketsRespsList()) + .extracting(PbProduceLogRespForBucket::getOriginalPartitionName) + .containsExactly("dt=2025-01-01", "dt=2025-01-02"); + + request.addAllBucketsReqs(Collections.singletonList(firstBucketRequest)); + assertThatThrownBy(() -> ServerRpcMessageUtils.toProduceLogDataForBuckets(request)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("duplicate table bucket"); + } + @Test void testHistoricalPutKvRequestAndResponsePreserveOriginalPartitions() throws Exception { long tableId = 1L; From 39a3df2ad13997ae3309fb805c25960640182599 Mon Sep 17 00:00:00 2001 From: luoyuxia Date: Thu, 27 Aug 2026 19:28:52 +0800 Subject: [PATCH 2/8] [client][server][paimon] Refine historical partition writes Simplify historical write routing, request handling, and Paimon tiering integration while removing redundant tests. Co-Authored-By: Codex AI-Model: gpt-5 AI-Contributed/Feature: 446/446 AI-Contributed/UT: 576/576 --- .../fluss/client/write/RecordAccumulator.java | 124 +++++++---- .../org/apache/fluss/client/write/Sender.java | 105 +++++---- .../fluss/client/write/WriterClient.java | 45 ++-- .../client/write/RecordAccumulatorTest.java | 88 -------- .../apache/fluss/client/write/SenderTest.java | 203 +----------------- .../apache/fluss/config/ConfigOptions.java | 2 +- .../source/split/TieringSplitGenerator.java | 48 +++-- .../lake/paimon/tiering/RecordWriter.java | 1 + .../tiering/mergetree/MergeTreeWriter.java | 18 -- .../lookup/HistoricalPartitionITCase.java | 68 ------ .../paimon/tiering/PaimonTieringTest.java | 113 ++++------ .../rpc/entity/ProduceLogResultForBucket.java | 1 + .../netty/client/ServerConnectionTest.java | 14 +- .../entity/ProduceLogDataForBucket.java | 1 + .../apache/fluss/server/replica/Replica.java | 82 ++++--- .../fluss/server/replica/ReplicaManager.java | 19 +- .../fluss/server/DynamicConfigChangeTest.java | 32 +-- .../HistoricalPartitionTaskExecutorTest.java | 31 --- ...istoricalPartitionTableValidationTest.java | 27 --- 19 files changed, 285 insertions(+), 737 deletions(-) diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java b/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java index 7546e48b8e5..84ee7331e19 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java @@ -342,41 +342,67 @@ public void reEnqueue(ReadyWriteBatch readyWriteBatch) { } /** - * Tries to route writes for an original partition path to the given physical target. + * Routes writes for an original partition path to the given physical target. * *

The accumulator keeps queues keyed by {@code originalPath}, while metadata lookup, leader * discovery, and RPC sending use {@code targetPath}. The target may therefore be either the * original partition itself or the shared historical partition. * - *

The first queue creation fixes the target for that original path. A later call succeeds - * only when it selects the same target; this method never moves queued or inflight batches - * between physical partitions. + *

A normal target can be replaced by the historical target only while the original path has + * no incomplete batch. This method never moves queued or inflight batches between physical + * partitions. * - * @return true if the target was installed or already matches, false if a different target was - * fixed previously + * @throws FlussRuntimeException if a different target was fixed previously */ - boolean tryRouteWritesTo( + void routeWritesTo( PhysicalTablePath originalPath, PhysicalTablePath targetPath, long targetPartitionId) { BucketAndWriteBatches resolvedTarget = new BucketAndWriteBatches(targetPartitionId, true, targetPath); // Install the route atomically before append can create the first queue for this path. BucketAndWriteBatches existing = writeBatches.putIfAbsent(originalPath, resolvedTarget); if (existing == null) { - return true; + return; } - // An append may already have fixed this path to a target. Keep that target and only accept - // the metadata result when it describes the same physical partition. - if (!existing.targetPath.equals(targetPath)) { - return false; + synchronized (existing) { + if (existing.targetPath.equals(targetPath)) { + existing.partitionId = targetPartitionId; + return; + } + if (!existing.isHistoricalWriteTarget() && resolvedTarget.isHistoricalWriteTarget()) { + if (hasIncompleteBatchFor(originalPath)) { + throw new FlussRuntimeException( + String.format( + "Cannot route writes for %s to %s while this writer has " + + "incomplete writes to %s.", + originalPath, targetPath, existing.targetPath)); + } + existing.targetPath = targetPath; + existing.partitionId = targetPartitionId; + return; + } + + throw new FlussRuntimeException( + String.format( + "Cannot route writes for %s to %s because this writer already routed " + + "the partition to %s.", + originalPath, targetPath, existing.targetPath)); } - existing.partitionId = targetPartitionId; - return true; } - /** Returns whether a write target has already been chosen for this original path. */ - boolean hasWriteTarget(PhysicalTablePath originalPath) { - return writeBatches.containsKey(originalPath); + private boolean hasIncompleteBatchFor(PhysicalTablePath physicalTablePath) { + for (WriteBatch batch : incomplete.copyAll()) { + if (batch.physicalTablePath().equals(physicalTablePath)) { + return true; + } + } + return false; + } + + /** Returns whether this original path is already routed to the historical partition. */ + boolean hasHistoricalWriteTarget(PhysicalTablePath originalPath) { + BucketAndWriteBatches writeTarget = writeBatches.get(originalPath); + return writeTarget != null && writeTarget.isHistoricalWriteTarget(); } /** Returns whether the target belongs to a table with historical partition support enabled. */ @@ -671,41 +697,43 @@ private RecordAppendResult appendNewBatch( Deque deque, List segments) throws Exception { - RecordAppendResult appendResult = tryAppend(writeRecord, callback, deque); - if (appendResult != null) { - // Somebody else found us a batch, return the one we waited for! Hopefully this doesn't - // happen often... - return appendResult; - } - PhysicalTablePath physicalTablePath = writeRecord.getPhysicalTablePath(); - PreAllocatedPagedOutputView outputView = new PreAllocatedPagedOutputView(segments); - int schemaId = tableInfo.getSchemaId(); - WriteFormat writeFormat = writeRecord.getWriteFormat(); BucketAndWriteBatches bucketAndWriteBatches = checkNotNull( writeBatches.get(physicalTablePath), "Write batches for %s must exist.", physicalTablePath); - String originalPartitionName = - bucketAndWriteBatches.isHistoricalWriteTarget() - ? checkNotNull(physicalTablePath.getPartitionName()) - : null; - final WriteBatch batch = - createWriteBatch( - writeRecord, - bucketId, - tableInfo, - writeFormat, - physicalTablePath, - outputView, - schemaId, - originalPartitionName); + synchronized (bucketAndWriteBatches) { + RecordAppendResult appendResult = tryAppend(writeRecord, callback, deque); + if (appendResult != null) { + // Somebody else found us a batch, return the one we waited for! Hopefully this + // doesn't happen often... + return appendResult; + } - batch.tryAppend(writeRecord, callback); - deque.addLast(batch); - incomplete.add(batch); - return new RecordAppendResult(deque.size() > 1 || batch.isClosed(), true, false); + PreAllocatedPagedOutputView outputView = new PreAllocatedPagedOutputView(segments); + int schemaId = tableInfo.getSchemaId(); + WriteFormat writeFormat = writeRecord.getWriteFormat(); + String originalPartitionName = + bucketAndWriteBatches.isHistoricalWriteTarget() + ? checkNotNull(physicalTablePath.getPartitionName()) + : null; + final WriteBatch batch = + createWriteBatch( + writeRecord, + bucketId, + tableInfo, + writeFormat, + physicalTablePath, + outputView, + schemaId, + originalPartitionName); + + batch.tryAppend(writeRecord, callback); + deque.addLast(batch); + incomplete.add(batch); + return new RecordAppendResult(deque.size() > 1 || batch.isClosed(), true, false); + } } private WriteBatch createWriteBatch( @@ -1084,6 +1112,8 @@ private List getAllBucketsInCurrentNode(Integer currentNode, Clu Set physicalTablePaths = cluster.getBucketLocationsByPath().keySet(); for (PhysicalTablePath path : physicalTablePaths) { BucketAndWriteBatches bucketAndWriteBatches = writeBatches.get(path); + // A historical route uses the original path only as the accumulator queue key. Its + // actual bucket locations come from the historical target and are added below. if (bucketAndWriteBatches != null && bucketAndWriteBatches.isHistoricalWriteTarget()) { continue; } @@ -1111,6 +1141,8 @@ private List getAllBucketsInCurrentNode(Integer currentNode, Clu bucketAndWriteBatches.targetPath)) { if (bucketLocation.getLeader() != null && Objects.equals(currentNode, bucketLocation.getLeader())) { + // Keep the original path so drain can find its original-keyed queue. The + // TableBucket, leader, and replicas still describe the historical RPC target. buckets.add( new BucketLocation( originalPath, @@ -1260,7 +1292,7 @@ public void destroyResources() { private static class BucketAndWriteBatches { public final boolean isPartitionedTable; /** The physical partition used for metadata lookup, leader discovery, and write RPCs. */ - private final PhysicalTablePath targetPath; + private volatile PhysicalTablePath targetPath; public volatile @Nullable Long partitionId; // Write batches for each bucket in queue. diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java b/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java index 1ecd39881ef..06f03ca1991 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java @@ -410,38 +410,24 @@ private void sendWriteRequest(int destination, short acks, List batches); } else { writeBatchByTable.forEach( - (tableId, writeBatches) -> { - boolean logBatches = isLogBatches(writeBatches); - for (List requestGroup : packRequestGroups(writeBatches)) { - if (logBatches) { - sendProduceLogRequestAndHandleResponse( - gateway, - makeProduceLogRequest( - tableId, acks, maxRequestTimeoutMs, requestGroup), - tableId, - requestGroup); - } else { - sendPutKvRequestAndHandleResponse( - gateway, - makePutKvRequest( - tableId, acks, maxRequestTimeoutMs, requestGroup), - tableId, - requestGroup); - } - } - }); + (tableId, writeBatches) -> + sendWriteRequestsForTable(gateway, tableId, acks, writeBatches)); } } /** - * Splits normal and historical batches into separate requests. + * Sends normal and historical batches in separate requests. * *

Normal and historical writes cannot share a request. Both write protocols correlate * historical responses by {@link TableBucket} and original partition name, so different * original partitions targeting the same historical table bucket can remain in one request. */ - private static List> packRequestGroups( + private void sendWriteRequestsForTable( + TabletServerGateway gateway, + long tableId, + short acks, List writeBatches) { + boolean logBatches = isLogBatches(writeBatches); List normalBatches = new ArrayList<>(); List historicalBatches = new ArrayList<>(); @@ -453,32 +439,50 @@ private static List> packRequestGroups( } } - List> requestGroups = new ArrayList<>(2); - if (!normalBatches.isEmpty()) { - requestGroups.add(normalBatches); + sendBatchesInRequest(gateway, tableId, acks, logBatches, normalBatches); + sendBatchesInRequest(gateway, tableId, acks, logBatches, historicalBatches); + } + + private void sendBatchesInRequest( + TabletServerGateway gateway, + long tableId, + short acks, + boolean logBatches, + List writeBatches) { + if (writeBatches.isEmpty()) { + return; } - if (!historicalBatches.isEmpty()) { - requestGroups.add(historicalBatches); + if (logBatches) { + sendProduceLogRequestAndHandleResponse( + gateway, + makeProduceLogRequest(tableId, acks, maxRequestTimeoutMs, writeBatches), + tableId, + writeBatches); + } else { + sendPutKvRequestAndHandleResponse( + gateway, + makePutKvRequest(tableId, acks, maxRequestTimeoutMs, writeBatches), + tableId, + writeBatches); } - return requestGroups; } - private static Map toBatchesByKey( + private static Map toWriteBatchesByKey( List writeBatches) { - Map recordsByKey = new HashMap<>(); + Map writeBatchesByKey = new HashMap<>(); for (ReadyWriteBatch readyWriteBatch : writeBatches) { WriteBatch writeBatch = readyWriteBatch.writeBatch(); WriteBatchKey key = new WriteBatchKey( readyWriteBatch.tableBucket(), writeBatch.getOriginalPartitionName()); - ReadyWriteBatch previous = recordsByKey.put(key, readyWriteBatch); + ReadyWriteBatch previous = writeBatchesByKey.put(key, readyWriteBatch); checkArgument( previous == null, "A write request contains duplicate table bucket %s and original partition %s.", readyWriteBatch.tableBucket(), writeBatch.getOriginalPartitionName()); } - return recordsByKey; + return writeBatchesByKey; } /** @@ -499,7 +503,7 @@ private void sendProduceLogRequestAndHandleResponse( ProduceLogRequest request, long tableId, List writeBatches) { - Map recordsByKey = toBatchesByKey(writeBatches); + Map writeBatchesByKey = toWriteBatchesByKey(writeBatches); long startTime = System.currentTimeMillis(); gateway.produceLog(request) .whenComplete( @@ -509,7 +513,8 @@ private void sendProduceLogRequestAndHandleResponse( if (e != null) { handleWriteRequestException(e, writeBatches); } else { - handleProduceLogResponse(produceLogResponse, tableId, recordsByKey); + handleProduceLogResponse( + produceLogResponse, tableId, writeBatchesByKey); } }); } @@ -519,7 +524,7 @@ private void sendPutKvRequestAndHandleResponse( PutKvRequest request, long tableId, List writeBatches) { - Map recordsByKey = toBatchesByKey(writeBatches); + Map writeBatchesByKey = toWriteBatchesByKey(writeBatches); long startTime = System.currentTimeMillis(); gateway.putKv(request) .whenComplete( @@ -529,7 +534,7 @@ private void sendPutKvRequestAndHandleResponse( if (e != null) { handleWriteRequestException(e, writeBatches); } else { - handlePutKvResponse(putKvResponse, tableId, recordsByKey); + handlePutKvResponse(putKvResponse, tableId, writeBatchesByKey); } }); } @@ -537,7 +542,7 @@ private void sendPutKvRequestAndHandleResponse( private void handleProduceLogResponse( ProduceLogResponse response, long tableId, - Map recordsByKey) { + Map writeBatchesByKey) { Set invalidMetadataTablesSet = new HashSet<>(); for (PbProduceLogRespForBucket logRespForBucket : response.getBucketsRespsList()) { TableBucket tb = @@ -548,7 +553,7 @@ private void handleProduceLogResponse( : null, logRespForBucket.getBucketId()); ReadyWriteBatch writeBatch = - recordsByKey.get( + writeBatchesByKey.get( new WriteBatchKey( tb, logRespForBucket.hasOriginalPartitionName() @@ -569,7 +574,7 @@ private void handleProduceLogResponse( private void handlePutKvResponse( PutKvResponse putKvResponse, long tableId, - Map recordsByKey) { + Map writeBatchesByKey) { Set invalidMetadataTablesSet = new HashSet<>(); for (PbPutKvRespForBucket respForBucket : putKvResponse.getBucketsRespsList()) { TableBucket tb = @@ -584,7 +589,7 @@ private void handlePutKvResponse( } ReadyWriteBatch writeBatch = - recordsByKey.get( + writeBatchesByKey.get( new WriteBatchKey( tb, respForBucket.hasOriginalPartitionName() @@ -721,6 +726,11 @@ private Set handleWriteBatchException( return invalidMetadataTables; } + /** + * Aborts pending writes when metadata confirms their target is missing. Such writes cannot make + * progress without a leader, and rerouting existing batches to the historical target is unsafe + * because their outcome and idempotent state may belong to the original target. + */ private void abortIfHistoricalWriteTargetMissing(Set unknownLeaderTables) throws Exception { for (PhysicalTablePath targetPath : unknownLeaderTables) { @@ -732,6 +742,18 @@ private void abortIfHistoricalWriteTargetMissing(Set unknownL } catch (Exception e) { Throwable t = ExceptionUtils.stripExecutionException(e); if (t instanceof PartitionNotExistException) { + // This target was considered usable when its batches were enqueued or first + // attempted, but is now confirmed missing. Transparently rerouting those + // batches to the historical partition is unsafe: an original-target attempt + // may have been accepted despite a lost response, and writer ID / batch + // sequence state cannot be reused across different physical TableBuckets. A + // safe failover must stop draining this path, wait for its in-flight requests, + // classify ambiguous outcomes, reset writer state, and preserve per-bucket + // ordering. This race requires partition retirement to overlap a writer that + // still holds the original route, so it is expected to be uncommon; fail + // closed for now. + // TODO: Implement safe in-flight historical failover if this path occurs + // frequently in practice. // Retrying a historical-enabled table without a leader would leave its // batches queued indefinitely. Fail only after checking the target itself so // ordinary writes in the bulk metadata request keep their existing behavior. @@ -823,6 +845,9 @@ void destroyResources() { private static final class WriteBatchKey { private final TableBucket tableBucket; + + // Distinguishes historical writes from different original partitions that share a target + // bucket. Normal writes have no original partition name. private final @Nullable String originalPartitionName; private WriteBatchKey(TableBucket tableBucket, @Nullable String originalPartitionName) { diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/WriterClient.java b/fluss-client/src/main/java/org/apache/fluss/client/write/WriterClient.java index cb46c627bed..e82bc57f8fb 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/WriterClient.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/WriterClient.java @@ -78,7 +78,6 @@ public class WriterClient { private static final Logger LOG = LoggerFactory.getLogger(WriterClient.class); public static final String SENDER_THREAD_PREFIX = "fluss-write-sender"; - private static final Duration MAX_DEFAULT_TIME_ZONE_DIFFERENCE = Duration.ofHours(26); /** * {@link ConfigOptions#CLIENT_WRITER_MAX_INFLIGHT_REQUESTS_PER_BUCKET} should be less than or * equal to this value when idempotence producer enabled to ensure message ordering. @@ -254,6 +253,18 @@ private void doSend(WriteRecord record, WriteCallback callback) { } } + /** + * Returns whether a partition is old enough that it may have expired under its retention + * policy. + * + *

This client-side precheck uses the time zone resolved from the table configuration. A + * {@code true} result does not confirm that the partition is missing; the caller must refresh + * metadata before routing the write to a historical partition. If the table does not explicitly + * configure a time zone and the Client and Coordinator use different defaults, they may + * classify partitions near the retention boundary differently. Late classification may fail a + * write to an already removed original partition, while early classification only causes an + * extra metadata refresh. + */ static boolean mayBeExpiredHistoricalPartition( PhysicalTablePath physicalTablePath, TableInfo tableInfo, Instant now) { String partitionName = physicalTablePath.getPartitionName(); @@ -264,25 +275,19 @@ static boolean mayBeExpiredHistoricalPartition( return false; } - // The table's default time zone is not persisted. Shift the expiration boundary by the - // largest IANA time-zone difference, then apply retention in the table's partition unit. - Instant latestPotentialServerTime = now.plus(MAX_DEFAULT_TIME_ZONE_DIFFERENCE); - if (!isPastAutoPartition(partitionName, strategy, latestPotentialServerTime)) { + if (!isPastAutoPartition(partitionName, strategy, now)) { return false; } - ZonedDateTime latestPotentialServerDateTime = - ZonedDateTime.ofInstant(latestPotentialServerTime, strategy.timeZone().toZoneId()); - String earliestPotentialRetainedPartition = + ZonedDateTime currentDateTime = + ZonedDateTime.ofInstant(now, strategy.timeZone().toZoneId()); + String earliestRetainedPartition = generateAutoPartitionTime( - latestPotentialServerDateTime, - -strategy.numToRetain(), - strategy.timeUnit(), - strategy); - return partitionName.compareTo(earliestPotentialRetainedPartition) < 0; + currentDateTime, -strategy.numToRetain(), strategy.timeUnit(), strategy); + return partitionName.compareTo(earliestRetainedPartition) < 0; } private synchronized void resolveHistoricalWriteTarget(PhysicalTablePath originalPath) { - if (accumulator.hasWriteTarget(originalPath)) { + if (accumulator.hasHistoricalWriteTarget(originalPath)) { return; } @@ -307,16 +312,8 @@ private synchronized void resolveHistoricalWriteTarget(PhysicalTablePath origina } } - if (!accumulator.tryRouteWritesTo( - originalPath, targetPath, metadataUpdater.getPartitionIdOrElseThrow(targetPath))) { - throw new FlussRuntimeException( - "Cannot route writes for " - + originalPath - + " to " - + targetPath - + " because the accumulator already contains writes for a different " - + "physical target."); - } + accumulator.routeWritesTo( + originalPath, targetPath, metadataUpdater.getPartitionIdOrElseThrow(targetPath)); } private void maybeAbortBatches(Throwable t) { diff --git a/fluss-client/src/test/java/org/apache/fluss/client/write/RecordAccumulatorTest.java b/fluss-client/src/test/java/org/apache/fluss/client/write/RecordAccumulatorTest.java index 21939d16888..acd1e4e2911 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/write/RecordAccumulatorTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/write/RecordAccumulatorTest.java @@ -43,7 +43,6 @@ import org.apache.fluss.row.BinaryRow; import org.apache.fluss.row.GenericRow; import org.apache.fluss.row.arrow.ArrowWriter; -import org.apache.fluss.row.encode.CompactedKeyEncoder; import org.apache.fluss.row.indexed.IndexedRow; import org.apache.fluss.rpc.GatewayClientProxy; import org.apache.fluss.rpc.RpcClient; @@ -72,21 +71,14 @@ import static org.apache.fluss.record.LogRecordBatch.CURRENT_LOG_MAGIC_VALUE; import static org.apache.fluss.record.LogRecordBatchFormat.recordBatchHeaderSize; import static org.apache.fluss.record.TestData.DATA1_PHYSICAL_TABLE_PATH; -import static org.apache.fluss.record.TestData.DATA1_PHYSICAL_TABLE_PATH_PK_PA_2024; import static org.apache.fluss.record.TestData.DATA1_ROW_TYPE; import static org.apache.fluss.record.TestData.DATA1_SCHEMA; -import static org.apache.fluss.record.TestData.DATA1_SCHEMA_PK; import static org.apache.fluss.record.TestData.DATA1_TABLE_ID; -import static org.apache.fluss.record.TestData.DATA1_TABLE_ID_PK; import static org.apache.fluss.record.TestData.DATA1_TABLE_INFO; -import static org.apache.fluss.record.TestData.DATA1_TABLE_INFO_PK; import static org.apache.fluss.record.TestData.DATA1_TABLE_PATH; -import static org.apache.fluss.record.TestData.DATA1_TABLE_PATH_PK; import static org.apache.fluss.record.TestData.DEFAULT_REMOTE_DATA_DIR; -import static org.apache.fluss.testutils.DataTestUtils.compactedRow; import static org.apache.fluss.testutils.DataTestUtils.indexedRow; import static org.apache.fluss.testutils.DataTestUtils.row; -import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -199,41 +191,6 @@ void testDrainBatches() throws Exception { verifyTableBucketInBatches(batches3, tb1, tb3); } - @Test - void testAppendAfterHistoricalTargetResolved() throws Exception { - long originalPartitionId = 11L; - long historicalPartitionId = 22L; - PhysicalTablePath originalPath = DATA1_PHYSICAL_TABLE_PATH_PK_PA_2024; - PhysicalTablePath anotherOriginalPath = PhysicalTablePath.of(DATA1_TABLE_PATH_PK, "2023"); - PhysicalTablePath historicalPath = - PhysicalTablePath.of(DATA1_TABLE_PATH_PK, HISTORICAL_PARTITION_VALUE); - TableBucket originalBucket = new TableBucket(DATA1_TABLE_ID_PK, originalPartitionId, 0); - TableBucket historicalBucket = new TableBucket(DATA1_TABLE_ID_PK, historicalPartitionId, 0); - cluster = - createPartitionedKvCluster( - originalPath, originalBucket, historicalPath, historicalBucket); - - RecordAccumulator accum = createTestRecordAccumulator(1024, 10L * 1024); - accum.tryRouteWritesTo(originalPath, historicalPath, historicalPartitionId); - accum.tryRouteWritesTo(anotherOriginalPath, historicalPath, historicalPartitionId); - accum.append(createKvRecord(originalPath), writeCallback, cluster, 0, false); - accum.append(createKvRecord(originalPath), writeCallback, cluster, 0, false); - accum.append(createKvRecord(anotherOriginalPath), writeCallback, cluster, 0, false); - - List drainedBatches = - accum.drain(cluster, Collections.singleton(node1.id()), Integer.MAX_VALUE) - .get(node1.id()); - - assertThat(drainedBatches).hasSize(2); - assertThat(drainedBatches) - .allSatisfy(batch -> assertThat(batch.tableBucket()).isEqualTo(historicalBucket)); - assertThat(drainedBatches) - .extracting(batch -> ((KvWriteBatch) batch.writeBatch()).getOriginalPartitionName()) - .containsExactlyInAnyOrder( - originalPath.getPartitionName(), anotherOriginalPath.getPartitionName()); - drainedBatches.forEach(batch -> accum.deallocate(batch.writeBatch())); - } - @Test void testDrainCompressedBatches() throws Exception { int batchSize = 10 * 1024; @@ -627,21 +584,6 @@ private WriteRecord createRecord(IndexedRow row, TableInfo tableInfo) { return WriteRecord.forIndexedAppend(tableInfo, DATA1_PHYSICAL_TABLE_PATH, row, null); } - private WriteRecord createKvRecord(PhysicalTablePath physicalTablePath) { - BinaryRow row = compactedRow(DATA1_ROW_TYPE, new Object[] {1, "a"}); - byte[] key = - new CompactedKeyEncoder(DATA1_ROW_TYPE, DATA1_SCHEMA_PK.getPrimaryKeyIndexes()) - .encodeKey(row); - return WriteRecord.forUpsert( - DATA1_TABLE_INFO_PK, - physicalTablePath, - row, - key, - key, - WriteFormat.COMPACTED_KV, - null); - } - private TableInfo withSchemaId(int schemaId) { return new TableInfo( DATA1_TABLE_INFO.getTablePath(), @@ -680,36 +622,6 @@ private Cluster updateCluster(List bucketLocations) { Collections.emptyMap()); } - private Cluster createPartitionedKvCluster( - PhysicalTablePath originalPath, - TableBucket originalBucket, - PhysicalTablePath historicalPath, - TableBucket historicalBucket) { - Map aliveTabletServersById = new HashMap<>(); - aliveTabletServersById.put(node1.id(), node1); - - Map> bucketsByPath = new HashMap<>(); - bucketsByPath.put( - originalPath, - Collections.singletonList( - new BucketLocation(originalPath, originalBucket, node1.id(), serverNodes))); - bucketsByPath.put( - historicalPath, - Collections.singletonList( - new BucketLocation( - historicalPath, historicalBucket, node1.id(), serverNodes))); - - Map partitionIdsByPath = new HashMap<>(); - partitionIdsByPath.put(originalPath, originalBucket.getPartitionId()); - partitionIdsByPath.put(historicalPath, historicalBucket.getPartitionId()); - return new Cluster( - aliveTabletServersById, - new ServerNode(0, "localhost", 89, ServerType.COORDINATOR), - bucketsByPath, - Collections.singletonMap(DATA1_TABLE_PATH_PK, DATA1_TABLE_ID_PK), - partitionIdsByPath); - } - private void delayedInterrupt(final Thread thread, final long delayMs) { Thread t = new Thread( diff --git a/fluss-client/src/test/java/org/apache/fluss/client/write/SenderTest.java b/fluss-client/src/test/java/org/apache/fluss/client/write/SenderTest.java index 566e0c9e6d6..925d92ab623 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/write/SenderTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/write/SenderTest.java @@ -43,7 +43,6 @@ import org.apache.fluss.row.BinaryRow; import org.apache.fluss.row.GenericRow; import org.apache.fluss.row.encode.CompactedKeyEncoder; -import org.apache.fluss.row.indexed.IndexedRow; import org.apache.fluss.rpc.entity.ProduceLogResultForBucket; import org.apache.fluss.rpc.entity.PutKvResultForBucket; import org.apache.fluss.rpc.messages.ApiMessage; @@ -93,7 +92,6 @@ import static org.apache.fluss.server.utils.ServerRpcMessageUtils.makePutKvResponse; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.toProduceLogDataForBuckets; import static org.apache.fluss.testutils.DataTestUtils.compactedRow; -import static org.apache.fluss.testutils.DataTestUtils.indexedRow; import static org.apache.fluss.testutils.DataTestUtils.row; import static org.apache.fluss.testutils.common.CommonTestUtils.retry; import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; @@ -129,44 +127,6 @@ public void teardown() throws Exception { sender.destroyResources(); } - @Test - void testSendsHistoricalPutWhenTargetResolvedBeforeAppend() throws Exception { - sender.destroyResources(); - String originalPartitionName = "20000101"; - TableInfo tableInfo = createHistoricalTableInfo(); - PhysicalTablePath originalPath = - PhysicalTablePath.of(tableInfo.getTablePath(), originalPartitionName); - PhysicalTablePath historicalPath = - PhysicalTablePath.of(tableInfo.getTablePath(), HISTORICAL_PARTITION_VALUE); - long historicalPartitionId = 22L; - TableBucket historicalBucket = - new TableBucket(tableInfo.getTableId(), historicalPartitionId, 0); - metadataUpdater = - new TestingMetadataUpdater( - Collections.singletonMap(tableInfo.getTablePath(), tableInfo)); - metadataUpdater.updateCluster( - partitionedCluster( - tableInfo, Collections.singletonMap(historicalPath, historicalBucket))); - sender = setupWithIdempotenceState(); - accumulator.tryRouteWritesTo(originalPath, historicalPath, historicalPartitionId); - - CompletableFuture future = - appendKvRecord(tableInfo, originalPath, 1, metadataUpdater.getCluster()); - - sender.runOnce(); - - assertThat(sender.numOfInFlightBatches(historicalBucket)).isOne(); - TestTabletServerGateway gateway = node1Gateway(); - PutKvRequest request = (PutKvRequest) gateway.getRequest(0); - assertThat(request.getBucketsReqAt(0).getPartitionId()).isEqualTo(historicalPartitionId); - assertThat(request.getBucketsReqAt(0).getOriginalPartitionName()) - .isEqualTo(originalPartitionName); - - gateway.response( - 0, createHistoricalPutKvResponse(historicalBucket, 1L, originalPartitionName)); - assertThat(future.get()).isNull(); - } - @Test void testPotentialExpirationUsesAutoPartitionTimeUnit() { TableInfo tableInfo = createHistoricalTableInfo(AutoPartitionTimeUnit.HOUR, 48); @@ -174,13 +134,13 @@ void testPotentialExpirationUsesAutoPartitionTimeUnit() { assertThat( WriterClient.mayBeExpiredHistoricalPartition( - PhysicalTablePath.of(tableInfo.getTablePath(), "2026082301"), + PhysicalTablePath.of(tableInfo.getTablePath(), "2026082123"), tableInfo, now)) .isTrue(); assertThat( WriterClient.mayBeExpiredHistoricalPartition( - PhysicalTablePath.of(tableInfo.getTablePath(), "2026082302"), + PhysicalTablePath.of(tableInfo.getTablePath(), "2026082200"), tableInfo, now)) .isFalse(); @@ -215,33 +175,7 @@ void testFailsWriteAfterMetadataConfirmsPartitionMissing() throws Exception { } @Test - void testMissingPartitionDoesNotAbortNormalWrites() throws Exception { - sender.destroyResources(); - TableInfo tableInfo = createNormalPartitionedTableInfo(); - PhysicalTablePath partitionPath = - PhysicalTablePath.of(tableInfo.getTablePath(), "20990101"); - TableBucket tableBucket = new TableBucket(tableInfo.getTableId(), 21L, 0); - metadataUpdater = missingPartitionMetadataUpdater(tableInfo); - metadataUpdater.updateCluster( - partitionedCluster( - tableInfo, Collections.singletonMap(partitionPath, tableBucket))); - sender = setupWithIdempotenceState(); - - CompletableFuture future = - appendKvRecord(tableInfo, partitionPath, 1, metadataUpdater.getCluster()); - sender.runOnce(); - - TestTabletServerGateway gateway = node1Gateway(); - gateway.response( - 0, createPutKvResponse(tableBucket, Errors.UNKNOWN_TABLE_OR_BUCKET_EXCEPTION)); - sender.runOnce(); - - assertThat(future).isNotDone(); - accumulator.abortAllBatches(new RuntimeException("Test cleanup.")); - } - - @Test - void testPackNormalAndHistoricalPutRequests() throws Exception { + void testNormalAndHistoricalPutRequests() throws Exception { sender.destroyResources(); TableInfo tableInfo = createHistoricalTableInfo(); PhysicalTablePath activePath = PhysicalTablePath.of(tableInfo.getTablePath(), "20990101"); @@ -265,9 +199,9 @@ void testPackNormalAndHistoricalPutRequests() throws Exception { CompletableFuture activeFuture = appendKvRecord(tableInfo, activePath, 1, metadataUpdater.getCluster()); - accumulator.tryRouteWritesTo( + accumulator.routeWritesTo( firstOriginalPath, historicalPath, historicalBucket.getPartitionId()); - accumulator.tryRouteWritesTo( + accumulator.routeWritesTo( secondOriginalPath, historicalPath, historicalBucket.getPartitionId()); CompletableFuture firstHistoricalFuture = appendKvRecord(tableInfo, firstOriginalPath, 2, metadataUpdater.getCluster()); @@ -312,80 +246,11 @@ void testPackNormalAndHistoricalPutRequests() throws Exception { historicalBucket, 1L, firstOriginalPath.getPartitionName())))); - assertThat(activeFuture).isDone(); - assertThat(firstHistoricalFuture).isDone(); - assertThat(secondHistoricalFuture).isDone(); assertThat(activeFuture.get()).isNull(); assertThat(firstHistoricalFuture.get()).isNull(); assertThat(secondHistoricalFuture.get()).isNull(); } - @Test - void testPackHistoricalProduceLogRequests() throws Exception { - sender.destroyResources(); - TableInfo tableInfo = createHistoricalLogTableInfo(); - PhysicalTablePath firstOriginalPath = - PhysicalTablePath.of(tableInfo.getTablePath(), "20000101"); - PhysicalTablePath secondOriginalPath = - PhysicalTablePath.of(tableInfo.getTablePath(), "20000102"); - PhysicalTablePath historicalPath = - PhysicalTablePath.of(tableInfo.getTablePath(), HISTORICAL_PARTITION_VALUE); - TableBucket historicalBucket = new TableBucket(tableInfo.getTableId(), 22L, 0); - - metadataUpdater = - new TestingMetadataUpdater( - Collections.singletonMap(tableInfo.getTablePath(), tableInfo)); - metadataUpdater.updateCluster( - partitionedCluster( - tableInfo, Collections.singletonMap(historicalPath, historicalBucket))); - sender = setupWithIdempotenceState(); - - accumulator.tryRouteWritesTo( - firstOriginalPath, historicalPath, historicalBucket.getPartitionId()); - accumulator.tryRouteWritesTo( - secondOriginalPath, historicalPath, historicalBucket.getPartitionId()); - CompletableFuture firstFuture = - appendLogRecord(tableInfo, firstOriginalPath, 1, metadataUpdater.getCluster()); - CompletableFuture secondFuture = - appendLogRecord(tableInfo, secondOriginalPath, 2, metadataUpdater.getCluster()); - - sender.runOnce(); - - TestTabletServerGateway gateway = node1Gateway(); - assertThat(gateway.pendingRequestSize()).isOne(); - ProduceLogRequest request = (ProduceLogRequest) gateway.getRequest(0); - assertThat(request.getBucketsReqsCount()).isEqualTo(2); - Set originalPartitionNames = new HashSet<>(); - for (int i = 0; i < request.getBucketsReqsCount(); i++) { - assertThat(request.getBucketsReqAt(i).getPartitionId()) - .isEqualTo(historicalBucket.getPartitionId()); - originalPartitionNames.add(request.getBucketsReqAt(i).getOriginalPartitionName()); - } - assertThat(originalPartitionNames) - .containsExactlyInAnyOrder( - firstOriginalPath.getPartitionName(), - secondOriginalPath.getPartitionName()); - - gateway.response( - 0, - makeProduceLogResponse( - Arrays.asList( - ProduceLogResultForBucket.historicalSuccess( - historicalBucket, - 1L, - 2L, - secondOriginalPath.getPartitionName()), - ProduceLogResultForBucket.historicalSuccess( - historicalBucket, - 0L, - 1L, - firstOriginalPath.getPartitionName())))); - assertThat(firstFuture).isDone(); - assertThat(secondFuture).isDone(); - assertThat(firstFuture.get()).isNull(); - assertThat(secondFuture.get()).isNull(); - } - @Test void testSimple() throws Exception { long offset = 0; @@ -1540,15 +1405,6 @@ private static TableInfo createHistoricalTableInfo() { private static TableInfo createHistoricalTableInfo( AutoPartitionTimeUnit timeUnit, int numToRetain) { - return createPartitionedKvTableInfo(timeUnit, numToRetain, true); - } - - private static TableInfo createNormalPartitionedTableInfo() { - return createPartitionedKvTableInfo(AutoPartitionTimeUnit.DAY, 7, false); - } - - private static TableInfo createPartitionedKvTableInfo( - AutoPartitionTimeUnit timeUnit, int numToRetain, boolean historicalPartitionEnabled) { Schema schema = Schema.newBuilder() .column("id", DataTypes.INT()) @@ -1566,9 +1422,7 @@ private static TableInfo createPartitionedKvTableInfo( .property(ConfigOptions.TABLE_AUTO_PARTITION_TIMEZONE, "UTC") .property(ConfigOptions.TABLE_DATALAKE_ENABLED, true) .property(ConfigOptions.TABLE_DATALAKE_FORMAT, DataLakeFormat.PAIMON) - .property( - ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_ENABLED, - historicalPartitionEnabled) + .property(ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_ENABLED, true) .build(); return TableInfo.of( DATA1_TABLE_PATH_PK, @@ -1595,26 +1449,6 @@ public boolean checkAndUpdatePartitionMetadata(PhysicalTablePath physicalTablePa }; } - private static TableInfo createHistoricalLogTableInfo() { - Schema schema = - Schema.newBuilder() - .column("id", DataTypes.INT()) - .column("dt", DataTypes.STRING()) - .build(); - TableDescriptor descriptor = - TableDescriptor.builder() - .schema(schema) - .partitionedBy("dt") - .distributedBy(1, "id") - .property(ConfigOptions.TABLE_AUTO_PARTITION_ENABLED, true) - .property(ConfigOptions.TABLE_DATALAKE_ENABLED, true) - .property(ConfigOptions.TABLE_DATALAKE_FORMAT, DataLakeFormat.PAIMON) - .property(ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_ENABLED, true) - .build(); - return TableInfo.of( - DATA1_TABLE_PATH, DATA1_TABLE_ID, 1, descriptor, DEFAULT_REMOTE_DATA_DIR, 1L, 1L); - } - private static Cluster partitionedCluster( TableInfo tableInfo, Map tableBucketsByPath) { int[] replicas = new int[] {TestingMetadataUpdater.NODE1.id()}; @@ -1670,23 +1504,6 @@ private CompletableFuture appendKvRecord( return future; } - private CompletableFuture appendLogRecord( - TableInfo tableInfo, PhysicalTablePath physicalTablePath, int id, Cluster cluster) - throws Exception { - IndexedRow row = - indexedRow( - tableInfo.getRowType(), - new Object[] {id, physicalTablePath.getPartitionName()}); - CompletableFuture future = new CompletableFuture<>(); - accumulator.append( - WriteRecord.forIndexedAppend(tableInfo, physicalTablePath, row, null), - (tableBucket, logEndOffset, error) -> future.complete(error), - cluster, - 0, - false); - return future; - } - private void appendToAccumulator(TableBucket tb, GenericRow row, WriteCallback writeCallback) throws Exception { appendToAccumulator(DATA1_TABLE_INFO, tb, row, writeCallback); @@ -1829,14 +1646,6 @@ private PutKvResponse createPutKvResponse(TableBucket tb, long endOffset) { Collections.singletonList(new PutKvResultForBucket(tb, endOffset))); } - private PutKvResponse createHistoricalPutKvResponse( - TableBucket tb, long endOffset, String originalPartitionName) { - return makePutKvResponse( - Collections.singletonList( - PutKvResultForBucket.historicalSuccess( - tb, endOffset, originalPartitionName))); - } - private PutKvResponse createPutKvResponse(TableBucket tb, long endOffset, float pressure) { return makePutKvResponse( Collections.singletonList(new PutKvResultForBucket(tb, endOffset, pressure))); diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java index b0f780e6ae8..b1fce39b115 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java @@ -444,7 +444,7 @@ public class ConfigOptions { .durationType() .defaultValue(Duration.ofMinutes(30)) .withDescription( - "The historical KV write idle time after which a fully tiered local overlay can be cleaned. " + "The idle time after which fully tiered historical KV write state in the local overlay can be cleaned. " + "Set to 0 to disable idle cleanup."); public static final ConfigOption SERVER_DATA_DISK_WRITE_LIMIT_RATIO = diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/split/TieringSplitGenerator.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/split/TieringSplitGenerator.java index fc2cbd9a03d..a6282cfeabd 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/split/TieringSplitGenerator.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/split/TieringSplitGenerator.java @@ -141,24 +141,31 @@ private List generatePartitionTableSplit( .boxed() .collect(Collectors.toList())); KvSnapshots latestKvSnapshots = null; - if (tableInfo.hasPrimaryKey() && !historicalPartition) { - // get the table partition latest kv snapshot info - try { + if (tableInfo.hasPrimaryKey()) { + if (historicalPartition) { + // Historical KV replicas use the lake snapshot as their durable base and tier + // only the retained WAL, so they have no local KV snapshots to tier. latestKvSnapshots = - flussAdmin - .getLatestKvSnapshots(tableInfo.getTablePath(), partitionName) - .get(); - } catch (Exception e) { - throw new FlinkRuntimeException( - String.format( - "Failed to get table snapshot for table %s and partition %s", - tableInfo.getTablePath(), partitionName), - ExceptionUtils.stripCompletionException(e)); + new KvSnapshots( + tableInfo.getTableId(), + partitionId, + Collections.emptyMap(), + Collections.emptyMap()); + } else { + try { + latestKvSnapshots = + flussAdmin + .getLatestKvSnapshots(tableInfo.getTablePath(), partitionName) + .get(); + } catch (Exception e) { + throw new FlinkRuntimeException( + String.format( + "Failed to get table snapshot for table %s and partition %s", + tableInfo.getTablePath(), partitionName), + ExceptionUtils.stripCompletionException(e)); + } } } - // Historical KV replicas do not create regular KV snapshots. Their lake snapshot is - // the durable base, so tier them from the retained WAL like log tables. - splits.addAll( generateTableSplit( tableInfo, @@ -166,8 +173,7 @@ private List generatePartitionTableSplit( partitionName, lakeSnapshotInfo, latestKvSnapshots, - latestBucketsOffset, - historicalPartition)); + latestBucketsOffset)); } return splits; } @@ -202,8 +208,7 @@ private List generateNonPartitionedTableSplit( null, lakeSnapshotInfo, latestKvSnapshots, - latestBucketsOffset, - false); + latestBucketsOffset); } private List generateTableSplit( @@ -212,11 +217,10 @@ private List generateTableSplit( @Nullable String partitionName, @Nullable LakeSnapshot lakeSnapshotInfo, @Nullable KvSnapshots latestKvSnapshots, - Map latestBucketsOffset, - boolean historicalPartition) { + Map latestBucketsOffset) { List splits = new ArrayList<>(); - if (tableInfo.hasPrimaryKey() && !historicalPartition) { + if (tableInfo.hasPrimaryKey()) { // it's primary key table checkState(latestKvSnapshots != null); for (int bucket = 0; bucket < tableInfo.getNumBuckets(); bucket++) { diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/RecordWriter.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/RecordWriter.java index 6e762a96141..25d43d8400f 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/RecordWriter.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/RecordWriter.java @@ -42,6 +42,7 @@ public abstract class RecordWriter implements AutoCloseable { protected final int bucket; protected final List partitionKeys; protected final boolean historicalPartition; + // Null for historical writers, which derive the original partition from each record. protected final @Nullable BinaryRow fixedPartition; protected final FlussRecordAsPaimonRow flussRecordAsPaimonRow; diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/mergetree/MergeTreeWriter.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/mergetree/MergeTreeWriter.java index 7d48c850e9a..4dbc8100cb5 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/mergetree/MergeTreeWriter.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/mergetree/MergeTreeWriter.java @@ -45,24 +45,6 @@ public class MergeTreeWriter extends RecordWriter { private final IOManager ioManager; - public MergeTreeWriter( - FileStoreTable fileStoreTable, - TableBucket tableBucket, - @Nullable String partition, - List partitionKeys, - RowType flussRowType, - boolean paimonIncludingSystemColumns) { - this( - fileStoreTable, - tableBucket, - partition, - partitionKeys, - flussRowType, - (String[]) null, - paimonIncludingSystemColumns, - false); - } - public MergeTreeWriter( FileStoreTable fileStoreTable, TableBucket tableBucket, diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/HistoricalPartitionITCase.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/HistoricalPartitionITCase.java index a135d520162..0da14e9f710 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/HistoricalPartitionITCase.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/HistoricalPartitionITCase.java @@ -118,48 +118,6 @@ void testWriteAndTierHistoricalKvToPaimon() throws Exception { } } - @Test - void testWriteAndTierHistoricalLogToPaimon() throws Exception { - TablePath tablePath = TablePath.of(DEFAULT_DB, "historical_log_write_tiering"); - Schema schema = partitionedLogSchema(); - long tableId = createTable(tablePath, partitionedLogDescriptor(schema)); - - try { - long historicalPartitionId = waitUntilHistoricalPartitionReady(tablePath, tableId); - - List expectedRows = - Arrays.asList( - row(1, EXPIRED_PARTITION_NAME, "Alice"), - row(2, SECOND_EXPIRED_PARTITION_NAME, "Bob")); - writeRows(tablePath, expectedRows, true); - assertThat(admin.listPartitionInfos(tablePath).get()) - .noneMatch( - partitionInfo -> - EXPIRED_PARTITION_NAME.equals(partitionInfo.getPartitionName()) - || SECOND_EXPIRED_PARTITION_NAME.equals( - partitionInfo.getPartitionName())); - - TableBucket historicalBucket = new TableBucket(tableId, historicalPartitionId, 0); - assertThat(getLeaderReplica(historicalBucket).getLocalLogEndOffset()).isEqualTo(2); - - JobClient jobClient = buildTieringJob(execEnv); - try { - assertReplicaStatus(historicalBucket, 2); - checkFlussOffsetsInSnapshot( - tablePath, Collections.singletonMap(historicalBucket, 2L)); - - assertThat(readPaimonRows(tablePath)) - .containsExactlyInAnyOrder( - "1|" + EXPIRED_PARTITION_NAME + "|Alice", - "2|" + SECOND_EXPIRED_PARTITION_NAME + "|Bob"); - } finally { - jobClient.cancel().get(); - } - } finally { - dropTable(tablePath); - } - } - @ParameterizedTest(name = "defaultBucketKey={0}") @ValueSource(booleans = {true, false}) void testLookupExpiredPartitionFromPaimon(boolean defaultBucketKey) throws Exception { @@ -387,14 +345,6 @@ private static Schema partitionedPkSchema(boolean defaultBucketKey) { .build(); } - private static Schema partitionedLogSchema() { - return Schema.newBuilder() - .column("id", DataTypes.INT()) - .column("dt", DataTypes.STRING()) - .column("name", DataTypes.STRING()) - .build(); - } - private static Schema evolvedPartitionedPkSchema(boolean defaultBucketKey) { if (defaultBucketKey) { return Schema.newBuilder() @@ -447,24 +397,6 @@ private static TableDescriptor partitionedPkDescriptor( return builder.build(); } - private static TableDescriptor partitionedLogDescriptor(Schema schema) { - return TableDescriptor.builder() - .schema(schema) - .distributedBy(1, "id") - .partitionedBy("dt") - .property(ConfigOptions.TABLE_AUTO_PARTITION_ENABLED, true) - .property(ConfigOptions.TABLE_AUTO_PARTITION_KEY, "dt") - .property(ConfigOptions.TABLE_AUTO_PARTITION_TIME_UNIT, AutoPartitionTimeUnit.DAY) - .property( - ConfigOptions.TABLE_AUTO_PARTITION_NUM_RETENTION, - EXPIRED_PARTITION_RETENTION) - .property(ConfigOptions.TABLE_AUTO_PARTITION_TIMEZONE, "UTC") - .property(ConfigOptions.TABLE_DATALAKE_ENABLED, true) - .property(ConfigOptions.TABLE_DATALAKE_FRESHNESS, Duration.ofMillis(500)) - .property(ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_ENABLED, true) - .build(); - } - private static InternalRow dataRow( boolean defaultBucketKey, int id, String subId, String name) { return dataRow(defaultBucketKey, id, subId, name, EXPIRED_PARTITION_NAME); diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringTest.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringTest.java index e689778cf12..b960caacdf5 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringTest.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringTest.java @@ -17,7 +17,6 @@ package org.apache.fluss.lake.paimon.tiering; -import org.apache.fluss.config.AutoPartitionTimeUnit; import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; import org.apache.fluss.lake.batch.ArrowRecordBatch; @@ -67,13 +66,11 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import org.junit.jupiter.params.provider.ValueSource; import javax.annotation.Nullable; import java.io.File; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -88,6 +85,7 @@ import static org.apache.fluss.metadata.TableDescriptor.BUCKET_COLUMN_NAME; import static org.apache.fluss.metadata.TableDescriptor.OFFSET_COLUMN_NAME; import static org.apache.fluss.metadata.TableDescriptor.TIMESTAMP_COLUMN_NAME; +import static org.apache.fluss.record.ChangeType.APPEND_ONLY; import static org.apache.fluss.record.ChangeType.DELETE; import static org.apache.fluss.record.ChangeType.INSERT; import static org.apache.fluss.record.ChangeType.UPDATE_AFTER; @@ -224,20 +222,15 @@ void testTieringWriteTable(boolean isPrimaryKeyTable, boolean isPartitioned) thr } } - @ParameterizedTest - @ValueSource(booleans = {false, true}) - void testHistoricalPartitionTiering(boolean isPrimaryKeyTable) throws Exception { - TablePath tablePath = - TablePath.of( - "paimon", "test_historical_" + (isPrimaryKeyTable ? "primary_key" : "log")); - TableInfo tableInfo = createHistoricalTable(tablePath, isPrimaryKeyTable); + @Test + void testHistoricalPrimaryKeyTiering() throws Exception { + TablePath tablePath = TablePath.of("paimon", "test_historical_primary_key"); + TableInfo tableInfo = createHistoricalTable(tablePath, true); long timestamp = 1_000L; List records = Arrays.asList( - historicalRecord( - 0L, timestamp, 1, "partition-1", "20240101", isPrimaryKeyTable), - historicalRecord( - 1L, timestamp, 1, "partition-2", "20240102", isPrimaryKeyTable)); + historicalRecord(0L, timestamp, 1, "20240101", INSERT), + historicalRecord(1L, timestamp, 1, "20240102", INSERT)); PaimonWriteResult writeResult; try (LakeWriter lakeWriter = @@ -248,16 +241,22 @@ void testHistoricalPartitionTiering(boolean isPrimaryKeyTable) throws Exception writeResult = lakeWriter.complete(); } - assertThat(writeResult.commitMessages()).hasSize(2); SimpleVersionedSerializer serializer = paimonLakeTieringFactory.getWriteResultSerializer(); - assertThat(serializer.getVersion()).isEqualTo(1); - byte[] serialized = serializer.serialize(writeResult); - writeResult = serializer.deserialize(serializer.getVersion(), serialized); - assertThat(writeResult.commitMessages()).hasSize(2); + writeResult = + serializer.deserialize( + serializer.getVersion(), serializer.serialize(writeResult)); + assertHistoricalPartitions(writeResult); - commitWriteResults(tablePath, tableInfo, Collections.singletonList(writeResult)); - verifyHistoricalRecords(tablePath, isPrimaryKeyTable, records); + try (LakeCommitter committer = + createLakeCommitter(tablePath, tableInfo, new Configuration())) { + committer.commit( + committer.toCommittable(Collections.singletonList(writeResult)), + Collections.emptyMap()); + } + assertThat(paimonCatalog.listPartitions(toPaimon(tablePath))) + .extracting(partition -> partition.spec().get("c3")) + .containsExactlyInAnyOrder("20240101", "20240102"); } @Test @@ -268,10 +267,8 @@ void testHistoricalArrowBatchTiering() throws Exception { long timestamp = 1_000L; List records = Arrays.asList( - historicalRecord( - baseOffset, timestamp, 1, "partition-1", "20240101", false), - historicalRecord( - baseOffset + 1, timestamp, 2, "partition-2", "20240102", false)); + historicalRecord(baseOffset, timestamp, 1, "20240101", APPEND_ONLY), + historicalRecord(baseOffset + 1, timestamp, 2, "20240102", APPEND_ONLY)); PaimonWriteResult writeResult; try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE); @@ -289,9 +286,7 @@ void testHistoricalArrowBatchTiering() throws Exception { writeResult = lakeWriter.complete(); } - assertThat(writeResult.commitMessages()).hasSize(2); - commitWriteResults(tablePath, tableInfo, Collections.singletonList(writeResult)); - verifyHistoricalRecords(tablePath, false, records); + assertHistoricalPartitions(writeResult); } @Test @@ -675,21 +670,10 @@ private void verifyLogTableRecordsThreePartition( actualRecords.close(); } - private void verifyHistoricalRecords( - TablePath tablePath, boolean isPrimaryKeyTable, List records) - throws Exception { - List partitions = Arrays.asList("20240101", "20240102"); - assertThat(paimonCatalog.listPartitions(toPaimon(tablePath))) - .extracting(partition -> partition.spec().get("c3")) - .containsExactlyInAnyOrderElementsOf(partitions); - for (int i = 0; i < partitions.size(); i++) { - String partition = partitions.get(i); - verifyTableRecords( - getPaimonRows(tablePath, partition, isPrimaryKeyTable, 0), - Collections.singletonList(records.get(i)), - 0, - partition); - } + private void assertHistoricalPartitions(PaimonWriteResult writeResult) { + assertThat(writeResult.commitMessages()) + .extracting(message -> message.partition().getString(0).toString()) + .containsExactlyInAnyOrder("20240101", "20240102"); } private void verifyTableRecords( @@ -837,18 +821,13 @@ private GenericRecord toRecord(long offset, GenericRow row, ChangeType changeTyp } private LogRecord historicalRecord( - long offset, - long timestamp, - int key, - String value, - String partition, - boolean isPrimaryKeyTable) { - GenericRow row = new GenericRow(3); - row.setField(0, key); - row.setField(1, BinaryString.fromString(value)); - row.setField(2, BinaryString.fromString(partition)); - return new GenericRecord( - offset, timestamp, isPrimaryKeyTable ? INSERT : ChangeType.APPEND_ONLY, row); + long offset, long timestamp, int key, String partition, ChangeType changeType) { + GenericRow row = + GenericRow.of( + key, + BinaryString.fromString("value"), + BinaryString.fromString(partition)); + return new GenericRecord(offset, timestamp, changeType, row); } private void writeArrowRows(VectorSchemaRoot root, List records) { @@ -859,9 +838,8 @@ private void writeArrowRows(VectorSchemaRoot root, List records) { for (int i = 0; i < records.size(); i++) { org.apache.fluss.row.InternalRow row = records.get(i).getRow(); keyVector.setSafe(i, row.getInt(0)); - valueVector.setSafe(i, row.getString(1).toString().getBytes(StandardCharsets.UTF_8)); - partitionVector.setSafe( - i, row.getString(2).toString().getBytes(StandardCharsets.UTF_8)); + valueVector.setSafe(i, row.getString(1).toBytes()); + partitionVector.setSafe(i, row.getString(2).toBytes()); } root.setRowCount(records.size()); } @@ -1063,29 +1041,10 @@ private TableInfo createHistoricalTable(TablePath tablePath, boolean isPrimaryKe .partitionedBy("c3") .distributedBy(1) .property(ConfigOptions.TABLE_DATALAKE_ENABLED, true) - .property(ConfigOptions.TABLE_AUTO_PARTITION_ENABLED, true) - .property(ConfigOptions.TABLE_AUTO_PARTITION_KEY, "c3") - .property( - ConfigOptions.TABLE_AUTO_PARTITION_TIME_UNIT, - AutoPartitionTimeUnit.DAY) .build(); return TableInfo.of(tablePath, 0, 1, descriptor, DEFAULT_REMOTE_DATA_DIR, 1L, 1L); } - private void commitWriteResults( - TablePath tablePath, TableInfo tableInfo, List writeResults) - throws Exception { - try (LakeCommitter committer = - createLakeCommitter(tablePath, tableInfo, new Configuration())) { - PaimonCommittable committable = committer.toCommittable(writeResults); - assertThat( - committer - .commit(committable, Collections.emptyMap()) - .getCommittedSnapshotId()) - .isOne(); - } - } - private void createMultiPartitionTable(TablePath tablePath) throws Exception { Schema.Builder builder = Schema.newBuilder() diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/ProduceLogResultForBucket.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/ProduceLogResultForBucket.java index 98812e97447..434e4c68f43 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/ProduceLogResultForBucket.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/ProduceLogResultForBucket.java @@ -31,6 +31,7 @@ @Internal public class ProduceLogResultForBucket extends WriteResultForBucket { private final long baseOffset; + // Identifies the original partition for a historical write; null for a normal write. private final @Nullable String originalPartitionName; public ProduceLogResultForBucket(TableBucket tableBucket, long baseOffset, long endOffset) { diff --git a/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/client/ServerConnectionTest.java b/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/client/ServerConnectionTest.java index 541c6f50ee2..3b4e0596f3e 100644 --- a/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/client/ServerConnectionTest.java +++ b/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/client/ServerConnectionTest.java @@ -70,7 +70,6 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; import static org.apache.fluss.metrics.MetricNames.CLIENT_BYTES_IN_RATE_AVG; import static org.apache.fluss.metrics.MetricNames.CLIENT_BYTES_IN_RATE_TOTAL; @@ -251,8 +250,7 @@ public ChannelFuture connect(String host, int port) { @Test void testRejectHistoricalWritesForOldServer() throws Exception { nettyServer.close(); - OldWriteGatewayService oldGatewayService = new OldWriteGatewayService(); - buildNettyServer(oldGatewayService); + buildNettyServer(new OldWriteGatewayService()); ServerConnection connection = new ServerConnection( @@ -264,7 +262,6 @@ void testRejectHistoricalWritesForOldServer() throws Exception { try { assertThat(connection.send(ApiKeys.PUT_KV, putKvRequest(null)).get()) .isInstanceOf(PutKvResponse.class); - assertThat(oldGatewayService.putKvRequests).hasValue(1); assertThatThrownBy( () -> @@ -275,11 +272,9 @@ void testRejectHistoricalWritesForOldServer() throws Exception { .isInstanceOf(UnsupportedVersionException.class) .hasMessageContaining("require PUT_KV version 3 or newer") .hasMessageContaining("negotiated version 2"); - assertThat(oldGatewayService.putKvRequests).hasValue(1); assertThat(connection.send(ApiKeys.PRODUCE_LOG, produceLogRequest(null)).get()) .isInstanceOf(ProduceLogResponse.class); - assertThat(oldGatewayService.produceLogRequests).hasValue(1); assertThatThrownBy( () -> @@ -292,7 +287,6 @@ void testRejectHistoricalWritesForOldServer() throws Exception { .isInstanceOf(UnsupportedVersionException.class) .hasMessageContaining("require PRODUCE_LOG version 1 or newer") .hasMessageContaining("negotiated version 0"); - assertThat(oldGatewayService.produceLogRequests).hasValue(1); } finally { connection.close().get(); } @@ -346,10 +340,6 @@ private void buildNettyServer(TestingGatewayService gatewayService) throws Excep } private static class OldWriteGatewayService extends TestingTabletGatewayService { - - private final AtomicInteger putKvRequests = new AtomicInteger(); - private final AtomicInteger produceLogRequests = new AtomicInteger(); - @Override public CompletableFuture apiVersions(ApiVersionsRequest request) { return super.apiVersions(request) @@ -368,13 +358,11 @@ public CompletableFuture apiVersions(ApiVersionsRequest req @Override public CompletableFuture putKv(PutKvRequest request) { - putKvRequests.incrementAndGet(); return CompletableFuture.completedFuture(new PutKvResponse()); } @Override public CompletableFuture produceLog(ProduceLogRequest request) { - produceLogRequests.incrementAndGet(); return CompletableFuture.completedFuture(new ProduceLogResponse()); } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/entity/ProduceLogDataForBucket.java b/fluss-server/src/main/java/org/apache/fluss/server/entity/ProduceLogDataForBucket.java index 50897fea1dc..96232246152 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/entity/ProduceLogDataForBucket.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/entity/ProduceLogDataForBucket.java @@ -26,6 +26,7 @@ public class ProduceLogDataForBucket { private final TableBucket tableBucket; private final MemoryLogRecords records; + // Identifies the original partition for a historical write; null for a normal write. private final @Nullable String originalPartitionName; public ProduceLogDataForBucket( diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java index 0f578468a23..c099d7a4acb 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java @@ -382,35 +382,28 @@ public long getLakeLogEndOffset() { return logTablet.getLakeLogEndOffset(); } - /** Returns whether the lake and local log end offsets match for historical KV cleanup. */ - public boolean isHistoricalKvCleanupReady() { - return inReadLock( - leaderIsrUpdateLock, - () -> { - long localLogEndOffset = logTablet.localLogEndOffset(); - return isHistoricalKvCleanupReady(localLogEndOffset); - }); - } - /** * Drops and recreates a fully tiered historical KV overlay if leadership and offsets still * match. * * @param expectedLeaderEpoch leader epoch captured when cleanup was scheduled * @param logEndOffset matching lake and local log end offset that triggered cleanup - * @param prepareLakeLookup marks the covering lake snapshot before the overlay is dropped + * @param beforeCleanup action to run before the overlay is dropped * @return whether the overlay was cleaned */ public boolean cleanupHistoricalKv( - int expectedLeaderEpoch, long logEndOffset, Runnable prepareLakeLookup) { - checkNotNull(prepareLakeLookup, "prepareLakeLookup must not be null"); + int expectedLeaderEpoch, long logEndOffset, Runnable beforeCleanup) { + checkNotNull(beforeCleanup, "beforeCleanup must not be null"); return inWriteLock( leaderIsrUpdateLock, () -> { long localLogEndOffset = logTablet.localLogEndOffset(); + // Keep the scheduled snapshot and offset paired: newer lake progress may make + // the current lake and local offsets match while this task still references an + // older snapshot. if (leaderEpoch != expectedLeaderEpoch || localLogEndOffset != logEndOffset - || !isHistoricalKvCleanupReady(localLogEndOffset)) { + || !isEligibleForHistoricalKvCleanup(localLogEndOffset)) { return false; } @@ -420,17 +413,14 @@ public boolean cleanupHistoricalKv( tableBucket, localLogEndOffset, logTablet.getLakeLogEndOffset()); - try { - // A lookup started after the rebuilt empty overlay is published must open - // a lake view that covers the state removed by this cleanup. - prepareLakeLookup.run(); - dropKv(); - createHistoricalKvAfterCleanup(); - return true; - } catch (RuntimeException e) { - fatalErrorHandler.onFatalError(e); - throw e; - } + // A lookup started after the rebuilt empty overlay is published must open a + // lake view that covers the state removed by this cleanup. + beforeCleanup.run(); + dropKv(); + // TODO: Retry rebuilding this historical bucket instead of waiting for + // failover or restart. + createKv(); + return true; }); } @@ -779,7 +769,9 @@ private void logTableConfigChanges(TableInfo oldTableInfo, TableInfo newTableInf } } - private boolean isHistoricalKvCleanupReady(long localLogEndOffset) { + private boolean isEligibleForHistoricalKvCleanup(long localLogEndOffset) { + // The overlay must have a known lake base, contain writes after that base, and have all + // those writes covered by lake before it can be discarded. return isLeader() && isHistoricalPartition() && isKvTable() @@ -804,18 +796,34 @@ private void createKv() { // init kv tablet and get the snapshot it uses to init if have any Optional snapshotUsed = Optional.empty(); + Exception lastError = null; for (int i = 1; i <= INIT_KV_TABLET_MAX_RETRY_TIMES; i++) { try { snapshotUsed = initKvTablet(); + lastError = null; break; } catch (Exception e) { + lastError = e; LOG.warn( - "Fail to init kv tablet for bucket {}, retrying for {} times", + "Failed to init kv tablet for bucket {} on attempt {}/{}.", tableBucket, i, + INIT_KV_TABLET_MAX_RETRY_TIMES, e); } } + if (lastError != null) { + try { + dropKv(); + } catch (Exception cleanupError) { + lastError.addSuppressed(cleanupError); + } + throw new KvStorageException( + String.format( + "Failed to create KV tablet for bucket %s after %s attempts.", + tableBucket, INIT_KV_TABLET_MAX_RETRY_TIMES), + lastError); + } // A historical KV tablet is a disposable overlay over the lake snapshot. It is recovered // by replaying WAL from the lake log end offset and does not create its own KV snapshots. if (!isHistoricalPartition()) { @@ -841,25 +849,6 @@ private void dropKv() { historicalKvBaseOffset = -1L; } - private void createHistoricalKvAfterCleanup() { - checkState(isHistoricalPartition(), "Only a historical KV overlay can be cleaned."); - try { - closeableRegistryForKv = new CloseableRegistry(); - closeableRegistry.registerCloseable(closeableRegistryForKv); - initKvTablet(); - } catch (Exception e) { - try { - dropKv(); - } catch (Exception cleanupError) { - e.addSuppressed(cleanupError); - } - throw new KvStorageException( - String.format( - "Failed to recreate historical KV overlay for bucket %s.", tableBucket), - e); - } - } - private void mayFlushKv(long newHighWatermark) { KvTablet kvTablet = this.kvTablet; if (kvTablet != null) { @@ -2637,4 +2626,5 @@ public SchemaGetter getSchemaGetter() { public PeriodicSnapshotManager getKvSnapshotManager() { return kvSnapshotManager; } + } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java index 67fc81a3350..663ff0b336e 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java @@ -127,6 +127,7 @@ import org.apache.fluss.utils.FileUtils; import org.apache.fluss.utils.FlussPaths; import org.apache.fluss.utils.clock.Clock; +import org.apache.fluss.utils.concurrent.FutureUtils; import org.apache.fluss.utils.concurrent.Scheduler; import org.slf4j.Logger; @@ -698,14 +699,11 @@ public void appendHistoricalRecordsToLog( Collection entriesPerBucket, @Nullable UserContext userContext, Consumer> responseCallback) { - if (entriesPerBucket.isEmpty()) { - responseCallback.accept(Collections.emptyList()); - return; - } - - List results = Collections.synchronizedList(new ArrayList<>()); - AtomicInteger remaining = new AtomicInteger(entriesPerBucket.size()); + List> resultFutures = + new ArrayList<>(entriesPerBucket.size()); for (ProduceLogDataForBucket bucketData : entriesPerBucket) { + CompletableFuture resultFuture = new CompletableFuture<>(); + resultFutures.add(resultFuture); String originalPartitionName = checkNotNull( bucketData.originalPartitionName(), @@ -728,12 +726,11 @@ public void appendHistoricalRecordsToLog( result.getBaseOffset(), result.getWriteLogEndOffset(), originalPartitionName); - results.add(historicalResult); - if (remaining.decrementAndGet() == 0) { - responseCallback.accept(new ArrayList<>(results)); - } + resultFuture.complete(historicalResult); }); } + FutureUtils.combineAll(resultFutures) + .thenAccept(results -> responseCallback.accept(new ArrayList<>(results))); } /** diff --git a/fluss-server/src/test/java/org/apache/fluss/server/DynamicConfigChangeTest.java b/fluss-server/src/test/java/org/apache/fluss/server/DynamicConfigChangeTest.java index adc6a68c63a..7f50bfc1203 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/DynamicConfigChangeTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/DynamicConfigChangeTest.java @@ -192,34 +192,10 @@ void testAlterLakehouseConfigs() throws Exception { } @Test - void testAlterHistoricalKvCleanupIdleTime() throws Exception { - DynamicConfigManager dynamicConfigManager = createManager(new Configuration()); - AtomicReference cleanupIdleTime = new AtomicReference<>(); - dynamicConfigManager.register( - new ServerReconfigurable() { - @Override - public void validate(Configuration newConfig) throws ConfigException {} - - @Override - public void reconfigure(Configuration newConfig) { - cleanupIdleTime.set( - newConfig.get( - ConfigOptions - .SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME)); - } - }); - dynamicConfigManager.startup(); - - alterConfig( - dynamicConfigManager, - ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME.key(), - "5min"); - - assertThat(cleanupIdleTime.get()).isEqualTo(Duration.ofMinutes(5)); - assertThat(zookeeperClient.fetchEntityConfig()) - .containsEntry( - ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME.key(), - "5min"); + void testAllowsHistoricalKvCleanupIdleTimeToChangeDynamically() { + assertThat(new DynamicServerConfig(new Configuration()).isAllowedConfig( + ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME.key())) + .isTrue(); } @Test diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionTaskExecutorTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionTaskExecutorTest.java index 4293fc3d0df..98574e7d9fd 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionTaskExecutorTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionTaskExecutorTest.java @@ -148,37 +148,6 @@ void testOrderedTaskContinuesAfterPreviousFailure() throws Exception { assertThat(taskExecutor.numInflightRequests()).isZero(); } - @Test - void testMaintenanceTaskKeepsOrderWithoutConsumingRequestPermit() throws Exception { - ManualExecutor executor = new ManualExecutor(); - HistoricalPartitionTaskExecutor taskExecutor = - new HistoricalPartitionTaskExecutor(configuration(1), executor); - List executionOrder = new ArrayList<>(); - - CompletableFuture write = - taskExecutor.submitOrdered( - "bucket", - () -> { - executionOrder.add("write"); - return "written"; - }, - () -> "throttled"); - CompletableFuture maintenance = - taskExecutor.submitOrderedMaintenance( - "bucket", () -> executionOrder.add("maintenance")); - - assertThat(taskExecutor.numInflightRequests()).isOne(); - assertThat(taskExecutor.submitOrdered("another-bucket", () -> "written", () -> "throttled")) - .isCompletedWithValue("throttled"); - - executor.runNext(); - executor.runNext(); - assertThat(write).isCompletedWithValue("written"); - assertThat(maintenance).isDone(); - assertThat(executionOrder).containsExactly("write", "maintenance"); - assertThat(taskExecutor.numInflightRequests()).isZero(); - } - @Test void testRejectNonPositiveRequestLimit() { assertThatThrownBy( diff --git a/fluss-server/src/test/java/org/apache/fluss/server/utils/HistoricalPartitionTableValidationTest.java b/fluss-server/src/test/java/org/apache/fluss/server/utils/HistoricalPartitionTableValidationTest.java index 7fa4a90a1d6..9b6aa6913ea 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/utils/HistoricalPartitionTableValidationTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/utils/HistoricalPartitionTableValidationTest.java @@ -26,7 +26,6 @@ import org.junit.jupiter.api.Test; -import static org.assertj.core.api.Assertions.assertThatCode; import static org.assertj.core.api.Assertions.assertThatThrownBy; class HistoricalPartitionTableValidationTest { @@ -82,30 +81,4 @@ void testReportsAllUnmetHistoricalPartitionRequirements() { + "(currently 'iceberg'); " + "the table must define exactly one partition key (found 0)."); } - - @Test - void testAllowsHistoricalPartitionForLogTable() { - TableDescriptor logTableDescriptor = - TableDescriptor.builder() - .schema( - Schema.newBuilder() - .column("id", DataTypes.INT()) - .column("dt", DataTypes.STRING()) - .build()) - .partitionedBy("dt") - .distributedBy(1, "id") - .property(ConfigOptions.TABLE_REPLICATION_FACTOR, 1) - .property(ConfigOptions.TABLE_AUTO_PARTITION_ENABLED, true) - .property(ConfigOptions.TABLE_AUTO_PARTITION_KEY, "dt") - .property(ConfigOptions.TABLE_DATALAKE_ENABLED, true) - .property(ConfigOptions.TABLE_DATALAKE_FORMAT, DataLakeFormat.PAIMON) - .property(ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_ENABLED, true) - .build(); - - assertThatCode( - () -> - TableDescriptorValidation.validateTableDescriptor( - logTableDescriptor, 100, DataLakeFormat.PAIMON)) - .doesNotThrowAnyException(); - } } From 2cd7160ed49a7b3d4a0296182ace9a33d3bb2cde Mon Sep 17 00:00:00 2001 From: luoyuxia Date: Thu, 27 Aug 2026 20:56:58 +0800 Subject: [PATCH 3/8] [server] Refine historical KV cleanup lifecycle Bind historical write state to the active KV overlay and defer idle cleanup until its deadline after lake progress catches up. Co-Authored-By: Codex AI-Model: gpt-5 AI-Contributed/Feature: 234/234 AI-Contributed/UT: 129/129 --- .../source/split/TieringSplitGenerator.java | 10 +- .../paimon/tiering/PaimonTieringTest.java | 7 +- .../apache/fluss/server/replica/Replica.java | 48 ++++- .../fluss/server/replica/ReplicaManager.java | 12 -- .../HistoricalPartitionManager.java | 164 ++++++++++-------- .../fluss/server/DynamicConfigChangeTest.java | 8 +- .../HistoricalPartitionManagerTest.java | 114 ++++++------ 7 files changed, 202 insertions(+), 161 deletions(-) diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/split/TieringSplitGenerator.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/split/TieringSplitGenerator.java index a6282cfeabd..5dd0bad2374 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/split/TieringSplitGenerator.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/split/TieringSplitGenerator.java @@ -155,7 +155,8 @@ private List generatePartitionTableSplit( try { latestKvSnapshots = flussAdmin - .getLatestKvSnapshots(tableInfo.getTablePath(), partitionName) + .getLatestKvSnapshots( + tableInfo.getTablePath(), partitionName) .get(); } catch (Exception e) { throw new FlinkRuntimeException( @@ -203,12 +204,7 @@ private List generateNonPartitionedTableSplit( } return generateTableSplit( - tableInfo, - null, - null, - lakeSnapshotInfo, - latestKvSnapshots, - latestBucketsOffset); + tableInfo, null, null, lakeSnapshotInfo, latestKvSnapshots, latestBucketsOffset); } private List generateTableSplit( diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringTest.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringTest.java index b960caacdf5..19c59600f92 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringTest.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringTest.java @@ -244,8 +244,7 @@ void testHistoricalPrimaryKeyTiering() throws Exception { SimpleVersionedSerializer serializer = paimonLakeTieringFactory.getWriteResultSerializer(); writeResult = - serializer.deserialize( - serializer.getVersion(), serializer.serialize(writeResult)); + serializer.deserialize(serializer.getVersion(), serializer.serialize(writeResult)); assertHistoricalPartitions(writeResult); try (LakeCommitter committer = @@ -824,9 +823,7 @@ private LogRecord historicalRecord( long offset, long timestamp, int key, String partition, ChangeType changeType) { GenericRow row = GenericRow.of( - key, - BinaryString.fromString("value"), - BinaryString.fromString(partition)); + key, BinaryString.fromString("value"), BinaryString.fromString(partition)); return new GenericRecord(offset, timestamp, changeType, row); } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java index c099d7a4acb..ef312728215 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java @@ -224,6 +224,9 @@ public final class Replica { private @Nullable PeriodicSnapshotManager kvSnapshotManager; // The lake log end offset used as the durable base of the current historical KV overlay. private volatile long historicalKvBaseOffset = -1L; + // Replaced together with the historical KV overlay so stale cleanup tasks can be fenced by + // identity without external replica lifecycle callbacks. + private volatile @Nullable HistoricalWriteState historicalWriteState; /** * Server-wide {@link ScannerManager}. Active sessions for this bucket are closed in {@link @@ -382,6 +385,11 @@ public long getLakeLogEndOffset() { return logTablet.getLakeLogEndOffset(); } + /** Returns the state owned by the active historical KV overlay, or null if none is active. */ + public @Nullable HistoricalWriteState getHistoricalWriteState() { + return historicalWriteState; + } + /** * Drops and recreates a fully tiered historical KV overlay if leadership and offsets still * match. @@ -824,9 +832,9 @@ private void createKv() { tableBucket, INIT_KV_TABLET_MAX_RETRY_TIMES), lastError); } - // A historical KV tablet is a disposable overlay over the lake snapshot. It is recovered - // by replaying WAL from the lake log end offset and does not create its own KV snapshots. - if (!isHistoricalPartition()) { + if (isHistoricalPartition()) { + historicalWriteState = new HistoricalWriteState(clock.milliseconds()); + } else { startPeriodicKvSnapshot(snapshotUsed.orElse(null)); } } @@ -846,6 +854,7 @@ private void dropKv() { kvManager.dropKv(tableBucket); kvTablet = null; } + historicalWriteState = null; historicalKvBaseOffset = -1L; } @@ -2627,4 +2636,37 @@ public PeriodicSnapshotManager getKvSnapshotManager() { return kvSnapshotManager; } + /** Write activity and maximum-size state owned by one historical KV overlay. */ + @ThreadSafe + public static final class HistoricalWriteState { + // Latched when the live SST size reaches the maximum. Replacing the overlay replaces this + // state, so transient RocksDB size changes cannot resume writes prematurely. + private final AtomicBoolean maxSizeReached = new AtomicBoolean(); + + private volatile long lastWriteMs; + + private HistoricalWriteState(long lastWriteMs) { + this.lastWriteMs = lastWriteMs; + } + + /** Returns whether historical writes are paused by the maximum-size limit. */ + public boolean maxSizeReached() { + return maxSizeReached.get(); + } + + /** Latches the maximum-size limit and returns whether this call changed the state. */ + public boolean markMaxSizeReached() { + return maxSizeReached.compareAndSet(false, true); + } + + /** Records the latest historical write activity time. */ + public void recordWrite(long timestampMs) { + lastWriteMs = timestampMs; + } + + /** Returns the latest historical write activity time. */ + public long lastWriteMs() { + return lastWriteMs; + } + } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java index 663ff0b336e..bea89b18ffb 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java @@ -1460,13 +1460,7 @@ private void makeLeaders( if (replica.isDataLakeEnabled()) { updateWithLakeTableSnapshot(replica); } - int previousLeaderEpoch = replica.getLeaderEpoch(); replica.makeLeader(data); - if (replica.isHistoricalPartition() - && replica.isKvTable() - && previousLeaderEpoch != replica.getLeaderEpoch()) { - historicalPartitionManager.onLeaderActivated(replica); - } // start the remote log tiering tasks for leaders remoteLogManager.startLogTiering(replica); @@ -1540,9 +1534,6 @@ private void makeFollowers( replicasBecomeFollower.add(replica); scannerManager.closeScannersForBucket(tb); } - if (replica.isHistoricalPartition()) { - historicalPartitionManager.onReplicaStopped(tb); - } // stop the remote log tiering tasks for followers remoteLogManager.stopLogTiering(replica); result.put(tb, new NotifyLeaderAndIsrResultForBucket(tb)); @@ -2345,9 +2336,6 @@ private StopReplicaResultForBucket stopReplica( HostedReplica replica = getReplica(tb); if (replica instanceof OnlineReplica) { Replica replicaToDelete = ((OnlineReplica) replica).getReplica(); - if (replicaToDelete.isHistoricalPartition()) { - historicalPartitionManager.onReplicaStopped(tb); - } if (deleteLocal) { if (allReplicas.remove(tb) != null) { serverMetricGroup.removeTableBucketMetricGroup( diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java index 62a66d61137..7bd230e3010 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java @@ -42,11 +42,11 @@ import org.apache.fluss.server.kv.historical.HistoricalValueLookup; import org.apache.fluss.server.log.LogAppendInfo; import org.apache.fluss.server.replica.Replica; +import org.apache.fluss.server.replica.Replica.HistoricalWriteState; import org.apache.fluss.server.storage.LocalDiskManager; import org.apache.fluss.utils.ByteArraySlice; import org.apache.fluss.utils.ByteArrayWrapper; import org.apache.fluss.utils.clock.Clock; -import org.apache.fluss.utils.clock.SystemClock; import org.apache.fluss.utils.concurrent.Scheduler; import org.slf4j.Logger; @@ -63,9 +63,6 @@ import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicBoolean; import static org.apache.fluss.utils.Preconditions.checkArgument; import static org.apache.fluss.utils.Preconditions.checkNotNull; @@ -80,12 +77,11 @@ public final class HistoricalPartitionManager implements AutoCloseable { private final HistoricalPartitionTaskExecutor taskExecutor; private final HistoricalLakeLookupManager lakeLookupManager; private final Clock clock; - private volatile long cleanupIdleTimeMs; + private final @Nullable Scheduler cleanupScheduler; private final long maxHistoricalKvSizeBytes; - // Per-physical-bucket state for coordinating historical write admission and overlay cleanup. - // The state is replaced when a new leader epoch is activated and removed when the local - // replica stops. - private final ConcurrentMap historicalWriteStates; + + private volatile long cleanupIdleTimeMs; + private volatile boolean closed; /** Creates a historical partition manager from the tablet server dependencies. */ public HistoricalPartitionManager( @@ -107,19 +103,8 @@ public HistoricalPartitionManager( dataDirVolumeBytes, scheduler), clock, - MAX_HISTORICAL_KV_SIZE_BYTES); - } - - @VisibleForTesting - HistoricalPartitionManager( - HistoricalPartitionTaskExecutor taskExecutor, - HistoricalLakeLookupManager lakeLookupManager) { - this( - new Configuration(), - taskExecutor, - lakeLookupManager, - SystemClock.getInstance(), - MAX_HISTORICAL_KV_SIZE_BYTES); + MAX_HISTORICAL_KV_SIZE_BYTES, + scheduler); } @VisibleForTesting @@ -128,7 +113,8 @@ public HistoricalPartitionManager( HistoricalPartitionTaskExecutor taskExecutor, HistoricalLakeLookupManager lakeLookupManager, Clock clock, - long maxHistoricalKvSizeBytes) { + long maxHistoricalKvSizeBytes, + @Nullable Scheduler cleanupScheduler) { Duration cleanupIdleTime = conf.get(ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME); checkArgument( @@ -141,9 +127,9 @@ public HistoricalPartitionManager( this.lakeLookupManager = checkNotNull(lakeLookupManager, "lakeLookupManager must not be null"); this.clock = checkNotNull(clock, "clock must not be null"); + this.cleanupScheduler = cleanupScheduler; this.cleanupIdleTimeMs = cleanupIdleTime.toMillis(); this.maxHistoricalKvSizeBytes = maxHistoricalKvSizeBytes; - this.historicalWriteStates = new ConcurrentHashMap<>(); } /** Starts the resources used by historical partition operations. */ @@ -151,17 +137,6 @@ public void startup(Scheduler scheduler) { lakeLookupManager.startup(scheduler); } - /** Starts tracking cleanup activity for a newly activated historical KV leader. */ - public void onLeaderActivated(Replica replica) { - historicalWriteStates.put( - replica.getTableBucket(), new HistoricalWriteState(clock.milliseconds())); - } - - /** Stops tracking cleanup activity for a replica that is no longer a local leader. */ - public void onReplicaStopped(TableBucket tableBucket) { - historicalWriteStates.remove(tableBucket); - } - /** Records new lake progress and schedules any cleanup that it makes eligible. */ public void onLakeProgress(Replica replica, long lakeSnapshotId, long lakeLogEndOffset) { if (!replica.isLeader() || !replica.isKvTable()) { @@ -172,11 +147,13 @@ public void onLakeProgress(Replica replica, long lakeSnapshotId, long lakeLogEnd if (lakeLogEndOffset != localLogEndOffset) { return; } - HistoricalWriteState state = historicalWriteStateFor(replica); - boolean maxSizeReached = state.maxSizeReached.get(); - // Lake progress is the cleanup trigger. The ordered cleanup task rechecks the latest write - // time when it actually runs, so a write accepted after this notification cancels an idle - // cleanup without being overtaken by it. + HistoricalWriteState state = replica.getHistoricalWriteState(); + if (state == null) { + return; + } + boolean maxSizeReached = state.maxSizeReached(); + // Lake progress establishes the cleanup candidate. Idle cleanup waits until the last write + // reaches its deadline, then enters the ordered queue behind already accepted writes. scheduleCleanup( replica, state, @@ -230,8 +207,11 @@ public CompletableFuture putKv( checkNotNull( putData.originalPartitionName(), "originalPartitionName must not be null"); - HistoricalWriteState state = historicalWriteStateFor(replica); - if (state.maxSizeReached.get()) { + HistoricalWriteState state = + checkNotNull( + replica.getHistoricalWriteState(), + "No active historical KV overlay for " + replica.getTableBucket()); + if (state.maxSizeReached()) { return CompletableFuture.completedFuture( maxSizeThrottledResult( putData, originalPartitionName, maxHistoricalKvSizeBytes)); @@ -243,7 +223,7 @@ public CompletableFuture putKv( maxSizeThrottledResult( putData, originalPartitionName, maxHistoricalKvSizeBytes)); } - state.lastHistoricalWriteMs = clock.milliseconds(); + state.recordWrite(clock.milliseconds()); return taskExecutor.submitOrdered( putData.tableBucket(), () -> { @@ -255,7 +235,6 @@ public CompletableFuture putKv( targetColumns, mergeMode, requiredAcks); - state.lastHistoricalWriteMs = clock.milliseconds(); return PutKvResultForBucket.historicalSuccess( putData.tableBucket(), appendInfo.lastOffset() + 1, @@ -405,7 +384,7 @@ LogAppendInfo processPut( } private void markMaxSizeReached(Replica replica, HistoricalWriteState state) { - if (state.maxSizeReached.compareAndSet(false, true)) { + if (state.markMaxSizeReached()) { LOG.warn( "Pausing historical writes for {} because its live SST size reached the " + "maximum size {} bytes.", @@ -414,12 +393,6 @@ private void markMaxSizeReached(Replica replica, HistoricalWriteState state) { } } - private HistoricalWriteState historicalWriteStateFor(Replica replica) { - return historicalWriteStates.computeIfAbsent( - replica.getTableBucket(), - ignored -> new HistoricalWriteState(clock.milliseconds())); - } - private void scheduleCleanup( Replica replica, HistoricalWriteState state, @@ -427,6 +400,19 @@ private void scheduleCleanup( long lakeSnapshotId, int expectedLeaderEpoch, long logEndOffset) { + if (closed + || replica.getHistoricalWriteState() != state + || expectedLeaderEpoch != replica.getLeaderEpoch() + || replica.getLocalLogEndOffset() != logEndOffset) { + return; + } + + if (!maxSizeReached + && deferIdleCleanupIfNeeded( + replica, state, lakeSnapshotId, expectedLeaderEpoch, logEndOffset)) { + return; + } + CompletableFuture cleanupFuture; cleanupFuture = taskExecutor.submitOrderedMaintenance( @@ -457,13 +443,13 @@ private void runCleanup( long lakeSnapshotId, int expectedLeaderEpoch, long logEndOffset) { - long now = clock.milliseconds(); - if (historicalWriteStates.get(replica.getTableBucket()) != state - || expectedLeaderEpoch != replica.getLeaderEpoch() - || (!maxSizeReached - && (cleanupIdleTimeMs <= 0L - || now < state.lastHistoricalWriteMs - || now - state.lastHistoricalWriteMs < cleanupIdleTimeMs))) { + if (replica.getHistoricalWriteState() != state + || expectedLeaderEpoch != replica.getLeaderEpoch()) { + return; + } + if (!maxSizeReached + && deferIdleCleanupIfNeeded( + replica, state, lakeSnapshotId, expectedLeaderEpoch, logEndOffset)) { return; } @@ -471,7 +457,6 @@ private void runCleanup( expectedLeaderEpoch, logEndOffset, () -> requireLakeSnapshot(replica.getTableBucket().getTableId(), lakeSnapshotId))) { - state.maxSizeReached.set(false); LOG.info( "Cleaned {} historical KV overlay for {}.", maxSizeReached ? "max-size-triggered" : "idle-triggered", @@ -479,6 +464,50 @@ private void runCleanup( } } + /** + * Returns whether idle cleanup must stop now. If the idle window has not elapsed, schedules the + * next check at its deadline. + */ + private boolean deferIdleCleanupIfNeeded( + Replica replica, + HistoricalWriteState state, + long lakeSnapshotId, + int expectedLeaderEpoch, + long logEndOffset) { + long idleTimeMs = cleanupIdleTimeMs; + if (idleTimeMs <= 0L) { + return true; + } + long delayMs = remainingIdleCleanupDelayMs(state, idleTimeMs); + if (delayMs <= 0L) { + return false; + } + checkNotNull(cleanupScheduler, "cleanupScheduler must not be null") + .scheduleOnce( + "historical-kv-idle-cleanup-" + replica.getTableBucket(), + () -> + scheduleCleanup( + replica, + state, + false, + lakeSnapshotId, + expectedLeaderEpoch, + logEndOffset), + delayMs); + return true; + } + + /** Returns the remaining delay before idle cleanup is eligible, or {@code 0} if it is due. */ + private long remainingIdleCleanupDelayMs(HistoricalWriteState state, long idleTimeMs) { + long now = clock.milliseconds(); + long lastWriteMs = state.lastWriteMs(); + if (now < lastWriteMs) { + // A backward clock jump restarts the idle window instead of cleaning prematurely. + return idleTimeMs; + } + return Math.max(0L, idleTimeMs - (now - lastWriteMs)); + } + private static PutKvResultForBucket requestLimitThrottledResult( PutKvDataForBucket putData, String originalPartitionName) { return PutKvResultForBucket.historicalFailure( @@ -514,7 +543,7 @@ private static PutKvResultForBucket maxSizeThrottledResult( @Override public void close() { - historicalWriteStates.clear(); + closed = true; taskExecutor.close(); lakeLookupManager.close(); } @@ -579,19 +608,4 @@ private LookupResultForBucket lookupInternal( tableBucket, originalPartitionName, ApiError.fromThrowable(e)); } } - - /** Per-bucket historical write activity and maximum-size state. */ - private static final class HistoricalWriteState { - // Latched when the live SST size reaches the maximum. It is cleared only after a cleanup - // covered by lake progress succeeds, so transient RocksDB size changes cannot resume - // writes prematurely. - private final AtomicBoolean maxSizeReached = new AtomicBoolean(); - - // Updated when a write is admitted and again when it completes successfully. - private volatile long lastHistoricalWriteMs; - - private HistoricalWriteState(long lastHistoricalWriteMs) { - this.lastHistoricalWriteMs = lastHistoricalWriteMs; - } - } } diff --git a/fluss-server/src/test/java/org/apache/fluss/server/DynamicConfigChangeTest.java b/fluss-server/src/test/java/org/apache/fluss/server/DynamicConfigChangeTest.java index 7f50bfc1203..6beccb61285 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/DynamicConfigChangeTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/DynamicConfigChangeTest.java @@ -193,8 +193,12 @@ void testAlterLakehouseConfigs() throws Exception { @Test void testAllowsHistoricalKvCleanupIdleTimeToChangeDynamically() { - assertThat(new DynamicServerConfig(new Configuration()).isAllowedConfig( - ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME.key())) + assertThat( + new DynamicServerConfig(new Configuration()) + .isAllowedConfig( + ConfigOptions + .SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME + .key())) .isTrue(); } diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java index eea8578ff2a..d5ca84c3173 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java @@ -97,6 +97,7 @@ import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -139,7 +140,7 @@ void testResolvesMultipleLakeMissesWithoutPrewriteRollback() throws Exception { TestingHistoricalLakeLookupManager lakeLookupManager = new TestingHistoricalLakeLookupManager(lookupConfiguration()); HistoricalPartitionManager historicalPartitionManager = - new HistoricalPartitionManager( + createNonCleanupManager( new HistoricalPartitionTaskExecutor(lookupConfiguration()), lakeLookupManager); @@ -207,7 +208,7 @@ void testWalFullRowUpsertDoesNotLookupLake() throws Exception { TestingHistoricalLakeLookupManager lakeLookupManager = new TestingHistoricalLakeLookupManager(lookupConfiguration()); HistoricalPartitionManager historicalPartitionManager = - new HistoricalPartitionManager( + createNonCleanupManager( new HistoricalPartitionTaskExecutor(lookupConfiguration()), lakeLookupManager); @@ -256,7 +257,7 @@ void testHistoricalInsertUpdateAndDelete() throws Exception { TestingHistoricalLakeLookupManager lakeLookupManager = new TestingHistoricalLakeLookupManager(lookupConfiguration()); HistoricalPartitionManager historicalPartitionManager = - new HistoricalPartitionManager( + createNonCleanupManager( new HistoricalPartitionTaskExecutor(lookupConfiguration()), lakeLookupManager); @@ -446,7 +447,7 @@ void testUpdateAndDeleteFromLakeFallback() throws Exception { TestingHistoricalLakeLookupManager lakeLookupManager = new TestingHistoricalLakeLookupManager(lookupConfiguration()); HistoricalPartitionManager historicalPartitionManager = - new HistoricalPartitionManager( + createNonCleanupManager( new HistoricalPartitionTaskExecutor(lookupConfiguration()), lakeLookupManager); @@ -547,7 +548,7 @@ void testLeaderChangeDuringLakeLookupFencesHistoricalWrite() throws Exception { TestingHistoricalLakeLookupManager lakeLookupManager = new TestingHistoricalLakeLookupManager(lookupConfiguration()); HistoricalPartitionManager historicalPartitionManager = - new HistoricalPartitionManager( + createNonCleanupManager( new HistoricalPartitionTaskExecutor(lookupConfiguration()), lakeLookupManager); CountDownLatch lakeLookupStarted = new CountDownLatch(1); @@ -604,7 +605,7 @@ void testRecoversHistoricalOverlayFromLakeCommitOffset() throws Exception { TestingHistoricalLakeLookupManager lakeLookupManager = new TestingHistoricalLakeLookupManager(lookupConfiguration()); HistoricalPartitionManager historicalPartitionManager = - new HistoricalPartitionManager( + createNonCleanupManager( new HistoricalPartitionTaskExecutor(lookupConfiguration()), lakeLookupManager); @@ -750,8 +751,7 @@ void testCleansFullyTieredHistoricalOverlayAfterWriteIdleTime() throws Exception TestingHistoricalLakeLookupManager lakeLookupManager = new TestingHistoricalLakeLookupManager(cleanupConf); HistoricalPartitionManager historicalPartitionManager = - createCleanupManager( - cleanupConf, executor, lakeLookupManager, replica, Long.MAX_VALUE); + createCleanupManager(cleanupConf, executor, lakeLookupManager, Long.MAX_VALUE); byte[] primaryKey = new CompactedKeyEncoder(HISTORICAL_KEY_TYPE).encodeKey(row(1, "us")); Object[] valueObjects = new Object[] {1, "us", ORIGINAL_PARTITION, "v1"}; @@ -774,60 +774,26 @@ void testCleansFullyTieredHistoricalOverlayAfterWriteIdleTime() throws Exception assertThat(putFuture.get(10, TimeUnit.SECONDS).failed()).isFalse(); flushAndWait(originalKvTablet, Long.MAX_VALUE); - manualClock.advanceTime(Duration.ofMinutes(1)); - // The idle policy cannot clean until lake progress covers the local WAL. - assertThat(executor.numQueuedRunnables()).isZero(); - assertThat(replica.getKvTablet()).isSameAs(originalKvTablet); - long tieredOffset = replica.getLocalLogEndOffset(); historicalPartitionManager.onLakeProgress(replica, 9L, tieredOffset - 1); assertThat(executor.numQueuedRunnables()).isZero(); replica.getLogTablet().updateLakeLogEndOffset(tieredOffset); historicalPartitionManager.onLakeProgress(replica, 10L, tieredOffset); - assertThat(executor.numQueuedRunnables()).isOne(); - assertThat(lakeLookupManager.requiredLakeSnapshotIds).isEmpty(); - - // A write admitted after idle cleanup was scheduled must either cancel that cleanup - // or run against the newly created overlay. It must never be lost during the reset. - CompletableFuture laterPut = - putHistoricalRecords( - historicalPartitionManager, - replica, - batch( - HISTORICAL_KEY_TYPE, - tableInfo.getRowType(), - Tuple2.of(new Object[] {1, "us"}, valueObjects))); - executor.triggerAll(); - assertThat(laterPut.get(10, TimeUnit.SECONDS).failed()).isFalse(); + // Lake normally catches up before the overlay becomes idle. Cleanup must wake at the + // write deadline even if no further lake progress notification arrives. + assertThat(executor.numQueuedRunnables()).isZero(); + assertThat(executor.getActiveNonPeriodicScheduledTask()).hasSize(1); assertThat(replica.getKvTablet()).isSameAs(originalKvTablet); - assertThat(lakeLookupManager.requiredLakeSnapshotIds).isEmpty(); - - flushAndWait(originalKvTablet, Long.MAX_VALUE); manualClock.advanceTime(Duration.ofMinutes(1)); - long latestTieredOffset = replica.getLocalLogEndOffset(); - replica.getLogTablet().updateLakeLogEndOffset(latestTieredOffset); - - Configuration longerIdleTimeConf = new Configuration(cleanupConf); - longerIdleTimeConf.set( - ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME, - Duration.ofMinutes(2)); - historicalPartitionManager.validate(longerIdleTimeConf); - historicalPartitionManager.reconfigure(longerIdleTimeConf); - historicalPartitionManager.onLakeProgress(replica, 11L, latestTieredOffset); + executor.triggerNonPeriodicScheduledTasks(); + assertThat(executor.numQueuedRunnables()).isOne(); assertThat(lakeLookupManager.requiredLakeSnapshotIds).isEmpty(); executor.triggerAll(); - assertThat(replica.getKvTablet()).isSameAs(originalKvTablet); - - Configuration shorterIdleTimeConf = new Configuration(cleanupConf); - historicalPartitionManager.validate(shorterIdleTimeConf); - historicalPartitionManager.reconfigure(shorterIdleTimeConf); - historicalPartitionManager.onLakeProgress(replica, 12L, latestTieredOffset); - executor.triggerAll(); KvTablet cleanedKvTablet = replica.getKvTablet(); assertThat(cleanedKvTablet).isNotNull().isNotSameAs(originalKvTablet); - assertThat(lakeLookupManager.requiredLakeSnapshotIds).containsExactly(12L); + assertThat(lakeLookupManager.requiredLakeSnapshotIds).containsExactly(10L); assertThat(cleanedKvTablet.getRocksDBKv().limitScan(10)).isEmpty(); assertThat(cleanedKvTablet.lookupHistoricalLocal(ORIGINAL_PARTITION, primaryKey)) .isEqualTo(KvStateLookupResult.notFound()); @@ -859,7 +825,8 @@ void testRejectsNegativeCleanupIdleTimeDuringReconfiguration() { new HistoricalPartitionTaskExecutor(cleanupConf, executor), new TestingHistoricalLakeLookupManager(cleanupConf), manualClock, - Long.MAX_VALUE); + Long.MAX_VALUE, + new TestingCleanupScheduler(executor)); Configuration invalidConf = new Configuration(cleanupConf); invalidConf.set( ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME, @@ -892,8 +859,7 @@ void testCleanupRequiresLakeAndLocalOffsetsToMatch() throws Exception { TestingHistoricalLakeLookupManager lakeLookupManager = new TestingHistoricalLakeLookupManager(cleanupConf); HistoricalPartitionManager historicalPartitionManager = - createCleanupManager( - cleanupConf, executor, lakeLookupManager, replica, Long.MAX_VALUE); + createCleanupManager(cleanupConf, executor, lakeLookupManager, Long.MAX_VALUE); KvRecordBatch records = batch( @@ -964,7 +930,6 @@ void testMaxSizeBlocksWritesUntilOverlayIsTieredAndCleaned() throws Exception { cleanupConf, executor, new TestingHistoricalLakeLookupManager(cleanupConf), - replica, 1L); KvRecordBatch firstBatch = @@ -1026,7 +991,7 @@ void testHistoricalLookupThrottledWhenPermitsExhausted() throws Exception { ManuallyTriggeredScheduledExecutorService executor = new ManuallyTriggeredScheduledExecutorService(); HistoricalPartitionManager historicalPartitionManager = - new HistoricalPartitionManager( + createNonCleanupManager( new HistoricalPartitionTaskExecutor(lookupConfiguration(), executor), new TestingHistoricalLakeLookupManager(lookupConfiguration())); @@ -1231,7 +1196,6 @@ private HistoricalPartitionManager createCleanupManager( Configuration configuration, ManuallyTriggeredScheduledExecutorService executor, TestingHistoricalLakeLookupManager lakeLookupManager, - Replica replica, long maxHistoricalKvSizeBytes) { HistoricalPartitionManager manager = new HistoricalPartitionManager( @@ -1239,11 +1203,47 @@ private HistoricalPartitionManager createCleanupManager( new HistoricalPartitionTaskExecutor(configuration, executor), lakeLookupManager, manualClock, - maxHistoricalKvSizeBytes); - manager.onLeaderActivated(replica); + maxHistoricalKvSizeBytes, + new TestingCleanupScheduler(executor)); return manager; } + private HistoricalPartitionManager createNonCleanupManager( + HistoricalPartitionTaskExecutor taskExecutor, + HistoricalLakeLookupManager lakeLookupManager) { + return new HistoricalPartitionManager( + new Configuration(), + taskExecutor, + lakeLookupManager, + manualClock, + Long.MAX_VALUE, + null); + } + + private static final class TestingCleanupScheduler + implements org.apache.fluss.utils.concurrent.Scheduler { + private final ManuallyTriggeredScheduledExecutorService executor; + + private TestingCleanupScheduler(ManuallyTriggeredScheduledExecutorService executor) { + this.executor = executor; + } + + @Override + public void startup() {} + + @Override + public void shutdown() {} + + @Override + public ScheduledFuture schedule( + String name, Runnable task, long delayMs, long periodMs) { + if (periodMs > 0L) { + return executor.scheduleAtFixedRate(task, delayMs, periodMs, TimeUnit.MILLISECONDS); + } + return executor.schedule(task, delayMs, TimeUnit.MILLISECONDS); + } + } + private static CompletableFuture putHistoricalRecords( HistoricalPartitionManager manager, Replica replica, KvRecordBatch records) { return manager.putKv( From 7dcf0431131e362cc742576d2505c0687450cc0d Mon Sep 17 00:00:00 2001 From: luoyuxia Date: Fri, 28 Aug 2026 09:43:15 +0800 Subject: [PATCH 4/8] [server] Refine historical KV cleanup state handling Return a retriable KV storage error while local historical KV state is being initialized or rebuilt. Clarify cleanup-state naming and terminology, and extend the Paimon integration test through post-recovery writes and restarted tiering. Co-Authored-By: Codex AI-Model: gpt-5 AI-Contributed/Feature: 244/244 AI-Contributed/UT: 239/239 --- .../fluss/client/write/RecordAccumulator.java | 11 +- .../apache/fluss/config/ConfigOptions.java | 4 +- .../lookup/HistoricalPartitionITCase.java | 111 +++++++++++++--- ...pendOnlyArrowBatchCaseSensitivityTest.java | 6 +- .../apache/fluss/server/replica/Replica.java | 106 ++++++++++----- .../fluss/server/replica/ReplicaManager.java | 6 +- .../HistoricalPartitionManager.java | 117 ++++++++++------- .../HistoricalPartitionManagerTest.java | 122 ++++++++++++------ 8 files changed, 334 insertions(+), 149 deletions(-) diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java b/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java index 84ee7331e19..19362d8a948 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java @@ -364,6 +364,9 @@ void routeWritesTo( return; } + // Pair with appendNewBatch(). For a historical route change, either a normal batch is + // registered as incomplete first and rejects the switch, or batch creation observes the + // historical target. synchronized (existing) { if (existing.targetPath.equals(targetPath)) { existing.partitionId = targetPartitionId; @@ -703,7 +706,13 @@ private RecordAppendResult appendNewBatch( writeBatches.get(physicalTablePath), "Write batches for %s must exist.", physicalTablePath); - synchronized (bucketAndWriteBatches) { + // Only historical-enabled tables need to coordinate with routeWritesTo(). Other tables + // reuse the deque monitor already held by the caller, avoiding cross-bucket serialization. + Object routeLock = + tableInfo.getTableConfig().isHistoricalPartitionEnabled() + ? bucketAndWriteBatches + : deque; + synchronized (routeLock) { RecordAppendResult appendResult = tryAppend(writeRecord, callback, deque); if (appendResult != null) { // Somebody else found us a batch, return the one we waited for! Hopefully this diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java index b1fce39b115..94ad63a9524 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java @@ -442,9 +442,9 @@ public class ConfigOptions { public static final ConfigOption SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME = key("server.historical-partition.kv-cleanup.idle-time") .durationType() - .defaultValue(Duration.ofMinutes(30)) + .defaultValue(Duration.ofHours(3)) .withDescription( - "The idle time after which fully tiered historical KV write state in the local overlay can be cleaned. " + "The idle time after all local historical KV writes are tiered before the local state can be cleaned. " + "Set to 0 to disable idle cleanup."); public static final ConfigOption SERVER_DATA_DISK_WRITE_LIMIT_RATIO = diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/HistoricalPartitionITCase.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/HistoricalPartitionITCase.java index 0da14e9f710..20b05d55f28 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/HistoricalPartitionITCase.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/HistoricalPartitionITCase.java @@ -86,33 +86,79 @@ void testWriteAndTierHistoricalKvToPaimon() throws Exception { long tableId = createTable( tablePath, - partitionedPkDescriptor(schema, true, EXPIRED_PARTITION_RETENTION)); + partitionedDescriptor(schema, true, EXPIRED_PARTITION_RETENTION)); try { long historicalPartitionId = waitUntilHistoricalPartitionReady(tablePath, tableId); - InternalRow expectedRow = dataRow(true, 1, "unused", "Alice"); + InternalRow tieredRow = dataRow(true, 1, "unused", "Alice"); assertThat(admin.listPartitionInfos(tablePath).get()) .noneMatch(p -> EXPIRED_PARTITION_NAME.equals(p.getPartitionName())); - writeRows(tablePath, Collections.singletonList(expectedRow), false); + writeRows(tablePath, Collections.singletonList(tieredRow), false); // Historical writes must not recreate the expired original partition. assertThat(admin.listPartitionInfos(tablePath).get()) .noneMatch(p -> EXPIRED_PARTITION_NAME.equals(p.getPartitionName())); TableBucket historicalBucket = new TableBucket(tableId, historicalPartitionId, 0); assertThat(getLeaderReplica(historicalBucket).getLocalLogEndOffset()).isEqualTo(1); - JobClient jobClient = buildTieringJob(execEnv); - try { - assertReplicaStatus(historicalBucket, 1); - checkFlussOffsetsInSnapshot( - tablePath, Collections.singletonMap(historicalBucket, 1L)); - assertThat(readPaimonRows(tablePath)) - .containsExactly("1|" + EXPIRED_PARTITION_NAME + "|Alice"); - } finally { - jobClient.cancel().get(); - } + tierAndVerifyPaimonRows( + tablePath, historicalBucket, 1L, "1|" + EXPIRED_PARTITION_NAME + "|Alice"); + + // Leave an untiered update after the Paimon snapshot. Restart recovery must apply this + // changelog over the tiered row. + InternalRow updatedRow = dataRow(true, 1, "unused", "Alice-updated"); + writeRows(tablePath, Collections.singletonList(updatedRow), false); + // FULL changelog emits UPDATE_BEFORE and UPDATE_AFTER for the update. + assertThat(getLeaderReplica(historicalBucket).getLocalLogEndOffset()).isEqualTo(3); + assertThat(getLeaderReplica(historicalBucket).getLakeLogEndOffset()).isEqualTo(1); + + restartLeaderAndVerifyLookup(tablePath, historicalBucket, schema, updatedRow); + + // Verify that the recovered local state can resolve the previous value for another + // update, and that a newly started tiering job can synchronize the resulting changelog. + InternalRow postRecoveryRow = dataRow(true, 1, "unused", "Alice-after-recovery"); + writeRows(tablePath, Collections.singletonList(postRecoveryRow), false); + assertThat(getLeaderReplica(historicalBucket).getLocalLogEndOffset()).isEqualTo(5); + assertThat(getLeaderReplica(historicalBucket).getLakeLogEndOffset()).isEqualTo(1); + tierAndVerifyPaimonRows( + tablePath, + historicalBucket, + 5L, + "1|" + EXPIRED_PARTITION_NAME + "|Alice-after-recovery"); + } finally { + dropTable(tablePath); + } + } + + @Test + void testWriteAndTierHistoricalLogToPaimon() throws Exception { + TablePath tablePath = TablePath.of(DEFAULT_DB, "historical_log_write_tiering"); + long tableId = + createTable( + tablePath, + partitionedDescriptor( + partitionedLogSchema(), true, EXPIRED_PARTITION_RETENTION)); - restartLeaderAndVerifyLookup(tablePath, historicalBucket, schema, expectedRow); + try { + long historicalPartitionId = waitUntilHistoricalPartitionReady(tablePath, tableId); + List rows = + Arrays.asList( + row(1, EXPIRED_PARTITION_NAME, "Alice"), + row(2, EXPIRED_PARTITION_NAME, "Bob")); + + writeRows(tablePath, rows, true); + // Historical writes must not recreate the expired original partition. + assertThat(admin.listPartitionInfos(tablePath).get()) + .noneMatch(p -> EXPIRED_PARTITION_NAME.equals(p.getPartitionName())); + + TableBucket historicalBucket = new TableBucket(tableId, historicalPartitionId, 0); + assertThat(getLeaderReplica(historicalBucket).getLocalLogEndOffset()).isEqualTo(2); + tierAndVerifyPaimonRows( + tablePath, + historicalBucket, + 2L, + "1|" + EXPIRED_PARTITION_NAME + "|Alice", + "2|" + EXPIRED_PARTITION_NAME + "|Bob"); } finally { dropTable(tablePath); } @@ -128,7 +174,7 @@ void testLookupExpiredPartitionFromPaimon(boolean defaultBucketKey) throws Excep ? "historical_lookup_default_bucket" : "historical_lookup_bucket_subset"); Schema oldSchema = partitionedPkSchema(defaultBucketKey); - long tableId = createTable(tablePath, partitionedPkDescriptor(oldSchema, false)); + long tableId = createTable(tablePath, partitionedDescriptor(oldSchema, false)); // Enable historical lookup through ALTER TABLE to cover dynamic creation of the // coordinator-owned historical system partition. @@ -298,6 +344,23 @@ private List readPaimonRows(TablePath tablePath) throws Exception { return actualRows; } + private void tierAndVerifyPaimonRows( + TablePath tablePath, + TableBucket historicalBucket, + long expectedLogEndOffset, + String... expectedRows) + throws Exception { + JobClient jobClient = buildTieringJob(execEnv); + try { + assertReplicaStatus(historicalBucket, expectedLogEndOffset); + checkFlussOffsetsInSnapshot( + tablePath, Collections.singletonMap(historicalBucket, expectedLogEndOffset)); + assertThat(readPaimonRows(tablePath)).containsExactlyInAnyOrder(expectedRows); + } finally { + jobClient.cancel().get(); + } + } + private void restartLeaderAndVerifyLookup( TablePath tablePath, TableBucket historicalBucket, @@ -345,6 +408,14 @@ private static Schema partitionedPkSchema(boolean defaultBucketKey) { .build(); } + private static Schema partitionedLogSchema() { + return Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("dt", DataTypes.STRING()) + .column("name", DataTypes.STRING()) + .build(); + } + private static Schema evolvedPartitionedPkSchema(boolean defaultBucketKey) { if (defaultBucketKey) { return Schema.newBuilder() @@ -365,19 +436,19 @@ private static Schema evolvedPartitionedPkSchema(boolean defaultBucketKey) { .build(); } - private static TableDescriptor partitionedPkDescriptor( + private static TableDescriptor partitionedDescriptor( Schema schema, boolean historicalPartitionEnabled) { - return partitionedPkDescriptor( + return partitionedDescriptor( schema, historicalPartitionEnabled, INITIAL_PARTITION_RETENTION); } - private static TableDescriptor partitionedPkDescriptor( + private static TableDescriptor partitionedDescriptor( Schema schema, boolean historicalPartitionEnabled, int partitionRetention) { TableDescriptor.Builder builder = TableDescriptor.builder() .schema(schema) - // This is the default bucket key for (id, dt), and a strict subset of the - // physical primary key for (id, sub_id, dt). + // For primary-key tables, id is the default bucket key for (id, dt) and a + // strict subset of the physical primary key for (id, sub_id, dt). .distributedBy(1, "id") .partitionedBy("dt") .property(ConfigOptions.TABLE_AUTO_PARTITION_ENABLED, true) diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyArrowBatchCaseSensitivityTest.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyArrowBatchCaseSensitivityTest.java index e3d961d6605..31b664e1e55 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyArrowBatchCaseSensitivityTest.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyArrowBatchCaseSensitivityTest.java @@ -28,6 +28,7 @@ import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.paimon.FileStore; +import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.io.BundleRecords; import org.apache.paimon.table.BucketMode; @@ -48,7 +49,6 @@ import static org.apache.fluss.metadata.TableDescriptor.TIMESTAMP_COLUMN_NAME; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -150,10 +150,10 @@ private Object[] writeAndRead( fileStoreTable, tableWrite, paimonRowType, 0, legacyTable); ArrowBatchData batch = new ArrowBatchData(root.slice(0, root.getRowCount()), 0L, 1L, 1)) { - helper.writeArrowBatch(batch, null); + helper.writeArrowBatch(batch, BinaryRow.EMPTY_ROW, false); ArgumentCaptor captor = ArgumentCaptor.forClass(BundleRecords.class); - verify(tableWrite).writeBundle(isNull(), eq(0), captor.capture()); + verify(tableWrite).writeBundle(eq(BinaryRow.EMPTY_ROW), eq(0), captor.capture()); Iterator rows = captor.getValue().iterator(); assertThat(rows.hasNext()).isTrue(); InternalRow row = rows.next(); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java index ef312728215..34f4cba9246 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java @@ -222,11 +222,11 @@ public final class Replica { private volatile @Nullable KvTablet kvTablet; private volatile @Nullable CloseableRegistry closeableRegistryForKv; private @Nullable PeriodicSnapshotManager kvSnapshotManager; - // The lake log end offset used as the durable base of the current historical KV overlay. + // The lake log end offset from which the current local historical KV state was rebuilt. private volatile long historicalKvBaseOffset = -1L; - // Replaced together with the historical KV overlay so stale cleanup tasks can be fenced by - // identity without external replica lifecycle callbacks. - private volatile @Nullable HistoricalWriteState historicalWriteState; + // Replaced whenever the local historical KV state is rebuilt so delayed cleanup tasks for an + // earlier state can be ignored. + private volatile @Nullable HistoricalKvCleanupState historicalKvCleanupState; /** * Server-wide {@link ScannerManager}. Active sessions for this bucket are closed in {@link @@ -332,8 +332,8 @@ public long logicalStorageLogSize() { public long logicalStorageKvSize() { if (isLeader() && isKvTable()) { if (isHistoricalPartition()) { - // Historical KV tablets do not create snapshots, so account for the local overlay - // using live SST files instead. + // Historical KV tablets do not create snapshots, so use live SST files to account + // for their local state instead. KvTablet currentKvTablet = kvTablet; return currentKvTablet == null ? 0L : currentKvTablet.liveSstFilesSize(); } @@ -385,22 +385,22 @@ public long getLakeLogEndOffset() { return logTablet.getLakeLogEndOffset(); } - /** Returns the state owned by the active historical KV overlay, or null if none is active. */ - public @Nullable HistoricalWriteState getHistoricalWriteState() { - return historicalWriteState; + /** Returns the cleanup state for the local historical KV state, or null if it is not ready. */ + public @Nullable HistoricalKvCleanupState getHistoricalKvCleanupState() { + return historicalKvCleanupState; } /** - * Drops and recreates a fully tiered historical KV overlay if leadership and offsets still - * match. + * Drops and recreates local historical KV state after its writes are fully tiered, provided + * leadership and offsets still match. * - * @param expectedLeaderEpoch leader epoch captured when cleanup was scheduled - * @param logEndOffset matching lake and local log end offset that triggered cleanup - * @param beforeCleanup action to run before the overlay is dropped - * @return whether the overlay was cleaned + * @param expectedLeaderEpoch leader epoch required when cleanup runs + * @param tieredLogEndOffset fully tiered log end offset required when cleanup runs + * @param beforeCleanup action to run before the local KV state is dropped + * @return whether the local KV state was cleaned */ public boolean cleanupHistoricalKv( - int expectedLeaderEpoch, long logEndOffset, Runnable beforeCleanup) { + int expectedLeaderEpoch, long tieredLogEndOffset, Runnable beforeCleanup) { checkNotNull(beforeCleanup, "beforeCleanup must not be null"); return inWriteLock( leaderIsrUpdateLock, @@ -410,19 +410,19 @@ public boolean cleanupHistoricalKv( // the current lake and local offsets match while this task still references an // older snapshot. if (leaderEpoch != expectedLeaderEpoch - || localLogEndOffset != logEndOffset + || localLogEndOffset != tieredLogEndOffset || !isEligibleForHistoricalKvCleanup(localLogEndOffset)) { return false; } LOG.info( - "Cleaning historical KV overlay for {} at local log end offset {} " + "Cleaning local historical KV state for {} at log end offset {} " + "covered by lake log end offset {}.", tableBucket, localLogEndOffset, logTablet.getLakeLogEndOffset()); - // A lookup started after the rebuilt empty overlay is published must open a - // lake view that covers the state removed by this cleanup. + // A lookup started after the empty local KV state is rebuilt must open a lake + // view that covers the data removed by this cleanup. beforeCleanup.run(); dropKv(); // TODO: Retry rebuilding this historical bucket instead of waiting for @@ -778,7 +778,7 @@ private void logTableConfigChanges(TableInfo oldTableInfo, TableInfo newTableInf } private boolean isEligibleForHistoricalKvCleanup(long localLogEndOffset) { - // The overlay must have a known lake base, contain writes after that base, and have all + // Local KV state must have a known lake base, contain writes after that base, and have all // those writes covered by lake before it can be discarded. return isLeader() && isHistoricalPartition() @@ -833,7 +833,7 @@ private void createKv() { lastError); } if (isHistoricalPartition()) { - historicalWriteState = new HistoricalWriteState(clock.milliseconds()); + historicalKvCleanupState = new HistoricalKvCleanupState(clock.milliseconds()); } else { startPeriodicKvSnapshot(snapshotUsed.orElse(null)); } @@ -854,7 +854,7 @@ private void dropKv() { kvManager.dropKv(tableBucket); kvTablet = null; } - historicalWriteState = null; + historicalKvCleanupState = null; historicalKvBaseOffset = -1L; } @@ -912,8 +912,8 @@ private Optional initKvTablet() { // get the offset from which, we should restore from. default is 0 long restoreStartOffset = isHistoricalPartition() ? historicalRecoveryStartOffset() : 0; - // The lake snapshot is the durable base for a historical overlay. Historical replicas - // therefore never restore a normal KV snapshot, even if one exists from older code. + // Lake is the durable base for local historical KV state. Historical replicas therefore + // never restore a normal KV snapshot, even if one exists from older code. Optional optCompletedSnapshot = isHistoricalPartition() ? Optional.empty() : getLatestSnapshot(tableBucket); try { @@ -1363,7 +1363,7 @@ public List findKeysRequiringLakeLookup( }); } - /** Writes records to the local historical KV overlay of the leader replica. */ + /** Writes records to the local historical KV state of the leader replica. */ public LogAppendInfo putHistoricalRecordsToLeader( KvRecordBatch kvRecords, @Nullable int[] targetColumns, @@ -1412,7 +1412,7 @@ private void validateHistoricalWrite(int expectedLeaderEpoch, int requiredAcks) validateInSyncReplicaSize(requiredAcks); } - /** Looks up keys from the local historical KV overlay of the leader replica. */ + /** Looks up keys from the local historical KV state of the leader replica. */ public List lookupHistoricalLocal( String originalPartitionName, List keys) throws Exception { return inReadLock( @@ -2636,16 +2636,17 @@ public PeriodicSnapshotManager getKvSnapshotManager() { return kvSnapshotManager; } - /** Write activity and maximum-size state owned by one historical KV overlay. */ + /** Tracks write activity and cleanup conditions for the current local historical KV state. */ @ThreadSafe - public static final class HistoricalWriteState { - // Latched when the live SST size reaches the maximum. Replacing the overlay replaces this - // state, so transient RocksDB size changes cannot resume writes prematurely. + public static final class HistoricalKvCleanupState { + // Latched when the live SST size reaches the maximum. Rebuilding the local KV state resets + // this flag, while transient RocksDB size changes do not resume writes prematurely. private final AtomicBoolean maxSizeReached = new AtomicBoolean(); + private volatile @Nullable CleanupCandidate cleanupCandidate; private volatile long lastWriteMs; - private HistoricalWriteState(long lastWriteMs) { + private HistoricalKvCleanupState(long lastWriteMs) { this.lastWriteMs = lastWriteMs; } @@ -2659,6 +2660,18 @@ public boolean markMaxSizeReached() { return maxSizeReached.compareAndSet(false, true); } + /** Updates the cleanup candidate for the current local historical KV state. */ + public void updateCleanupCandidate( + long lakeSnapshotId, int expectedLeaderEpoch, long tieredLogEndOffset) { + cleanupCandidate = + new CleanupCandidate(lakeSnapshotId, expectedLeaderEpoch, tieredLogEndOffset); + } + + /** Returns the cleanup candidate, or null if none is available. */ + public @Nullable CleanupCandidate cleanupCandidate() { + return cleanupCandidate; + } + /** Records the latest historical write activity time. */ public void recordWrite(long timestampMs) { lastWriteMs = timestampMs; @@ -2668,5 +2681,34 @@ public void recordWrite(long timestampMs) { public long lastWriteMs() { return lastWriteMs; } + + /** A candidate for cleaning up local historical KV state. */ + public static final class CleanupCandidate { + private final long lakeSnapshotId; + private final int expectedLeaderEpoch; + private final long tieredLogEndOffset; + + private CleanupCandidate( + long lakeSnapshotId, int expectedLeaderEpoch, long tieredLogEndOffset) { + this.lakeSnapshotId = lakeSnapshotId; + this.expectedLeaderEpoch = expectedLeaderEpoch; + this.tieredLogEndOffset = tieredLogEndOffset; + } + + /** Returns the lake snapshot ID that covers the local KV state. */ + public long lakeSnapshotId() { + return lakeSnapshotId; + } + + /** Returns the leader epoch required to run cleanup. */ + public int expectedLeaderEpoch() { + return expectedLeaderEpoch; + } + + /** Returns the log end offset covered by the lake snapshot. */ + public long tieredLogEndOffset() { + return tieredLogEndOffset; + } + } } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java index bea89b18ffb..37b47d3d4ed 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java @@ -1487,9 +1487,9 @@ private void updateWithLakeTableSnapshot(Replica replica) throws Exception { .getLogEndOffset(tb) .ifPresent(replica.getLogTablet()::updateLakeLogEndOffset); if (replica.isHistoricalPartition()) { - // The historical overlay will be rebuilt from this snapshot's lake offset. + // Local historical KV state will be rebuilt from this snapshot's lake offset. // Refresh a cached lookuper before it becomes the fallback for data omitted - // from the rebuilt overlay. + // from the rebuilt local state. historicalPartitionManager.requireLakeSnapshot( replica.getTableBucket().getTableId(), snapshotId); } @@ -1498,7 +1498,7 @@ private void updateWithLakeTableSnapshot(Replica replica) throws Exception { if (replica.isHistoricalPartition()) { // Historical recovery uses the lake offset as its durable base and replays the // retained WAL from that offset. Reject leader activation if the latest lake - // progress cannot be loaded, instead of rebuilding the overlay from stale state. + // progress cannot be loaded, instead of rebuilding local KV from stale state. throw e; } // Lake commit cleanup can race with this best-effort refresh and remove the diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java index 7bd230e3010..93d720ae6fe 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java @@ -24,6 +24,7 @@ import org.apache.fluss.exception.ConfigException; import org.apache.fluss.exception.HistoricalPartitionThrottledException; import org.apache.fluss.exception.InvalidPartitionException; +import org.apache.fluss.exception.KvStorageException; import org.apache.fluss.lake.lakestorage.LakeTableLookuper; import org.apache.fluss.metadata.ResolvedPartitionSpec; import org.apache.fluss.metadata.TableBucket; @@ -42,7 +43,7 @@ import org.apache.fluss.server.kv.historical.HistoricalValueLookup; import org.apache.fluss.server.log.LogAppendInfo; import org.apache.fluss.server.replica.Replica; -import org.apache.fluss.server.replica.Replica.HistoricalWriteState; +import org.apache.fluss.server.replica.Replica.HistoricalKvCleanupState; import org.apache.fluss.server.storage.LocalDiskManager; import org.apache.fluss.utils.ByteArraySlice; import org.apache.fluss.utils.ByteArrayWrapper; @@ -147,23 +148,15 @@ public void onLakeProgress(Replica replica, long lakeSnapshotId, long lakeLogEnd if (lakeLogEndOffset != localLogEndOffset) { return; } - HistoricalWriteState state = replica.getHistoricalWriteState(); - if (state == null) { + HistoricalKvCleanupState cleanupState = replica.getHistoricalKvCleanupState(); + if (cleanupState == null) { return; } - boolean maxSizeReached = state.maxSizeReached(); - // Lake progress establishes the cleanup candidate. Idle cleanup waits until the last write - // reaches its deadline, then enters the ordered queue behind already accepted writes. - scheduleCleanup( - replica, - state, - maxSizeReached, - lakeSnapshotId, - expectedLeaderEpoch, - lakeLogEndOffset); + cleanupState.updateCleanupCandidate(lakeSnapshotId, expectedLeaderEpoch, lakeLogEndOffset); + tryScheduleCleanup(replica, cleanupState); } - /** Looks up historical keys from the local overlay and then lake storage. */ + /** Looks up historical keys from local KV state and then lake storage. */ public CompletableFuture lookup( Replica replica, LookupDataForBucket lookupData, @@ -195,7 +188,7 @@ public CompletableFuture lookup( } } - /** Writes records to the local overlay of a historical partition. */ + /** Writes records to the local KV state of a historical partition. */ public CompletableFuture putKv( Replica replica, PutKvDataForBucket putData, @@ -207,23 +200,26 @@ public CompletableFuture putKv( checkNotNull( putData.originalPartitionName(), "originalPartitionName must not be null"); - HistoricalWriteState state = - checkNotNull( - replica.getHistoricalWriteState(), - "No active historical KV overlay for " + replica.getTableBucket()); - if (state.maxSizeReached()) { + HistoricalKvCleanupState cleanupState = replica.getHistoricalKvCleanupState(); + if (cleanupState == null) { + throw new KvStorageException( + "Local historical KV state is not ready for " + + replica.getTableBucket() + + " because its KV tablet is being initialized or rebuilt."); + } + if (cleanupState.maxSizeReached()) { return CompletableFuture.completedFuture( maxSizeThrottledResult( putData, originalPartitionName, maxHistoricalKvSizeBytes)); } long liveSstSize = replica.logicalStorageKvSize(); if (liveSstSize >= maxHistoricalKvSizeBytes) { - markMaxSizeReached(replica, state); + markMaxSizeReached(replica, cleanupState); return CompletableFuture.completedFuture( maxSizeThrottledResult( putData, originalPartitionName, maxHistoricalKvSizeBytes)); } - state.recordWrite(clock.milliseconds()); + cleanupState.recordWrite(clock.milliseconds()); return taskExecutor.submitOrdered( putData.tableBucket(), () -> { @@ -383,33 +379,55 @@ LogAppendInfo processPut( requiredAcks); } - private void markMaxSizeReached(Replica replica, HistoricalWriteState state) { - if (state.markMaxSizeReached()) { + private void markMaxSizeReached(Replica replica, HistoricalKvCleanupState cleanupState) { + if (cleanupState.markMaxSizeReached()) { LOG.warn( "Pausing historical writes for {} because its live SST size reached the " + "maximum size {} bytes.", replica.getTableBucket(), maxHistoricalKvSizeBytes); + + tryScheduleCleanup(replica, cleanupState); + } + } + + private void tryScheduleCleanup(Replica replica, HistoricalKvCleanupState cleanupState) { + HistoricalKvCleanupState.CleanupCandidate candidate = cleanupState.cleanupCandidate(); + if (candidate == null) { + return; } + scheduleCleanup( + replica, + cleanupState, + cleanupState.maxSizeReached(), + candidate.lakeSnapshotId(), + candidate.expectedLeaderEpoch(), + candidate.tieredLogEndOffset()); } private void scheduleCleanup( Replica replica, - HistoricalWriteState state, + HistoricalKvCleanupState cleanupState, boolean maxSizeReached, long lakeSnapshotId, int expectedLeaderEpoch, - long logEndOffset) { + long tieredLogEndOffset) { + // Reject cleanup if the replica no longer matches the cleanup state, leader epoch, or + // tiered log end offset used when it was scheduled. if (closed - || replica.getHistoricalWriteState() != state + || replica.getHistoricalKvCleanupState() != cleanupState || expectedLeaderEpoch != replica.getLeaderEpoch() - || replica.getLocalLogEndOffset() != logEndOffset) { + || replica.getLocalLogEndOffset() != tieredLogEndOffset) { return; } if (!maxSizeReached && deferIdleCleanupIfNeeded( - replica, state, lakeSnapshotId, expectedLeaderEpoch, logEndOffset)) { + replica, + cleanupState, + lakeSnapshotId, + expectedLeaderEpoch, + tieredLogEndOffset)) { return; } @@ -420,11 +438,11 @@ && deferIdleCleanupIfNeeded( () -> runCleanup( replica, - state, + cleanupState, maxSizeReached, lakeSnapshotId, expectedLeaderEpoch, - logEndOffset)); + tieredLogEndOffset)); cleanupFuture.whenComplete( (ignored, error) -> { if (error != null) { @@ -438,27 +456,31 @@ && deferIdleCleanupIfNeeded( private void runCleanup( Replica replica, - HistoricalWriteState state, + HistoricalKvCleanupState cleanupState, boolean maxSizeReached, long lakeSnapshotId, int expectedLeaderEpoch, - long logEndOffset) { - if (replica.getHistoricalWriteState() != state + long tieredLogEndOffset) { + if (replica.getHistoricalKvCleanupState() != cleanupState || expectedLeaderEpoch != replica.getLeaderEpoch()) { return; } if (!maxSizeReached && deferIdleCleanupIfNeeded( - replica, state, lakeSnapshotId, expectedLeaderEpoch, logEndOffset)) { + replica, + cleanupState, + lakeSnapshotId, + expectedLeaderEpoch, + tieredLogEndOffset)) { return; } if (replica.cleanupHistoricalKv( expectedLeaderEpoch, - logEndOffset, + tieredLogEndOffset, () -> requireLakeSnapshot(replica.getTableBucket().getTableId(), lakeSnapshotId))) { LOG.info( - "Cleaned {} historical KV overlay for {}.", + "Cleaned {} local historical KV state for {}.", maxSizeReached ? "max-size-triggered" : "idle-triggered", replica.getTableBucket()); } @@ -470,15 +492,15 @@ && deferIdleCleanupIfNeeded( */ private boolean deferIdleCleanupIfNeeded( Replica replica, - HistoricalWriteState state, + HistoricalKvCleanupState cleanupState, long lakeSnapshotId, int expectedLeaderEpoch, - long logEndOffset) { + long tieredLogEndOffset) { long idleTimeMs = cleanupIdleTimeMs; if (idleTimeMs <= 0L) { return true; } - long delayMs = remainingIdleCleanupDelayMs(state, idleTimeMs); + long delayMs = remainingIdleCleanupDelayMs(cleanupState, idleTimeMs); if (delayMs <= 0L) { return false; } @@ -488,19 +510,20 @@ private boolean deferIdleCleanupIfNeeded( () -> scheduleCleanup( replica, - state, + cleanupState, false, lakeSnapshotId, expectedLeaderEpoch, - logEndOffset), + tieredLogEndOffset), delayMs); return true; } /** Returns the remaining delay before idle cleanup is eligible, or {@code 0} if it is due. */ - private long remainingIdleCleanupDelayMs(HistoricalWriteState state, long idleTimeMs) { + private long remainingIdleCleanupDelayMs( + HistoricalKvCleanupState cleanupState, long idleTimeMs) { long now = clock.milliseconds(); - long lastWriteMs = state.lastWriteMs(); + long lastWriteMs = cleanupState.lastWriteMs(); if (now < lastWriteMs) { // A backward clock jump restarts the idle window instead of cleaning prematurely. return idleTimeMs; @@ -532,12 +555,12 @@ private static PutKvResultForBucket maxSizeThrottledResult( + putData.tableBucket() + " (original partition " + originalPartitionName - + ") because its historical KV overlay reached the live " + + ") because its local historical KV state reached the live " + "SST maximum size of " + maxHistoricalKvSize + " bytes. New writes are paused until lake tiering " - + "covers all previously accepted writes and the local " - + "overlay cleanup completes.")), + + "covers all previously accepted writes and cleanup of " + + "the local historical KV state completes.")), originalPartitionName); } diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java index d5ca84c3173..526623b050e 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java @@ -22,7 +22,6 @@ import org.apache.fluss.config.AutoPartitionTimeUnit; import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; -import org.apache.fluss.exception.ConfigException; import org.apache.fluss.exception.HistoricalPartitionThrottledException; import org.apache.fluss.exception.InvalidPartitionException; import org.apache.fluss.lake.lakestorage.LakeTableLookuper; @@ -269,7 +268,7 @@ void testHistoricalInsertUpdateAndDelete() throws Exception { byte[] primaryKey = new CompactedKeyEncoder(keyType).encodeKey(row(1, "us")); try { - // The first write misses both local state and lake, so it creates a local overlay. + // The first write misses both local KV state and lake, so it creates a local value. KvRecordBatch insertBatch = batch( keyType, @@ -328,7 +327,7 @@ void testHistoricalInsertUpdateAndDelete() throws Exception { row(1, "us", "20240108", "another")); assertThat(lakeLookupManager.lookupCount).hasValue(2); - // Exercise the ReplicaManager entry point; the update should reuse the local overlay. + // Exercise the ReplicaManager entry point; the update should reuse local KV state. KvRecordBatch updateBatch = batch( keyType, @@ -372,7 +371,7 @@ void testHistoricalInsertUpdateAndDelete() throws Exception { row(1, "us", "20240108", "another")); assertThat(lakeLookupManager.lookupCount).hasValue(2); - // Historical lookup should observe the updated value from the local overlay. + // Historical lookup should observe the updated value from local KV state. CompletableFuture> lookupResponse = new CompletableFuture<>(); replicaManager.historicalLookups( @@ -599,7 +598,7 @@ void testLeaderChangeDuringLakeLookupFencesHistoricalWrite() throws Exception { } @Test - void testRecoversHistoricalOverlayFromLakeCommitOffset() throws Exception { + void testRecoversLocalHistoricalKvFromLakeCommitOffset() throws Exception { TableInfo tableInfo = registerHistoricalTableAndBecomeLeader(); Replica replica = replicaManager.getReplicaOrException(TABLE_BUCKET); TestingHistoricalLakeLookupManager lakeLookupManager = @@ -686,7 +685,7 @@ void testRecoversHistoricalOverlayFromLakeCommitOffset() throws Exception { // Persist the exclusive end offset of the first write as the lake recovery point. The // replica has not received this offset locally, so becoming leader must load it before - // creating the historical overlay. + // creating the local historical KV state. long lakeCommitOffset = firstAppend.lastOffset() + 1; new LakeTableHelper(zkClient, DEFAULT_REMOTE_DATA_DIR) .registerLakeTableSnapshotV1( @@ -695,8 +694,8 @@ void testRecoversHistoricalOverlayFromLakeCommitOffset() throws Exception { 1L, Collections.singletonMap(TABLE_BUCKET, lakeCommitOffset))); assertThat(replica.getLakeLogEndOffset()).isEqualTo(-1L); - // Dropping and recreating the leader KV tablet forces the overlay to be rebuilt only - // from WAL after the lake commit offset. The recovered tombstone must remain + // Dropping and recreating the leader KV tablet rebuilds its state only from WAL after + // the lake commit offset. The recovered tombstone must remain // authoritative over lake fallback. assertThat(replica.makeFollower(followerState())).isTrue(); CompletableFuture> leaderFuture = @@ -736,7 +735,7 @@ void testRecoversHistoricalOverlayFromLakeCommitOffset() throws Exception { } @Test - void testCleansFullyTieredHistoricalOverlayAfterWriteIdleTime() throws Exception { + void testCleansLocalHistoricalKvAfterTieringAndWriteIdleTime() throws Exception { TableInfo tableInfo = registerHistoricalTableAndBecomeLeader(); Replica replica = replicaManager.getReplicaOrException(TABLE_BUCKET); KvTablet originalKvTablet = replica.getKvTablet(); @@ -780,8 +779,8 @@ void testCleansFullyTieredHistoricalOverlayAfterWriteIdleTime() throws Exception replica.getLogTablet().updateLakeLogEndOffset(tieredOffset); historicalPartitionManager.onLakeProgress(replica, 10L, tieredOffset); - // Lake normally catches up before the overlay becomes idle. Cleanup must wake at the - // write deadline even if no further lake progress notification arrives. + // Lake normally catches up before local KV state becomes idle. Cleanup must wake at + // the write deadline even if no further lake progress notification arrives. assertThat(executor.numQueuedRunnables()).isZero(); assertThat(executor.getActiveNonPeriodicScheduledTask()).hasSize(1); assertThat(replica.getKvTablet()).isSameAs(originalKvTablet); @@ -808,41 +807,13 @@ void testCleansFullyTieredHistoricalOverlayAfterWriteIdleTime() throws Exception (lookupTimeNanos, lookupFileDownloaded) -> {}); executor.triggerAll(); assertThat(lookupFuture.get(10, TimeUnit.SECONDS).lookupValues()) + .extracting(ByteArraySlice::toByteArray) .containsExactly(lakeValue); } finally { historicalPartitionManager.close(); } } - @Test - void testRejectsNegativeCleanupIdleTimeDuringReconfiguration() { - Configuration cleanupConf = lookupConfiguration(); - ManuallyTriggeredScheduledExecutorService executor = - new ManuallyTriggeredScheduledExecutorService(); - HistoricalPartitionManager historicalPartitionManager = - new HistoricalPartitionManager( - cleanupConf, - new HistoricalPartitionTaskExecutor(cleanupConf, executor), - new TestingHistoricalLakeLookupManager(cleanupConf), - manualClock, - Long.MAX_VALUE, - new TestingCleanupScheduler(executor)); - Configuration invalidConf = new Configuration(cleanupConf); - invalidConf.set( - ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME, - Duration.ofMillis(-1)); - - try { - assertThatThrownBy(() -> historicalPartitionManager.validate(invalidConf)) - .isInstanceOf(ConfigException.class) - .hasMessageContaining( - ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME.key(), - "must not be negative"); - } finally { - historicalPartitionManager.close(); - } - } - @Test void testCleanupRequiresLakeAndLocalOffsetsToMatch() throws Exception { TableInfo tableInfo = registerHistoricalTableAndBecomeLeader(); @@ -914,7 +885,7 @@ void testCleanupRequiresLakeAndLocalOffsetsToMatch() throws Exception { } @Test - void testMaxSizeBlocksWritesUntilOverlayIsTieredAndCleaned() throws Exception { + void testMaxSizeBlocksWritesUntilAcceptedWritesAreTieredAndLocalKvIsCleaned() throws Exception { TableInfo tableInfo = registerHistoricalTableAndBecomeLeader(); Replica replica = replicaManager.getReplicaOrException(TABLE_BUCKET); KvTablet originalKvTablet = replica.getKvTablet(); @@ -984,6 +955,75 @@ void testMaxSizeBlocksWritesUntilOverlayIsTieredAndCleaned() throws Exception { } } + @Test + void testMaxSizeCleanupWhenLakeCaughtUpBeforeLimitIsObserved() throws Exception { + TableInfo tableInfo = registerHistoricalTableAndBecomeLeader(); + Replica replica = replicaManager.getReplicaOrException(TABLE_BUCKET); + KvTablet originalKvTablet = replica.getKvTablet(); + assertThat(originalKvTablet).isNotNull(); + + Configuration cleanupConf = lookupConfiguration(); + cleanupConf.set( + ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME, Duration.ZERO); + ManuallyTriggeredScheduledExecutorService executor = + new ManuallyTriggeredScheduledExecutorService(); + TestingHistoricalLakeLookupManager lakeLookupManager = + new TestingHistoricalLakeLookupManager(cleanupConf); + HistoricalPartitionManager historicalPartitionManager = + createCleanupManager(cleanupConf, executor, lakeLookupManager, 1L); + + KvRecordBatch firstBatch = + batch( + HISTORICAL_KEY_TYPE, + tableInfo.getRowType(), + Tuple2.of( + new Object[] {1, "us"}, + new Object[] {1, "us", ORIGINAL_PARTITION, "v1"})); + KvRecordBatch secondBatch = + batch( + HISTORICAL_KEY_TYPE, + tableInfo.getRowType(), + Tuple2.of( + new Object[] {2, "eu"}, + new Object[] {2, "eu", ORIGINAL_PARTITION, "v2"})); + + try { + CompletableFuture firstPut = + putHistoricalRecords(historicalPartitionManager, replica, firstBatch); + executor.triggerAll(); + assertThat(firstPut.get(10, TimeUnit.SECONDS).failed()).isFalse(); + flushAndWait(originalKvTablet, Long.MAX_VALUE); + try (FlushOptions flushOptions = new FlushOptions().setWaitForFlush(true)) { + originalKvTablet.getRocksDBKv().getDb().flush(flushOptions); + } + assertThat(originalKvTablet.liveSstFilesSize()).isPositive(); + + // Lake catches up before another request observes that the SST size reached the limit. + long tieredOffset = replica.getLocalLogEndOffset(); + replica.getLogTablet().updateLakeLogEndOffset(tieredOffset); + historicalPartitionManager.onLakeProgress(replica, 12L, tieredOffset); + assertThat(executor.numQueuedRunnables()).isZero(); + + PutKvResultForBucket blockedWrite = + putHistoricalRecords(historicalPartitionManager, replica, secondBatch) + .get(10, TimeUnit.SECONDS); + assertThat(blockedWrite.getError().error()) + .isEqualTo(Errors.HISTORICAL_PARTITION_THROTTLED); + assertThat(executor.numQueuedRunnables()).isOne(); + + executor.triggerAll(); + assertThat(replica.getKvTablet()).isNotNull().isNotSameAs(originalKvTablet); + assertThat(lakeLookupManager.requiredLakeSnapshotIds).containsExactly(12L); + + CompletableFuture resumedWrite = + putHistoricalRecords(historicalPartitionManager, replica, secondBatch); + executor.triggerAll(); + assertThat(resumedWrite.get(10, TimeUnit.SECONDS).failed()).isFalse(); + } finally { + historicalPartitionManager.close(); + } + } + @Test void testHistoricalLookupThrottledWhenPermitsExhausted() throws Exception { registerHistoricalTableAndBecomeLeader(); From 58af499c8d88caec78c1f7bbb5f704c53e9ba81f Mon Sep 17 00:00:00 2001 From: luoyuxia Date: Sun, 30 Aug 2026 19:41:32 +0800 Subject: [PATCH 5/8] [lake/paimon][server] Refresh historical lookup files in place Refresh registered Paimon partition-bucket file sets when the required lake snapshot changes, allowing unchanged local lookup files to remain reusable. Co-Authored-By: Codex AI-Model: gpt-5 AI-Contributed/Feature: 161/215 AI-Contributed/UT: 95/125 --- .../lake/lakestorage/LakeTableLookuper.java | 11 ++ .../lakestorage/PluginLakeStorageWrapper.java | 7 + .../lookup/PaimonLakeTableLookuper.java | 161 +++++++++++++----- .../lookup/PaimonLakeTableLookuperTest.java | 95 +++++++++++ .../HistoricalLakeLookupManager.java | 34 ++-- .../HistoricalPartitionManager.java | 2 +- .../HistoricalLakeLookupManagerTest.java | 30 ++-- 7 files changed, 276 insertions(+), 64 deletions(-) diff --git a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeTableLookuper.java b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeTableLookuper.java index ab2d201b72b..386e15f4bfe 100644 --- a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeTableLookuper.java +++ b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeTableLookuper.java @@ -60,6 +60,17 @@ interface LookupMetricRecorder { @Nullable byte[] lookup(byte[] key, LookupContext context) throws Exception; + /** + * Refreshes the registered lake data files while preserving reusable local lookup files. + * + *

This method may be called concurrently with {@link #lookup(byte[], LookupContext)}. + * Implementations must ensure that it is thread-safe. + */ + default void refresh() throws Exception { + throw new UnsupportedOperationException( + "Refreshing registered files is not supported by this lake table lookuper."); + } + /** Context for a lake table point lookup. */ final class LookupContext { private final ResolvedPartitionSpec partitionSpec; diff --git a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/PluginLakeStorageWrapper.java b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/PluginLakeStorageWrapper.java index 2f778bc0a90..7b3d95c0f16 100644 --- a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/PluginLakeStorageWrapper.java +++ b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/PluginLakeStorageWrapper.java @@ -166,6 +166,13 @@ private ClassLoaderFixingLakeTableLookuper(LakeTableLookuper inner, ClassLoader } } + @Override + public void refresh() throws Exception { + try (TemporaryClassLoaderContext ignored = TemporaryClassLoaderContext.of(loader)) { + inner.refresh(); + } + } + @Override public void close() throws Exception { try (TemporaryClassLoaderContext ignored = TemporaryClassLoaderContext.of(loader)) { diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java index 43497c23781..10a5d06fad6 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java @@ -61,10 +61,13 @@ import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Supplier; import static org.apache.fluss.config.ConfigOptions.KV_FORMAT_VERSION_2; import static org.apache.fluss.lake.paimon.PaimonLakeCatalog.LEGACY_SYSTEM_COLUMNS; @@ -87,11 +90,16 @@ * lookup I/O failure refreshes that partition-bucket with the files from the latest snapshot and * retries once. * + *

An explicit refresh rescans every registered partition-bucket and updates its file set in + * place. Paimon keeps lookup files for data files that remain active and lazily downloads lookup + * files only for newly added data files. + * *

Calls to {@link LocalTableQuery#lookup} are serialized because Paimon 2.0 shares mutable * lookup-store comparator state across local lookup files. * *

Close is expected only after the owner has drained active lookups. It is synchronized with - * lazy initialization, but deliberately does not add a lifecycle lock to every lookup. + * lazy initialization and file-set updates, but deliberately does not add a lifecycle lock to every + * lookup. */ public class PaimonLakeTableLookuper implements LakeTableLookuper { @@ -104,7 +112,8 @@ public class PaimonLakeTableLookuper implements LakeTableLookuper { private final ThreadLocal lookupFileDownloaded; private final Object paimonLookupLock; - private final Object initializationLock; + // Guards lazy initialization, close, and registered file-set updates. + private final Object lookupStateLock; private final Map> registeredFiles; private @Nullable Catalog catalog; @@ -117,7 +126,7 @@ public class PaimonLakeTableLookuper implements LakeTableLookuper { private @Nullable CompactedKeyDecoder compactedKeyDecoder; private volatile @Nullable LocalTableQuery localTableQuery; - // Guarded by initializationLock. + // Guarded by lookupStateLock. private volatile boolean closed; /** Creates a lookuper with the specified local lookup cache limit. */ @@ -138,7 +147,7 @@ public PaimonLakeTableLookuper( this.diskWriteGuard = checkNotNull(diskWriteGuard, "diskWriteGuard must not be null."); this.lookupFileDownloaded = new ThreadLocal<>(); this.paimonLookupLock = new Object(); - this.initializationLock = new Object(); + this.lookupStateLock = new Object(); this.registeredFiles = new ConcurrentHashMap<>(); } @@ -161,9 +170,26 @@ public PaimonLakeTableLookuper( } } + @Override + public void refresh() { + synchronized (lookupStateLock) { + checkNotClosed(); + Map> filesBeforeRefresh = + new LinkedHashMap<>(registeredFiles); + Map> latestFiles = + scanDataFiles(filesBeforeRefresh.keySet()); + filesBeforeRefresh.forEach( + (partitionBucket, files) -> + refreshFiles( + partitionBucket, + files, + () -> latestFiles.get(partitionBucket))); + } + } + @Override public void close() { - synchronized (initializationLock) { + synchronized (lookupStateLock) { if (closed) { return; } @@ -189,7 +215,7 @@ private void checkNotClosed() { private void ensureInitialized(RowType valueRowType) throws Exception { if (localTableQuery == null) { - synchronized (initializationLock) { + synchronized (lookupStateLock) { if (localTableQuery == null) { initialize(valueRowType); } @@ -338,11 +364,12 @@ private org.apache.paimon.data.BinaryRow getKey(byte[] key, LookupContext contex int bucket, org.apache.paimon.data.InternalRow key) throws IOException { - List filesBeforeLookup = initializeFiles(partition, bucket); + PaimonPartitionBucket partitionBucket = new PaimonPartitionBucket(partition, bucket); + List filesBeforeLookup = getOrInitializeFiles(partitionBucket); try { return lookupLocalTable(partition, bucket, key); } catch (IOException firstError) { - refreshFilesIfUnchanged(partition, bucket, filesBeforeLookup); + refreshFiles(partitionBucket, filesBeforeLookup, () -> scanDataFiles(partitionBucket)); try { return lookupLocalTable(partition, bucket, key); } catch (IOException retryError) { @@ -366,56 +393,106 @@ private org.apache.paimon.data.BinaryRow getKey(byte[] key, LookupContext contex } } - private List initializeFiles( - org.apache.paimon.data.BinaryRow partition, int bucket) { - PaimonPartitionBucket partitionBucket = new PaimonPartitionBucket(partition, bucket); - return registeredFiles.computeIfAbsent( - partitionBucket, - ignored -> scanAndUpdateFiles(partition, bucket, Collections.emptyList())); + private List getOrInitializeFiles(PaimonPartitionBucket partitionBucket) { + List files = registeredFiles.get(partitionBucket); + if (files != null) { + return files; + } + // Coordinate only first registration with bulk refresh; existing entries use the fast path. + synchronized (lookupStateLock) { + return registeredFiles.computeIfAbsent( + partitionBucket, + ignored -> { + List latestFiles = scanDataFiles(partitionBucket); + return applyFileRefresh( + partitionBucket, Collections.emptyList(), latestFiles); + }); + } } - private void refreshFilesIfUnchanged( - org.apache.paimon.data.BinaryRow partition, - int bucket, - List filesBeforeLookup) { - PaimonPartitionBucket partitionBucket = new PaimonPartitionBucket(partition, bucket); - registeredFiles.compute( - partitionBucket, - (ignored, currentFiles) -> { - List files = - checkNotNull( - currentFiles, "Partition-bucket files must be initialized."); - return files == filesBeforeLookup - ? scanAndUpdateFiles(partition, bucket, filesBeforeLookup) - : files; - }); + private void refreshFiles( + PaimonPartitionBucket partitionBucket, + List filesBeforeRefresh, + Supplier> latestFilesSupplier) { + synchronized (lookupStateLock) { + registeredFiles.compute( + partitionBucket, + (ignored, currentFiles) -> { + List files = + checkNotNull( + currentFiles, + "Partition-bucket files must be initialized."); + // File lists are immutable and replaced on refresh. Identity equality means + // no other refresh has updated this partition-bucket since the caller + // captured it. + return files == filesBeforeRefresh + ? applyFileRefresh( + partitionBucket, + filesBeforeRefresh, + latestFilesSupplier.get()) + : files; + }); + } } - private List scanAndUpdateFiles( - org.apache.paimon.data.BinaryRow partition, - int bucket, - List filesBeforeRefresh) { - List latestFiles = scanDataFiles(partition, bucket); + private List applyFileRefresh( + PaimonPartitionBucket partitionBucket, + List filesBeforeRefresh, + List latestFiles) { + org.apache.paimon.data.BinaryRow partition = partitionBucket.getPartition(); + int bucket = partitionBucket.getBucket(); localTableQuery.refreshFiles(partition, bucket, filesBeforeRefresh, latestFiles); return latestFiles; } - private List scanDataFiles( - org.apache.paimon.data.BinaryRow partition, int bucket) { - LinkedHashMap dataFilesByName = new LinkedHashMap<>(); + private List scanDataFiles(PaimonPartitionBucket partitionBucket) { + return scanDataFiles(Collections.singleton(partitionBucket)).get(partitionBucket); + } + + private Map> scanDataFiles( + Set partitionBuckets) { + if (partitionBuckets.isEmpty()) { + return Collections.emptyMap(); + } + + Set partitions = new HashSet<>(); + Set buckets = new HashSet<>(); + Map> dataFilesByPartitionBucket = + new LinkedHashMap<>(); + for (PaimonPartitionBucket partitionBucket : partitionBuckets) { + partitions.add(partitionBucket.getPartition()); + buckets.add(partitionBucket.getBucket()); + dataFilesByPartitionBucket.put(partitionBucket, new LinkedHashMap<>()); + } + InnerTableScan tableScan = fileStoreTable .newScan() - .withPartitionFilter(Collections.singletonList(partition)) - .withBucket(bucket); + .withPartitionFilter(new ArrayList<>(partitions)) + .withBucketFilter(buckets::contains); for (Split split : tableScan.plan().splits()) { if (split instanceof DataSplit) { - for (DataFileMeta file : ((DataSplit) split).dataFiles()) { - dataFilesByName.put(file.fileName(), file); + DataSplit dataSplit = (DataSplit) split; + LinkedHashMap dataFilesByName = + dataFilesByPartitionBucket.get( + new PaimonPartitionBucket( + dataSplit.partition(), dataSplit.bucket())); + if (dataFilesByName != null) { + for (DataFileMeta file : dataSplit.dataFiles()) { + dataFilesByName.put(file.fileName(), file); + } } } } - return Collections.unmodifiableList(new ArrayList<>(dataFilesByName.values())); + + Map> latestFiles = new LinkedHashMap<>(); + dataFilesByPartitionBucket.forEach( + (partitionBucket, dataFilesByName) -> + latestFiles.put( + partitionBucket, + Collections.unmodifiableList( + new ArrayList<>(dataFilesByName.values())))); + return latestFiles; } private byte[] encodeValue( diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java index f493f62317c..6492e53c88e 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java @@ -231,6 +231,101 @@ void testConcurrentFirstLookupsForDifferentPartitions() throws Exception { } } + @Test + void testRefreshesFilesWhenLakeSnapshotChanges() throws Exception { + TablePath tablePath = TablePath.of(DB, "refresh_registered_files"); + Schema schema = pkSchema(); + FileStoreTable table = createPaimonTable(tablePath, partitionedPkDescriptor(schema)); + writeAndCommitData( + table, + Collections.singletonMap( + 0, + Arrays.asList( + paimonRow(1, "20240101", "Alice"), + paimonRow(2, "20240102", "Bob")))); + + try (LakeTableLookuper lookuper = + new PaimonLakeTableLookuper( + paimonConfig, + tablePath, + tempWarehouseDir.getAbsolutePath(), + tableConfig(KvFormat.COMPACTED), + LOOKUP_CACHE_MAX_DISK_BYTES, + NO_OP_DISK_WRITE_GUARD)) { + LakeTableLookuper.LookupContext firstPartition = + lookupContext(schema, "20240101", 0, SCHEMA_ID); + LakeTableLookuper.LookupContext secondPartition = + lookupContext(schema, "20240102", 0, SCHEMA_ID); + // Register two partition-buckets and populate their local lookup files. + assertThat(lookuper.lookup(paimonKey(schema, 1, "20240101"), firstPartition)) + .isNotNull(); + assertThat(lookuper.lookup(paimonKey(schema, 2, "20240102"), secondPartition)) + .isNotNull(); + + writeAndCommitData( + table, + Collections.singletonMap( + 0, Collections.singletonList(paimonRow(3, "20240101", "Carol")))); + // The newly committed file is not registered before the explicit refresh. + assertThat(lookuper.lookup(paimonKey(schema, 3, "20240101"), firstPartition)).isNull(); + + List newFileDownloads = new ArrayList<>(); + List retainedFileDownloads = new ArrayList<>(); + List unchangedPartitionDownloads = new ArrayList<>(); + lookuper.refresh(); + + BinaryValue newFileValue = + decodeValue( + lookuper.lookup( + paimonKey(schema, 3, "20240101"), + lookupContext( + schema, + "20240101", + 0, + SCHEMA_ID, + (lookupTimeNanos, lookupFileDownloaded) -> + newFileDownloads.add(lookupFileDownloaded))), + SCHEMA_ID, + schema); + BinaryValue retainedFileValue = + decodeValue( + lookuper.lookup( + paimonKey(schema, 1, "20240101"), + lookupContext( + schema, + "20240101", + 0, + SCHEMA_ID, + (lookupTimeNanos, lookupFileDownloaded) -> + retainedFileDownloads.add( + lookupFileDownloaded))), + SCHEMA_ID, + schema); + BinaryValue unchangedValue = + decodeValue( + lookuper.lookup( + paimonKey(schema, 2, "20240102"), + lookupContext( + schema, + "20240102", + 0, + SCHEMA_ID, + (lookupTimeNanos, lookupFileDownloaded) -> + unchangedPartitionDownloads.add( + lookupFileDownloaded))), + SCHEMA_ID, + schema); + + assertRow(newFileValue.row, 3, "20240101", "Carol"); + assertRow(retainedFileValue.row, 1, "20240101", "Alice"); + assertRow(unchangedValue.row, 2, "20240102", "Bob"); + // Only the new data file needs a local lookup-file download after the bulk refresh. + assertThat(newFileDownloads).containsExactly(true); + assertThat(retainedFileDownloads).containsExactly(false); + assertThat(unchangedPartitionDownloads).containsExactly(false); + } + } + @Test void testDiskWriteLockBlocksOnlyLookupFileDownloads() throws Exception { TablePath tablePath = TablePath.of(DB, "disk_write_lock"); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManager.java index af86581e44d..ea5e93b2337 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManager.java @@ -22,6 +22,7 @@ import org.apache.fluss.config.Configuration; import org.apache.fluss.config.TableConfig; import org.apache.fluss.exception.FlussRuntimeException; +import org.apache.fluss.exception.KvStorageException; import org.apache.fluss.exception.LakeStorageNotConfiguredException; import org.apache.fluss.lake.lakestorage.LakeStorage; import org.apache.fluss.lake.lakestorage.LakeStoragePlugin; @@ -80,7 +81,7 @@ * than table path to prevent a deleted and recreated table from reusing the old table's lookuper. A * cached lookuper is replaced when its schema ID or lake configuration version no longer matches * the current request. Active lookups can finish on the old lookuper, which is closed after its - * last lookup releases it. + * last lookup releases it. A new required lake snapshot refreshes the cached lookuper in place. * *

Up to ten table lookupers are cached. Each lookuper receives one tenth of the server-level * disk budget, and Caffeine evicts lookupers when the table limit is exceeded. @@ -272,7 +273,7 @@ void invalidateTableLookuper(long tableId) { lakeTableLookupers.invalidate(tableId); } - /** Records the required opaque lake snapshot ID, which is compared only by equality. */ + /** Records the required opaque lake snapshot ID for the next in-place file refresh. */ void requireLakeSnapshot(long tableId, long snapshotId) { requiredLakeSnapshotIds.put(tableId, snapshotId); } @@ -481,12 +482,16 @@ private CachedLakeTableLookuper acquireLookuper(LookupContext context, TableInfo long currentLakeConfigVersion = lakeConfigVersion; Configuration currentConf = conf; long cacheSizeBytes = lookupCacheMaxDiskBytesPerTable; - Long requiredLakeSnapshotId = requiredLakeSnapshotIds.get(context.tableId); return lakeTableLookupers .asMap() .compute( context.tableId, (ignored, currentLookuper) -> { + // Read inside the per-table atomic update to avoid refreshing back to a + // snapshot captured while waiting for another lookup to finish updating + // this lookuper. + Long requiredLakeSnapshotId = + requiredLakeSnapshotIds.get(context.tableId); CachedLakeTableLookuper selectedLookuper = currentLookuper; // Create the lookuper lazily, and recreate it after schema, // lake configuration, or server cache size changes so it @@ -496,10 +501,7 @@ private CachedLakeTableLookuper acquireLookuper(LookupContext context, TableInfo || selectedLookuper.schemaId != context.schemaId || selectedLookuper.lakeConfigVersion != currentLakeConfigVersion - || selectedLookuper.cacheSizeBytes != cacheSizeBytes - || !Objects.equals( - selectedLookuper.lakeSnapshotId, - requiredLakeSnapshotId)) { + || selectedLookuper.cacheSizeBytes != cacheSizeBytes) { File tableLookupDir = FlussPaths.historicalLookupTableDir( historicalLookupCacheRootDir, @@ -526,7 +528,7 @@ private CachedLakeTableLookuper acquireLookuper(LookupContext context, TableInfo // Pin the lookuper before leaving the atomic cache update. // Eviction or invalidation can then defer closing it until // this lookup releases it. - selectedLookuper.acquire(); + selectedLookuper.acquire(requiredLakeSnapshotId); return selectedLookuper; }); } @@ -555,8 +557,8 @@ private static final class CachedLakeTableLookuper { private final int schemaId; private final long lakeConfigVersion; private final long cacheSizeBytes; - /** The required opaque lake snapshot ID when this lookuper was created, or null if none. */ - private final @Nullable Long lakeSnapshotId; + /** The opaque lake snapshot ID covered by the last file refresh, or null if none. */ + private @Nullable Long lakeSnapshotId; private final File tableLookupDir; private final LakeTableLookuper lookuper; @@ -583,10 +585,20 @@ private CachedLakeTableLookuper( this.lookuper = lookuper; } - private synchronized void acquire() { + private synchronized void acquire(@Nullable Long requiredLakeSnapshotId) { if (invalidated) { throw new IllegalStateException("Lake table lookuper has been invalidated."); } + if (!Objects.equals(lakeSnapshotId, requiredLakeSnapshotId)) { + try { + lookuper.refresh(); + } catch (Exception e) { + throw new KvStorageException( + "Failed to refresh historical lake lookup files for " + tablePath + ".", + e); + } + lakeSnapshotId = requiredLakeSnapshotId; + } activeLookups++; } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java index 93d720ae6fe..b12f60c9500 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java @@ -286,7 +286,7 @@ public void invalidateTableLookuper(long tableId) { lakeLookupManager.invalidateTableLookuper(tableId); } - /** Requires future fallback lookups to reload after the given lake snapshot notification. */ + /** Requires future fallback lookups to refresh after the given lake snapshot notification. */ public void requireLakeSnapshot(long tableId, long lakeSnapshotId) { lakeLookupManager.requireLakeSnapshot(tableId, lakeSnapshotId); } diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManagerTest.java index 364ec46bedc..0be4672c01b 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManagerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManagerTest.java @@ -162,29 +162,30 @@ void testInvalidatesLookuperOnSchemaAndLifecycleChanges() throws Exception { } @Test - void testRefreshesLookuperWhenLakeSnapshotChanges() throws Exception { + void testRefreshesFilesWhenLakeSnapshotChanges() throws Exception { TestingHistoricalLakeLookupManager manager = createTestingManager(); lookup(manager, PARTITION_TABLE_INFO); - TestingLakeTableLookuper initialLookuper = manager.createdLookupers.get(0); + TestingLakeTableLookuper lookuper = manager.createdLookupers.get(0); manager.requireLakeSnapshot(PARTITION_TABLE_ID, 10L); lookup(manager, PARTITION_TABLE_INFO); - assertThat(initialLookuper.closed).isTrue(); - assertThat(manager.createdLookupers).hasSize(2); + assertThat(lookuper.closed).isFalse(); + assertThat(lookuper.refreshCount).isOne(); + assertThat(manager.createdLookupers).containsExactly(lookuper); - TestingLakeTableLookuper snapshotTenLookuper = manager.createdLookupers.get(1); // Snapshot IDs are opaque; a numerically smaller ID may identify a newer snapshot. manager.requireLakeSnapshot(PARTITION_TABLE_ID, 9L); lookup(manager, PARTITION_TABLE_INFO); - assertThat(snapshotTenLookuper.closed).isTrue(); - assertThat(manager.createdLookupers).hasSize(3); + assertThat(lookuper.closed).isFalse(); + assertThat(lookuper.refreshCount).isEqualTo(2); + assertThat(manager.createdLookupers).containsExactly(lookuper); - TestingLakeTableLookuper snapshotNineLookuper = manager.createdLookupers.get(2); manager.requireLakeSnapshot(PARTITION_TABLE_ID, 9L); lookup(manager, PARTITION_TABLE_INFO); - assertThat(snapshotNineLookuper.closed).isFalse(); - assertThat(manager.createdLookupers).hasSize(3); + assertThat(lookuper.closed).isFalse(); + assertThat(lookuper.refreshCount).isEqualTo(2); + assertThat(manager.createdLookupers).containsExactly(lookuper); } @Test @@ -421,6 +422,7 @@ private static final class TestingLakeTableLookuper implements LakeTableLookuper private final long cacheFileBytes; private boolean closed; private boolean cacheFileDownloaded; + private int refreshCount; private final List lookupContexts = new ArrayList<>(); private TestingLakeTableLookuper(File lookupDir, long cacheFileBytes) { @@ -447,6 +449,14 @@ public byte[] lookup(byte[] key, LookupContext context) throws Exception { return key; } + @Override + public void refresh() { + if (closed) { + throw new IllegalStateException("Lookuper is already closed."); + } + refreshCount++; + } + @Override public void close() throws Exception { closed = true; From 7781edc539d5d9f01753adda3107313c9bae5170 Mon Sep 17 00:00:00 2001 From: luoyuxia Date: Sun, 30 Aug 2026 21:02:39 +0800 Subject: [PATCH 6/8] [test] Exclude lake lookuper from coverage Exclude the plugin-facing LakeTableLookuper interface from the aggregate per-class coverage rule. Co-Authored-By: Codex AI-Model: gpt-5 AI-Contributed/Feature: 3/3 AI-Contributed/UT: 0/0 --- fluss-test-coverage/pom.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fluss-test-coverage/pom.xml b/fluss-test-coverage/pom.xml index d8428c4a013..226a89eeea6 100644 --- a/fluss-test-coverage/pom.xml +++ b/fluss-test-coverage/pom.xml @@ -491,6 +491,9 @@ org.apache.fluss.lake.source.* org.apache.fluss.lake.lakestorage.LakeStorage + + org.apache.fluss.lake.lakestorage.LakeTableLookuper + org.apache.fluss.lake.lakestorage.LakeStorage.LookuperContext From 839c54dd10dfe7009d4507da836c44e2d11f171c Mon Sep 17 00:00:00 2001 From: luoyuxia Date: Mon, 31 Aug 2026 14:10:39 +0800 Subject: [PATCH 7/8] [client][server][paimon] Address historical write review Remove the coupled historical KV cleanup mechanism, defer lake lookup refresh I/O until lookup initialization, and simplify historical write version and routing checks. Reroute missing original targets when no request remains in flight, abort only the affected target when handoff is unsafe, and document the remaining retirement ambiguity. Co-Authored-By: Codex AI-Model: gpt-5.6-sol AI-Contributed/Feature: 56/56 AI-Contributed/UT: 2/2 --- .../fluss/client/write/RecordAccumulator.java | 135 +++++- .../org/apache/fluss/client/write/Sender.java | 157 +++++-- .../apache/fluss/client/write/WriteBatch.java | 16 +- .../fluss/client/write/WriterClient.java | 36 +- .../apache/fluss/client/write/SenderTest.java | 112 ++++- .../apache/fluss/config/ConfigOptions.java | 8 - .../lake/lakestorage/LakeTableLookuper.java | 8 +- .../lakestorage/PluginLakeStorageWrapper.java | 2 +- .../lookup/PaimonLakeTableLookuper.java | 58 ++- .../rpc/netty/client/ServerConnection.java | 26 +- .../fluss/server/DynamicServerConfig.java | 2 - .../HistoricalLookupCacheConfigValidator.java | 11 +- .../apache/fluss/server/replica/Replica.java | 169 +------- .../fluss/server/replica/ReplicaManager.java | 21 +- .../HistoricalLakeLookupManager.java | 9 +- .../HistoricalPartitionManager.java | 284 +------------ .../HistoricalPartitionTaskExecutor.java | 64 +-- .../fluss/server/DynamicConfigChangeTest.java | 11 - .../replica/HighWatermarkPersistenceTest.java | 27 +- .../HistoricalPartitionManagerTest.java | 393 +----------------- 20 files changed, 483 insertions(+), 1066 deletions(-) diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java b/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java index 19362d8a948..9d78ae5da89 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java @@ -117,8 +117,9 @@ public final class RecordAccumulator { private final ConcurrentMap writeBatches = new CopyOnWriteMap<>(); - /** Tables observed by this writer with historical partition support enabled. */ - private final Set historicalPartitionEnabledTables = ConcurrentHashMap.newKeySet(); + /** Whether tables observed by this writer have historical partition support enabled. */ + private final ConcurrentMap historicalPartitionEnabledByTable = + new ConcurrentHashMap<>(); private final IncompleteBatches incomplete; @@ -203,9 +204,6 @@ public RecordAppendResult append( throws Exception { PhysicalTablePath physicalTablePath = writeRecord.getPhysicalTablePath(); TableInfo tableInfo = writeRecord.getTableInfo(); - if (tableInfo.getTableConfig().isHistoricalPartitionEnabled()) { - historicalPartitionEnabledTables.add(tableInfo.getTablePath()); - } // The metadata may return null for the partition id, but it is fine to pass null here, // because we will fill the partitionId in bucketReady() before send the batch. Optional partitionIdOpt = cluster.getPartitionId(physicalTablePath); @@ -283,12 +281,10 @@ public ReadyCheckResult ready(Cluster cluster) { // Go table by table so that we can get queue sizes for buckets in a table and calculate // cumulative frequency table (used in bucket assigner). - for (Map.Entry writeBatchesEntry : - writeBatches.entrySet()) { + for (BucketAndWriteBatches bucketAndWriteBatches : writeBatches.values()) { nextReadyCheckDelayMs = bucketReady( - writeBatchesEntry.getKey(), - writeBatchesEntry.getValue(), + bucketAndWriteBatches, readyNodes, unknownLeaderTables, cluster, @@ -329,6 +325,9 @@ public Map> drain( public void reEnqueue(ReadyWriteBatch readyWriteBatch) { WriteBatch batch = readyWriteBatch.writeBatch(); + if (batch.isDone()) { + return; + } batch.reEnqueued(); Deque deque = getOrCreateDeque(readyWriteBatch.tableBucket(), batch.physicalTablePath()); @@ -380,8 +379,7 @@ void routeWritesTo( + "incomplete writes to %s.", originalPath, targetPath, existing.targetPath)); } - existing.targetPath = targetPath; - existing.partitionId = targetPartitionId; + existing.switchToHistoricalTarget(targetPath, targetPartitionId); return; } @@ -408,9 +406,88 @@ boolean hasHistoricalWriteTarget(PhysicalTablePath originalPath) { return writeTarget != null && writeTarget.isHistoricalWriteTarget(); } - /** Returns whether the target belongs to a table with historical partition support enabled. */ - boolean isHistoricalPartitionEnabled(PhysicalTablePath targetPath) { - return historicalPartitionEnabledTables.contains(targetPath.getTablePath()); + /** + * Checks whether a table has historical partition support enabled and caches the result. + * + *

This method is called from the per-record write path. Reading the value from {@link + * TableInfo#getTableConfig()} for every record would repeatedly enter the synchronized {@code + * Configuration} lookup path. Caching both enabled and disabled results keeps subsequent checks + * to a concurrent-map read. The cache is shared here because WriterClient, batch creation, and + * Sender must use the same table classification. + */ + boolean checkAndCacheHistoricalPartitionEnabled(TableInfo tableInfo) { + TablePath tablePath = tableInfo.getTablePath(); + Boolean cached = historicalPartitionEnabledByTable.get(tablePath); + if (cached != null) { + return cached; + } + + boolean historicalPartitionEnabled = + tableInfo.getTableConfig().isHistoricalPartitionEnabled(); + // A writer observes stable table configuration, so concurrent initializers compute the + // same value even if another thread wins putIfAbsent. + historicalPartitionEnabledByTable.putIfAbsent(tablePath, historicalPartitionEnabled); + return historicalPartitionEnabled; + } + + /** Returns the cached historical partition setting for a table. */ + boolean isHistoricalPartitionEnabled(TablePath tablePath) { + return Boolean.TRUE.equals(historicalPartitionEnabledByTable.get(tablePath)); + } + + /** Reroutes queued batches for {@code originalPath} to the historical target. */ + void rerouteQueuedWritesToHistorical( + PhysicalTablePath originalPath, + PhysicalTablePath historicalPath, + long historicalPartitionId) { + BucketAndWriteBatches writeTarget = + checkNotNull( + writeBatches.get(originalPath), + "Write target for %s must exist.", + originalPath); + String originalPartitionName = checkNotNull(originalPath.getPartitionName()); + long originalPartitionId; + synchronized (writeTarget) { + if (writeTarget.isHistoricalWriteTarget()) { + writeTarget.partitionId = historicalPartitionId; + return; + } + // New appends observe the historical route and are created with the original partition + // name. Existing queued batches are converted below before the Sender can drain again. + originalPartitionId = + writeTarget.switchToHistoricalTarget(historicalPath, historicalPartitionId); + } + // The caller has confirmed that no request to the original target remains in flight. + // Convert every queued bucket under its deque lock so append and drain cannot observe a + // batch between detaching its original idempotence state and marking it as historical. + for (Map.Entry> entry : writeTarget.batches.entrySet()) { + Deque deque = entry.getValue(); + synchronized (deque) { + for (WriteBatch batch : deque) { + if (idempotenceManager.idempotenceEnabled() && batch.hasBatchSequence()) { + idempotenceManager.removeInFlightBatch( + new ReadyWriteBatch( + new TableBucket( + batch.tableId(), + originalPartitionId, + entry.getKey()), + batch)); + batch.resetWriterState(NO_WRITER_ID, NO_BATCH_SEQUENCE); + } + batch.rerouteToHistoricalPartition(originalPartitionName); + } + } + } + } + + /** Aborts incomplete batches whose current RPC target is {@code targetPath}. */ + void abortBatches(PhysicalTablePath targetPath, Exception reason) { + for (WriteBatch batch : incomplete.copyAll()) { + BucketAndWriteBatches writeTarget = writeBatches.get(batch.physicalTablePath()); + if (writeTarget != null && writeTarget.targetPath.equals(targetPath)) { + abortBatch(reason, batch); + } + } } /** Abort all incomplete batches (whether they have been sent or not). */ @@ -561,7 +638,6 @@ private List allocateMemorySegments( /** Check whether there are bucket ready for input table. */ private long bucketReady( - PhysicalTablePath physicalTablePath, BucketAndWriteBatches bucketAndWriteBatches, Set readyNodes, Set unknownLeaderTables, @@ -575,8 +651,7 @@ private long bucketReady( bucketAndWriteBatches.partitionId = optionIdOpt.get(); } else { LOG.debug( - "Partition not exists for {}, bucket will not be set to ready", - physicalTablePath); + "Partition not exists for {}, bucket will not be set to ready", targetPath); // TODO: we shouldn't add unready partitions to unknownLeaderTables, // because it cases PartitionNotExistException later unknownLeaderTables.add(targetPath); @@ -613,7 +688,7 @@ private long bucketReady( } int bucketId = entry.getKey(); - Optional tableIdOpt = cluster.getTableId(physicalTablePath.getTablePath()); + Optional tableIdOpt = cluster.getTableId(targetPath.getTablePath()); if (!tableIdOpt.isPresent()) { unknownLeaderTables.add(targetPath); } else { @@ -706,10 +781,10 @@ private RecordAppendResult appendNewBatch( writeBatches.get(physicalTablePath), "Write batches for %s must exist.", physicalTablePath); - // Only historical-enabled tables need to coordinate with routeWritesTo(). Other tables - // reuse the deque monitor already held by the caller, avoiding cross-bucket serialization. + // Historical-enabled tables coordinate with routeWritesTo(). Other tables reuse the + // deque monitor already held by the caller, avoiding cross-bucket serialization. Object routeLock = - tableInfo.getTableConfig().isHistoricalPartitionEnabled() + isHistoricalPartitionEnabled(physicalTablePath.getTablePath()) ? bucketAndWriteBatches : deque; synchronized (routeLock) { @@ -1319,5 +1394,23 @@ private BucketAndWriteBatches( public boolean isHistoricalWriteTarget() { return HISTORICAL_PARTITION_VALUE.equals(targetPath.getPartitionName()); } + + /** + * Atomically switches the target path and partition ID to the historical partition. + * + *

This method must be called while the current target is still the original partition. + * The returned original partition ID is used to detach queued batches from their original + * idempotence state before rerouting them. + */ + private synchronized long switchToHistoricalTarget( + PhysicalTablePath historicalPath, long historicalPartitionId) { + long originalPartitionId = + checkNotNull( + partitionId, + "Original partition ID must be resolved before rerouting."); + targetPath = historicalPath; + partitionId = historicalPartitionId; + return originalPartitionId; + } } } diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java b/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java index 06f03ca1991..544a78668ab 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java @@ -232,7 +232,7 @@ private void sendWriteData() throws Exception { // unready partitions Throwable t = ExceptionUtils.stripExecutionException(e); if (t instanceof PartitionNotExistException) { - abortIfHistoricalWriteTargetMissing(readyCheckResult.unknownLeaderTables); + handlePartitionNotExistException(readyCheckResult.unknownLeaderTables); } else { throw e; } @@ -634,12 +634,7 @@ private Set handleWriteBatchException( WriteBatch writeBatch = readyWriteBatch.writeBatch(); // Historical queues use the original path as their accumulator key, so capture the actual // RPC target before any retry handling. - PhysicalTablePath requestTargetPath = - writeBatch.getOriginalPartitionName() == null - ? writeBatch.physicalTablePath() - : PhysicalTablePath.of( - writeBatch.physicalTablePath().getTablePath(), - HISTORICAL_PARTITION_VALUE); + PhysicalTablePath writeTargetPath = writeBatch.writeTargetPath(); if (error.exception() instanceof StorageBackpressureException) { // Hard rejection: the storage engine reached its slowdown trigger and rejected the // write. Map it to full pressure (internal hard-rejection value 1.0f) so the bucket is @@ -711,7 +706,7 @@ private Set handleWriteBatchException( // A historical batch remains keyed by its original partition path in the // accumulator, but its RPC is sent to the internal historical partition. Invalidate // the actual RPC target so the retry refreshes the historical bucket metadata. - invalidMetadataTables.add(requestTargetPath); + invalidMetadataTables.add(writeTargetPath); } } else { LOG.warn( @@ -727,14 +722,13 @@ private Set handleWriteBatchException( } /** - * Aborts pending writes when metadata confirms their target is missing. Such writes cannot make - * progress without a leader, and rerouting existing batches to the historical target is unsafe - * because their outcome and idempotent state may belong to the original target. + * Rechecks unknown-leader partitions after a bulk metadata update reports {@link + * PartitionNotExistException}, and handles missing partitions for tables with historical + * partition support. */ - private void abortIfHistoricalWriteTargetMissing(Set unknownLeaderTables) - throws Exception { + private void handlePartitionNotExistException(Set unknownLeaderTables) { for (PhysicalTablePath targetPath : unknownLeaderTables) { - if (!accumulator.isHistoricalPartitionEnabled(targetPath)) { + if (!accumulator.isHistoricalPartitionEnabled(targetPath.getTablePath())) { continue; } try { @@ -742,36 +736,123 @@ private void abortIfHistoricalWriteTargetMissing(Set unknownL } catch (Exception e) { Throwable t = ExceptionUtils.stripExecutionException(e); if (t instanceof PartitionNotExistException) { - // This target was considered usable when its batches were enqueued or first - // attempted, but is now confirmed missing. Transparently rerouting those - // batches to the historical partition is unsafe: an original-target attempt - // may have been accepted despite a lost response, and writer ID / batch - // sequence state cannot be reused across different physical TableBuckets. A - // safe failover must stop draining this path, wait for its in-flight requests, - // classify ambiguous outcomes, reset writer state, and preserve per-bucket - // ordering. This race requires partition retirement to overlap a writer that - // still holds the original route, so it is expected to be uncommon; fail - // closed for now. - // TODO: Implement safe in-flight historical failover if this path occurs - // frequently in practice. - // Retrying a historical-enabled table without a leader would leave its - // batches queued indefinitely. Fail only after checking the target itself so - // ordinary writes in the bulk metadata request keep their existing behavior. - PartitionNotExistException missingTargetException = - new PartitionNotExistException( - "Write target " - + targetPath - + " for a historical-partition-enabled table no " - + "longer exists according to refreshed metadata."); - missingTargetException.initCause(t); - maybeAbortBatches(missingTargetException); - return; + handleMissingPartition(targetPath, t); + continue; } throw e; } } } + /** + * Handles a write target after refreshed metadata confirms that it no longer exists. + * + *

A missing historical target has no further fallback and is aborted. An original target is + * rerouted only after all requests to it have left the in-flight set, so no response from the + * old physical target can race with resetting its queued batches to the historical target. + */ + private void handleMissingPartition(PhysicalTablePath targetPath, Throwable cause) { + if (HISTORICAL_PARTITION_VALUE.equals(targetPath.getPartitionName())) { + abortBatches( + targetPath, + newPartitionNotExistException( + "Cannot continue historical writes to " + + targetPath + + " because refreshed metadata confirms that the partition " + + "no longer exists.", + cause)); + return; + } + + if (hasInFlightBatches(targetPath)) { + // Supporting this case requires waiting for the outstanding responses, detaching the + // batches from the original TableBucket's idempotence state, preserving their order, + // and assigning new sequences for the historical TableBucket. Keep this transition + // simple by failing only the affected target. + // TODO: If this race occurs frequently in practice, implement the coordinated + // in-flight handoff instead of aborting the batches. + abortBatches( + targetPath, + newPartitionNotExistException( + "Cannot safely reroute writes from missing partition " + + targetPath + + " while requests to it are still in flight because rerouting " + + "them to the historical partition could break idempotence.", + cause)); + return; + } + + // TODO: No current in-flight request does not rule out an earlier attempt that was + // accepted while its response was lost. Rerouting such a batch can duplicate the write; + // target-scoped abort only exposes the ambiguity and cannot prevent an upper-layer replay + // from duplicating it. Resolve this with the durable FREEZING/RETIRED handoff; see + // https://github.com/apache/fluss/issues/4161. + PhysicalTablePath historicalPath = + PhysicalTablePath.of(targetPath.getTablePath(), HISTORICAL_PARTITION_VALUE); + @Nullable Throwable historicalTargetCause = null; + try { + if (metadataUpdater.checkAndUpdatePartitionMetadata(historicalPath)) { + accumulator.rerouteQueuedWritesToHistorical( + targetPath, + historicalPath, + metadataUpdater.getPartitionIdOrElseThrow(historicalPath)); + return; + } + } catch (PartitionNotExistException e) { + historicalTargetCause = e; + } + + abortBatches( + targetPath, + newPartitionNotExistException( + "Cannot reroute writes from " + + targetPath + + " because historical write target " + + historicalPath + + " does not exist according to refreshed metadata.", + historicalTargetCause)); + } + + /** Returns whether a request to {@code targetPath} is still awaiting a response. */ + private boolean hasInFlightBatches(PhysicalTablePath targetPath) { + return !getInFlightBatches(targetPath).isEmpty(); + } + + private List getInFlightBatches(PhysicalTablePath targetPath) { + List matchedBatches = new ArrayList<>(); + synchronized (inFlightBatchesLock) { + for (List batches : inFlightBatches.values()) { + for (ReadyWriteBatch batch : batches) { + if (batch.writeBatch().writeTargetPath().equals(targetPath)) { + matchedBatches.add(batch); + } + } + } + } + return matchedBatches; + } + + private void abortBatches(PhysicalTablePath targetPath, PartitionNotExistException exception) { + List matchingInFlightBatches = getInFlightBatches(targetPath); + // Make the batches terminal before detaching their in-flight bookkeeping. + accumulator.abortBatches(targetPath, exception); + for (ReadyWriteBatch batch : matchingInFlightBatches) { + maybeRemoveFromInflightBatches(batch); + if (idempotenceManager.idempotenceEnabled()) { + idempotenceManager.removeInFlightBatch(batch); + } + } + } + + private static PartitionNotExistException newPartitionNotExistException( + String message, @Nullable Throwable cause) { + PartitionNotExistException exception = new PartitionNotExistException(message); + if (cause != null) { + exception.initCause(cause); + } + return exception; + } + private void updateWriterMetrics(Map> batches) { batches.values() .forEach( diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/WriteBatch.java b/fluss-client/src/main/java/org/apache/fluss/client/write/WriteBatch.java index a60e8ca6645..2ea1c3bbc40 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/WriteBatch.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/WriteBatch.java @@ -36,6 +36,7 @@ import java.util.concurrent.atomic.AtomicReference; import static org.apache.fluss.record.LogRecordBatchFormat.NO_BATCH_SEQUENCE; +import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; import static org.apache.fluss.utils.Preconditions.checkNotNull; /** The abstract write batch contains write callback object to wait write request feedback. */ @@ -60,7 +61,7 @@ public abstract class WriteBatch { *

It is null for a normal write and contains the logical partition namespace for a * historical write. */ - private final @Nullable String originalPartitionName; + private volatile @Nullable String originalPartitionName; protected boolean reopened; protected int recordCount; @@ -215,11 +216,24 @@ public PhysicalTablePath physicalTablePath() { return physicalTablePath; } + /** Returns the physical partition path used as the write RPC target. */ + public PhysicalTablePath writeTargetPath() { + return originalPartitionName == null + ? physicalTablePath + : PhysicalTablePath.of( + physicalTablePath.getTablePath(), HISTORICAL_PARTITION_VALUE); + } + /** Returns the original partition name for a historical write, or null for a normal write. */ public @Nullable String getOriginalPartitionName() { return originalPartitionName; } + /** Marks this batch as targeting the historical partition. */ + void rerouteToHistoricalPartition(String partitionName) { + originalPartitionName = checkNotNull(partitionName); + } + public RequestFuture getRequestFuture() { return requestFuture; } diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/WriterClient.java b/fluss-client/src/main/java/org/apache/fluss/client/write/WriterClient.java index e82bc57f8fb..2239b98d95e 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/WriterClient.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/WriterClient.java @@ -58,7 +58,6 @@ import static org.apache.fluss.utils.ExceptionUtils.toException; import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; import static org.apache.fluss.utils.PartitionUtils.generateAutoPartitionTime; -import static org.apache.fluss.utils.PartitionUtils.isPastAutoPartition; /** * A client that write records to server. @@ -202,7 +201,11 @@ private void doSend(WriteRecord record, WriteCallback callback) { PhysicalTablePath physicalTablePath = record.getPhysicalTablePath(); // Skip the call entirely on non-partitioned tables; there is no partition to create. if (tableInfo.isPartitioned()) { - if (mayBeExpiredHistoricalPartition(physicalTablePath, tableInfo, Instant.now())) { + boolean historicalPartitionEnabled = + accumulator.checkAndCacheHistoricalPartitionEnabled(tableInfo); + if (historicalPartitionEnabled + && mayBeExpiredHistoricalPartition( + physicalTablePath, tableInfo, Instant.now())) { resolveHistoricalWriteTarget(physicalTablePath); } else { dynamicPartitionCreator.checkAndCreatePartitionAsync( @@ -254,8 +257,8 @@ private void doSend(WriteRecord record, WriteCallback callback) { } /** - * Returns whether a partition is old enough that it may have expired under its retention - * policy. + * Returns whether a partition of a historical-partition-enabled table is old enough that it may + * have expired under its retention policy. * *

This client-side precheck uses the time zone resolved from the table configuration. A * {@code true} result does not confirm that the partition is missing; the caller must refresh @@ -267,17 +270,19 @@ private void doSend(WriteRecord record, WriteCallback callback) { */ static boolean mayBeExpiredHistoricalPartition( PhysicalTablePath physicalTablePath, TableInfo tableInfo, Instant now) { + // TODO: Move this per-record configuration and time calculation off the hot path by using + // periodically refreshed, server-authoritative partition status; see + // https://github.com/apache/fluss/issues/4161. String partitionName = physicalTablePath.getPartitionName(); - AutoPartitionStrategy strategy = tableInfo.getTableConfig().getAutoPartitionStrategy(); - if (partitionName == null - || !tableInfo.getTableConfig().isHistoricalPartitionEnabled() - || strategy.numToRetain() < 0) { + if (partitionName == null) { return false; } - if (!isPastAutoPartition(partitionName, strategy, now)) { + AutoPartitionStrategy strategy = tableInfo.getTableConfig().getAutoPartitionStrategy(); + if (strategy.numToRetain() < 0) { return false; } + ZonedDateTime currentDateTime = ZonedDateTime.ofInstant(now, strategy.timeZone().toZoneId()); String earliestRetainedPartition = @@ -286,16 +291,20 @@ static boolean mayBeExpiredHistoricalPartition( return partitionName.compareTo(earliestRetainedPartition) < 0; } - private synchronized void resolveHistoricalWriteTarget(PhysicalTablePath originalPath) { + private void resolveHistoricalWriteTarget(PhysicalTablePath originalPath) { + // Keep refreshing while the target is still the original partition so its retirement can + // be detected before more records are appended to the stale route. Ideally, the Client + // should learn the server-authoritative partition status without synchronously refreshing + // metadata on the per-record path; see https://github.com/apache/fluss/issues/4161. if (accumulator.hasHistoricalWriteTarget(originalPath)) { return; } + PhysicalTablePath targetPath = originalPath; // The time check only limits metadata traffic. Invalidate a potentially stale cached route // and authoritatively choose the target before the record enters the queue. metadataUpdater.invalidPhysicalTableBucketAndPartitionMeta( Collections.singleton(originalPath)); - PhysicalTablePath targetPath = originalPath; try { if (!metadataUpdater.checkAndUpdatePartitionMetadata(originalPath)) { throw new FlussRuntimeException( @@ -304,8 +313,9 @@ private synchronized void resolveHistoricalWriteTarget(PhysicalTablePath origina } catch (PartitionNotExistException ignored) { targetPath = PhysicalTablePath.of(originalPath.getTablePath(), HISTORICAL_PARTITION_VALUE); - // TODO: Activate this target only after Server retirement guarantees that all accepted - // original writes have been tiered to the lake. + // TODO: Activate this target only after the lake-aware partition retirement protocol + // guarantees that all accepted original writes are readable from the lake; see + // https://github.com/apache/fluss/pull/3820. if (!metadataUpdater.checkAndUpdatePartitionMetadata(targetPath)) { throw new PartitionNotExistException( "Historical partition " + targetPath + " does not exist."); diff --git a/fluss-client/src/test/java/org/apache/fluss/client/write/SenderTest.java b/fluss-client/src/test/java/org/apache/fluss/client/write/SenderTest.java index 925d92ab623..132da6cc112 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/write/SenderTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/write/SenderTest.java @@ -52,6 +52,7 @@ import org.apache.fluss.rpc.messages.PutKvResponse; import org.apache.fluss.rpc.protocol.Errors; import org.apache.fluss.server.entity.ProduceLogDataForBucket; +import org.apache.fluss.server.entity.PutKvDataForBucket; import org.apache.fluss.server.tablet.TestTabletServerGateway; import org.apache.fluss.types.DataTypes; import org.apache.fluss.utils.clock.SystemClock; @@ -91,6 +92,7 @@ import static org.apache.fluss.server.utils.ServerRpcMessageUtils.makeProduceLogResponse; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.makePutKvResponse; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.toProduceLogDataForBuckets; +import static org.apache.fluss.server.utils.ServerRpcMessageUtils.toPutKvDataForBuckets; import static org.apache.fluss.testutils.DataTestUtils.compactedRow; import static org.apache.fluss.testutils.DataTestUtils.row; import static org.apache.fluss.testutils.common.CommonTestUtils.retry; @@ -147,31 +149,114 @@ void testPotentialExpirationUsesAutoPartitionTimeUnit() { } @Test - void testFailsWriteAfterMetadataConfirmsPartitionMissing() throws Exception { + void testReroutesWriteAfterExplicitMissingPartitionResponse() throws Exception { sender.destroyResources(); TableInfo tableInfo = createHistoricalTableInfo(); PhysicalTablePath originalPath = PhysicalTablePath.of(tableInfo.getTablePath(), "20990101"); + PhysicalTablePath historicalPath = + PhysicalTablePath.of(tableInfo.getTablePath(), HISTORICAL_PARTITION_VALUE); TableBucket originalBucket = new TableBucket(tableInfo.getTableId(), 21L, 0); - metadataUpdater = missingPartitionMetadataUpdater(tableInfo); - metadataUpdater.updateCluster( - partitionedCluster( - tableInfo, Collections.singletonMap(originalPath, originalBucket))); - sender = setupWithIdempotenceState(); + TableBucket historicalBucket = new TableBucket(tableInfo.getTableId(), 22L, 0); + metadataUpdater = missingPartitionMetadataUpdater(tableInfo, originalPath); + Map tableBucketsByPath = new HashMap<>(); + tableBucketsByPath.put(originalPath, originalBucket); + tableBucketsByPath.put(historicalPath, historicalBucket); + metadataUpdater.updateCluster(partitionedCluster(tableInfo, tableBucketsByPath)); + IdempotenceManager idempotenceManager = createIdempotenceManager(true); + idempotenceManager.setWriterId(0L); + sender = setupWithIdempotenceState(idempotenceManager); CompletableFuture future = appendKvRecord(tableInfo, originalPath, 1, metadataUpdater.getCluster()); + // Send the first attempt to the original partition. sender.runOnce(); TestTabletServerGateway gateway = node1Gateway(); + // The explicit rejection re-enqueues the batch and invalidates the original metadata. gateway.response( 0, createPutKvResponse(originalBucket, Errors.UNKNOWN_TABLE_OR_BUCKET_EXCEPTION)); assertThat(future).isNotDone(); + // With no request left in flight, the missing original target can be safely rerouted. + sender.runOnce(); + assertThat(future).isNotDone(); + assertThat(idempotenceManager.inflightBatchSize(originalBucket)).isZero(); + + // The retry targets the historical bucket while preserving the original partition name. + sender.runOnce(); + PutKvRequest historicalRequest = (PutKvRequest) gateway.getRequest(0); + assertThat(historicalRequest.getBucketsReqsCount()).isOne(); + assertThat(historicalRequest.getBucketsReqAt(0).getPartitionId()) + .isEqualTo(historicalBucket.getPartitionId()); + assertThat(historicalRequest.getBucketsReqAt(0).getOriginalPartitionName()) + .isEqualTo(originalPath.getPartitionName()); + List historicalData = toPutKvDataForBuckets(historicalRequest); + assertThat(historicalData).hasSize(1); + assertThat(historicalData.get(0).records().writerId()).isEqualTo(0L); + assertThat(historicalData.get(0).records().batchSequence()).isZero(); + assertThat(idempotenceManager.inflightBatchSize(historicalBucket)).isOne(); + + gateway.response( + 0, + makePutKvResponse( + Collections.singletonList( + PutKvResultForBucket.historicalSuccess( + historicalBucket, 1L, originalPath.getPartitionName())))); + assertThat(future.get()).isNull(); + } + + @Test + void testAbortsOnlyMissingPartitionWhenRerouteIsUnsafe() throws Exception { + sender.destroyResources(); + TableInfo tableInfo = createHistoricalTableInfo(); + PhysicalTablePath missingPath = PhysicalTablePath.of(tableInfo.getTablePath(), "20990101"); + PhysicalTablePath activePath = PhysicalTablePath.of(tableInfo.getTablePath(), "20990102"); + PhysicalTablePath historicalPath = + PhysicalTablePath.of(tableInfo.getTablePath(), HISTORICAL_PARTITION_VALUE); + TableBucket missingBucket = new TableBucket(tableInfo.getTableId(), 21L, 0); + TableBucket historicalBucket = new TableBucket(tableInfo.getTableId(), 22L, 0); + TableBucket activeBucket = new TableBucket(tableInfo.getTableId(), 23L, 0); + metadataUpdater = missingPartitionMetadataUpdater(tableInfo, missingPath); + Map tableBucketsByPath = new HashMap<>(); + tableBucketsByPath.put(missingPath, missingBucket); + tableBucketsByPath.put(activePath, activeBucket); + tableBucketsByPath.put(historicalPath, historicalBucket); + metadataUpdater.updateCluster(partitionedCluster(tableInfo, tableBucketsByPath)); + sender = setupWithIdempotenceState(); + + CompletableFuture firstMissingFuture = + appendKvRecord(tableInfo, missingPath, 1, metadataUpdater.getCluster()); sender.runOnce(); - assertThat(future.get()) + // Keep a second request to the same original target in flight. + CompletableFuture secondMissingFuture = + appendKvRecord(tableInfo, missingPath, 2, metadataUpdater.getCluster()); + sender.runOnce(); + + TestTabletServerGateway gateway = node1Gateway(); + // Reject the first request while the second one is still awaiting a response. + gateway.response( + 0, createPutKvResponse(missingBucket, Errors.UNKNOWN_TABLE_OR_BUCKET_EXCEPTION)); + assertThat(firstMissingFuture).isNotDone(); + assertThat(secondMissingFuture).isNotDone(); + + // A write to another partition verifies that abort remains scoped to the missing target. + CompletableFuture activeFuture = + appendKvRecord(tableInfo, activePath, 3, metadataUpdater.getCluster()); + sender.runOnce(); + + assertThat(firstMissingFuture.get()) + .isInstanceOf(PartitionNotExistException.class) + .hasMessageContaining(missingPath.toString()); + assertThat(secondMissingFuture.get()) .isInstanceOf(PartitionNotExistException.class) - .hasMessageContaining(originalPath.toString()) - .hasCauseInstanceOf(PartitionNotExistException.class); + .hasMessageContaining(missingPath.toString()); + assertThat(activeFuture).isNotDone(); + + // Complete the request that was still in flight when its batches were aborted. + gateway.response( + 0, createPutKvResponse(missingBucket, Errors.UNKNOWN_TABLE_OR_BUCKET_EXCEPTION)); + gateway.response(0, createPutKvResponse(activeBucket, 1L)); + assertThat(activeFuture.get()).isNull(); } @Test @@ -1434,7 +1519,8 @@ private static TableInfo createHistoricalTableInfo( 1L); } - private static TestingMetadataUpdater missingPartitionMetadataUpdater(TableInfo tableInfo) { + private static TestingMetadataUpdater missingPartitionMetadataUpdater( + TableInfo tableInfo, PhysicalTablePath missingPath) { return new TestingMetadataUpdater( Collections.singletonMap(tableInfo.getTablePath(), tableInfo)) { @Override @@ -1444,7 +1530,10 @@ public void updatePhysicalTableMetadata(Set physicalTablePath @Override public boolean checkAndUpdatePartitionMetadata(PhysicalTablePath physicalTablePath) { - throw new PartitionNotExistException("Partition does not exist."); + if (physicalTablePath.equals(missingPath)) { + throw new PartitionNotExistException("Partition does not exist."); + } + return getCluster().getPartitionId(physicalTablePath).isPresent(); } }; } @@ -1478,6 +1567,7 @@ private static Cluster partitionedCluster( private CompletableFuture appendKvRecord( TableInfo tableInfo, PhysicalTablePath physicalTablePath, int id, Cluster cluster) throws Exception { + accumulator.checkAndCacheHistoricalPartitionEnabled(tableInfo); BinaryRow row = compactedRow( tableInfo.getRowType(), diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java index 94ad63a9524..d3157eeb798 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java @@ -439,14 +439,6 @@ public class ConfigOptions { .withDescription( "The duration after which an idle historical partition table lookuper is removed from the cache."); - public static final ConfigOption SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME = - key("server.historical-partition.kv-cleanup.idle-time") - .durationType() - .defaultValue(Duration.ofHours(3)) - .withDescription( - "The idle time after all local historical KV writes are tiered before the local state can be cleaned. " - + "Set to 0 to disable idle cleanup."); - public static final ConfigOption SERVER_DATA_DISK_WRITE_LIMIT_RATIO = key("server.data-disk.write-limit-ratio") .doubleType() diff --git a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeTableLookuper.java b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeTableLookuper.java index 386e15f4bfe..1389e294d48 100644 --- a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeTableLookuper.java +++ b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeTableLookuper.java @@ -61,12 +61,12 @@ interface LookupMetricRecorder { byte[] lookup(byte[] key, LookupContext context) throws Exception; /** - * Refreshes the registered lake data files while preserving reusable local lookup files. + * Requests that registered lake data files be refreshed before the next lookup. * - *

This method may be called concurrently with {@link #lookup(byte[], LookupContext)}. - * Implementations must ensure that it is thread-safe. + *

This method must not perform I/O and may be called concurrently with {@link + * #lookup(byte[], LookupContext)}. Implementations must ensure that it is thread-safe. */ - default void refresh() throws Exception { + default void refresh() { throw new UnsupportedOperationException( "Refreshing registered files is not supported by this lake table lookuper."); } diff --git a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/PluginLakeStorageWrapper.java b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/PluginLakeStorageWrapper.java index 7b3d95c0f16..7e2932dd168 100644 --- a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/PluginLakeStorageWrapper.java +++ b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/PluginLakeStorageWrapper.java @@ -167,7 +167,7 @@ private ClassLoaderFixingLakeTableLookuper(LakeTableLookuper inner, ClassLoader } @Override - public void refresh() throws Exception { + public void refresh() { try (TemporaryClassLoaderContext ignored = TemporaryClassLoaderContext.of(loader)) { inner.refresh(); } diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java index 10a5d06fad6..67a4c016af0 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java @@ -90,9 +90,9 @@ * lookup I/O failure refreshes that partition-bucket with the files from the latest snapshot and * retries once. * - *

An explicit refresh rescans every registered partition-bucket and updates its file set in - * place. Paimon keeps lookup files for data files that remain active and lazily downloads lookup - * files only for newly added data files. + *

An explicit refresh request is applied during the next lookup initialization. It rescans every + * registered partition-bucket and updates its file set in place. Paimon keeps lookup files for data + * files that remain active and lazily downloads lookup files only for newly added data files. * *

Calls to {@link LocalTableQuery#lookup} are serialized because Paimon 2.0 shares mutable * lookup-store comparator state across local lookup files. @@ -128,6 +128,8 @@ public class PaimonLakeTableLookuper implements LakeTableLookuper { private volatile @Nullable LocalTableQuery localTableQuery; // Guarded by lookupStateLock. private volatile boolean closed; + private volatile boolean refreshRequired; + private volatile boolean refreshInProgress; /** Creates a lookuper with the specified local lookup cache limit. */ public PaimonLakeTableLookuper( @@ -156,7 +158,7 @@ public PaimonLakeTableLookuper( checkNotNull(key, "key must not be null."); checkNotNull(context, "context must not be null."); checkNotClosed(); - ensureInitialized(context.valueRowType()); + initialize(context.valueRowType()); try (TrackingMetrics ignored = new TrackingMetrics(lookupFileDownloaded, context)) { return lookupInternal(key, context); @@ -172,19 +174,8 @@ public PaimonLakeTableLookuper( @Override public void refresh() { - synchronized (lookupStateLock) { - checkNotClosed(); - Map> filesBeforeRefresh = - new LinkedHashMap<>(registeredFiles); - Map> latestFiles = - scanDataFiles(filesBeforeRefresh.keySet()); - filesBeforeRefresh.forEach( - (partitionBucket, files) -> - refreshFiles( - partitionBucket, - files, - () -> latestFiles.get(partitionBucket))); - } + checkNotClosed(); + refreshRequired = true; } @Override @@ -213,17 +204,31 @@ private void checkNotClosed() { } } - private void ensureInitialized(RowType valueRowType) throws Exception { - if (localTableQuery == null) { + private void initialize(RowType valueRowType) throws Exception { + if (localTableQuery == null || refreshRequired || refreshInProgress) { synchronized (lookupStateLock) { if (localTableQuery == null) { - initialize(valueRowType); + initializeLookupState(valueRowType); + } + if (refreshRequired) { + // Clear the flag before refreshing so a concurrent request is retained for the + // next lookup. Restore it if this refresh fails. + refreshInProgress = true; + refreshRequired = false; + try { + refreshFilesFromLatestSnapshot(); + } catch (RuntimeException e) { + refreshRequired = true; + throw e; + } finally { + refreshInProgress = false; + } } } } } - private void initialize(RowType valueRowType) throws Exception { + private void initializeLookupState(RowType valueRowType) throws Exception { Catalog newCatalog = null; IOManager newIOManager = null; LocalTableQuery newLocalTableQuery = null; @@ -281,6 +286,17 @@ private void initialize(RowType valueRowType) throws Exception { } } + private void refreshFilesFromLatestSnapshot() { + Map> filesBeforeRefresh = + new LinkedHashMap<>(registeredFiles); + Map> latestFiles = + scanDataFiles(filesBeforeRefresh.keySet()); + filesBeforeRefresh.forEach( + (partitionBucket, files) -> + refreshFiles( + partitionBucket, files, () -> latestFiles.get(partitionBucket))); + } + private FileStoreTable withLookupCacheOptions(FileStoreTable table) { String key = CoreOptions.LOOKUP_CACHE_MAX_DISK_SIZE.key(); String maxDiskSize = new MemorySize(lookupCacheMaxDiskBytes).toString(); diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/client/ServerConnection.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/client/ServerConnection.java index f28cf1e9b76..1bc678452eb 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/client/ServerConnection.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/client/ServerConnection.java @@ -385,20 +385,18 @@ private void validateVersionCompatibility( } } - if (apiKey != ApiKeys.PUT_KV || version >= HISTORICAL_PUT_KV_MIN_VERSION) { - return; - } - - PutKvRequest putKvRequest = (PutKvRequest) rawRequest; - if (hasHistoricalPut(putKvRequest)) { - throw new UnsupportedVersionException( - "Historical partition writes require PUT_KV version " - + HISTORICAL_PUT_KV_MIN_VERSION - + " or newer, but server " - + node - + " negotiated version " - + version - + '.'); + if (apiKey == ApiKeys.PUT_KV && version < HISTORICAL_PUT_KV_MIN_VERSION) { + PutKvRequest putKvRequest = (PutKvRequest) rawRequest; + if (hasHistoricalPut(putKvRequest)) { + throw new UnsupportedVersionException( + "Historical partition writes require PUT_KV version " + + HISTORICAL_PUT_KV_MIN_VERSION + + " or newer, but server " + + node + + " negotiated version " + + version + + '.'); + } } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/DynamicServerConfig.java b/fluss-server/src/main/java/org/apache/fluss/server/DynamicServerConfig.java index 5f6e860dbe4..98d721e152e 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/DynamicServerConfig.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/DynamicServerConfig.java @@ -56,7 +56,6 @@ import static org.apache.fluss.config.ConfigOptions.REMOTE_DATA_DIRS_WEIGHTS; import static org.apache.fluss.config.ConfigOptions.SERVER_DATA_DISK_WRITE_LIMIT_RATIO; import static org.apache.fluss.config.ConfigOptions.SERVER_DATA_DISK_WRITE_RECOVER_RATIO; -import static org.apache.fluss.config.ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME; import static org.apache.fluss.config.ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUPER_CACHE_EXPIRE_AFTER_ACCESS; import static org.apache.fluss.config.ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO; import static org.apache.fluss.config.ConfigOptions.SERVER_SASL_CREDENTIALS; @@ -85,7 +84,6 @@ class DynamicServerConfig { KV_SNAPSHOT_INTERVAL.key(), SERVER_DATA_DISK_WRITE_RECOVER_RATIO.key(), SERVER_DATA_DISK_WRITE_LIMIT_RATIO.key(), - SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME.key(), SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO.key(), SERVER_HISTORICAL_PARTITION_LOOKUPER_CACHE_EXPIRE_AFTER_ACCESS.key(), // Config options for remote.data.dirs diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/HistoricalLookupCacheConfigValidator.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/HistoricalLookupCacheConfigValidator.java index 337b9542d45..c9d763f410a 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/HistoricalLookupCacheConfigValidator.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/HistoricalLookupCacheConfigValidator.java @@ -24,20 +24,11 @@ import java.time.Duration; -/** Validates dynamic historical partition settings used outside the coordinator. */ +/** Validates dynamic historical lookup cache settings. */ final class HistoricalLookupCacheConfigValidator implements ServerReconfigurable { @Override public void validate(Configuration newConfig) throws ConfigException { - Duration newCleanupIdleTime = - newConfig.get(ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME); - if (newCleanupIdleTime.isNegative()) { - throw new ConfigException( - String.format( - "Invalid configuration for %s, it must not be negative.", - ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME.key())); - } - double newMaxRatio = newConfig.get( ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java index 34f4cba9246..fd37583db2c 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java @@ -222,11 +222,6 @@ public final class Replica { private volatile @Nullable KvTablet kvTablet; private volatile @Nullable CloseableRegistry closeableRegistryForKv; private @Nullable PeriodicSnapshotManager kvSnapshotManager; - // The lake log end offset from which the current local historical KV state was rebuilt. - private volatile long historicalKvBaseOffset = -1L; - // Replaced whenever the local historical KV state is rebuilt so delayed cleanup tasks for an - // earlier state can be ignored. - private volatile @Nullable HistoricalKvCleanupState historicalKvCleanupState; /** * Server-wide {@link ScannerManager}. Active sessions for this bucket are closed in {@link @@ -385,53 +380,6 @@ public long getLakeLogEndOffset() { return logTablet.getLakeLogEndOffset(); } - /** Returns the cleanup state for the local historical KV state, or null if it is not ready. */ - public @Nullable HistoricalKvCleanupState getHistoricalKvCleanupState() { - return historicalKvCleanupState; - } - - /** - * Drops and recreates local historical KV state after its writes are fully tiered, provided - * leadership and offsets still match. - * - * @param expectedLeaderEpoch leader epoch required when cleanup runs - * @param tieredLogEndOffset fully tiered log end offset required when cleanup runs - * @param beforeCleanup action to run before the local KV state is dropped - * @return whether the local KV state was cleaned - */ - public boolean cleanupHistoricalKv( - int expectedLeaderEpoch, long tieredLogEndOffset, Runnable beforeCleanup) { - checkNotNull(beforeCleanup, "beforeCleanup must not be null"); - return inWriteLock( - leaderIsrUpdateLock, - () -> { - long localLogEndOffset = logTablet.localLogEndOffset(); - // Keep the scheduled snapshot and offset paired: newer lake progress may make - // the current lake and local offsets match while this task still references an - // older snapshot. - if (leaderEpoch != expectedLeaderEpoch - || localLogEndOffset != tieredLogEndOffset - || !isEligibleForHistoricalKvCleanup(localLogEndOffset)) { - return false; - } - - LOG.info( - "Cleaning local historical KV state for {} at log end offset {} " - + "covered by lake log end offset {}.", - tableBucket, - localLogEndOffset, - logTablet.getLakeLogEndOffset()); - // A lookup started after the empty local KV state is rebuilt must open a lake - // view that covers the data removed by this cleanup. - beforeCleanup.run(); - dropKv(); - // TODO: Retry rebuilding this historical bucket instead of waiting for - // failover or restart. - createKv(); - return true; - }); - } - public boolean isDataLakeEnabled() { return getTableConfig().isDataLakeEnabled(); } @@ -777,18 +725,6 @@ private void logTableConfigChanges(TableInfo oldTableInfo, TableInfo newTableInf } } - private boolean isEligibleForHistoricalKvCleanup(long localLogEndOffset) { - // Local KV state must have a known lake base, contain writes after that base, and have all - // those writes covered by lake before it can be discarded. - return isLeader() - && isHistoricalPartition() - && isKvTable() - && kvTablet != null - && historicalKvBaseOffset >= 0L - && historicalKvBaseOffset < localLogEndOffset - && logTablet.getLakeLogEndOffset() == localLogEndOffset; - } - private void createKv() { try { // create a closeable registry for the closable related to kv @@ -832,9 +768,7 @@ private void createKv() { tableBucket, INIT_KV_TABLET_MAX_RETRY_TIMES), lastError); } - if (isHistoricalPartition()) { - historicalKvCleanupState = new HistoricalKvCleanupState(clock.milliseconds()); - } else { + if (!isHistoricalPartition()) { startPeriodicKvSnapshot(snapshotUsed.orElse(null)); } } @@ -854,8 +788,6 @@ private void dropKv() { kvManager.dropKv(tableBucket); kvTablet = null; } - historicalKvCleanupState = null; - historicalKvBaseOffset = -1L; } private void mayFlushKv(long newHighWatermark) { @@ -977,9 +909,6 @@ private Optional initKvTablet() { logTablet.updateMinRetainOffset(restoreStartOffset); recoverKvTablet(restoreStartOffset, rowCount, autoIncIDRange); - if (isHistoricalPartition()) { - historicalKvBaseOffset = restoreStartOffset; - } } catch (Exception e) { throw new KvStorageException( String.format( @@ -1128,8 +1057,13 @@ private long historicalRecoveryStartOffset() { long lakeLogEndOffset = logTablet.getLakeLogEndOffset(); long localLogEndOffset = logTablet.localLogEndOffset(); long logStartOffset = logTablet.logStartOffset(); - long recoveryStartOffset = - lakeLogEndOffset >= 0 ? Math.min(lakeLogEndOffset, localLogEndOffset) : 0L; + checkState( + lakeLogEndOffset < 0 || lakeLogEndOffset <= localLogEndOffset, + "Cannot recover historical KV state: lake log end offset %s is beyond the " + + "local log end offset %s.", + lakeLogEndOffset, + localLogEndOffset); + long recoveryStartOffset = lakeLogEndOffset >= 0 ? lakeLogEndOffset : 0L; checkState( recoveryStartOffset >= logStartOffset, "Cannot recover historical KV state: recovery start offset %s is before the " @@ -1245,14 +1179,11 @@ public LogAppendInfo appendRecordsToLeader(MemoryLogRecords memoryLogRecords, in "Leader not local for bucket %s on tabletServer %d", tableBucket, localTabletServerId)); } - // Historical primary-key writes must go through PUT_KV so the server can - // preserve the original partition namespace and consult the lake on a local - // miss. Append-only records already contain their partition columns, so a log - // table can append them directly to its historical system partition. - if (isHistoricalPartition() && isKvTable()) { + // Primary-key writes must go through PUT_KV so the log and local KV state are + // updated together. PRODUCE_LOG is only valid for append-only tables. + if (isKvTable()) { throw new InvalidPartitionException( - "Produce-log request must not target the historical partition of " - + "a primary-key table."); + "Produce-log request must not target a primary-key table."); } validateInSyncReplicaSize(requiredAcks); @@ -2635,80 +2566,4 @@ public SchemaGetter getSchemaGetter() { public PeriodicSnapshotManager getKvSnapshotManager() { return kvSnapshotManager; } - - /** Tracks write activity and cleanup conditions for the current local historical KV state. */ - @ThreadSafe - public static final class HistoricalKvCleanupState { - // Latched when the live SST size reaches the maximum. Rebuilding the local KV state resets - // this flag, while transient RocksDB size changes do not resume writes prematurely. - private final AtomicBoolean maxSizeReached = new AtomicBoolean(); - - private volatile @Nullable CleanupCandidate cleanupCandidate; - private volatile long lastWriteMs; - - private HistoricalKvCleanupState(long lastWriteMs) { - this.lastWriteMs = lastWriteMs; - } - - /** Returns whether historical writes are paused by the maximum-size limit. */ - public boolean maxSizeReached() { - return maxSizeReached.get(); - } - - /** Latches the maximum-size limit and returns whether this call changed the state. */ - public boolean markMaxSizeReached() { - return maxSizeReached.compareAndSet(false, true); - } - - /** Updates the cleanup candidate for the current local historical KV state. */ - public void updateCleanupCandidate( - long lakeSnapshotId, int expectedLeaderEpoch, long tieredLogEndOffset) { - cleanupCandidate = - new CleanupCandidate(lakeSnapshotId, expectedLeaderEpoch, tieredLogEndOffset); - } - - /** Returns the cleanup candidate, or null if none is available. */ - public @Nullable CleanupCandidate cleanupCandidate() { - return cleanupCandidate; - } - - /** Records the latest historical write activity time. */ - public void recordWrite(long timestampMs) { - lastWriteMs = timestampMs; - } - - /** Returns the latest historical write activity time. */ - public long lastWriteMs() { - return lastWriteMs; - } - - /** A candidate for cleaning up local historical KV state. */ - public static final class CleanupCandidate { - private final long lakeSnapshotId; - private final int expectedLeaderEpoch; - private final long tieredLogEndOffset; - - private CleanupCandidate( - long lakeSnapshotId, int expectedLeaderEpoch, long tieredLogEndOffset) { - this.lakeSnapshotId = lakeSnapshotId; - this.expectedLeaderEpoch = expectedLeaderEpoch; - this.tieredLogEndOffset = tieredLogEndOffset; - } - - /** Returns the lake snapshot ID that covers the local KV state. */ - public long lakeSnapshotId() { - return lakeSnapshotId; - } - - /** Returns the leader epoch required to run cleanup. */ - public int expectedLeaderEpoch() { - return expectedLeaderEpoch; - } - - /** Returns the log end offset covered by the lake snapshot. */ - public long tieredLogEndOffset() { - return tieredLogEndOffset; - } - } - } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java index 37b47d3d4ed..327c086ba24 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java @@ -375,8 +375,7 @@ public ReplicaManager( localDiskManager, dataDir, dataDirVolumeBytes, - scheduler, - clock); + scheduler); registerMetrics(); } @@ -420,7 +419,6 @@ public int getCoordinatorEpoch() { public void validate(Configuration newConfig) throws ConfigException { // Type validation is already handled by DynamicServerConfig. // Here we only do basic sanity checks. - historicalPartitionManager.validate(newConfig); int newMinInSyncReplicas = newConfig.get(ConfigOptions.LOG_REPLICA_MIN_IN_SYNC_REPLICAS_NUMBER); if (newMinInSyncReplicas <= 0) { @@ -1395,8 +1393,7 @@ public void notifyLakeTableOffset( lakeBucketOffsets.entrySet()) { TableBucket tb = lakeBucketOffsetEntry.getKey(); LakeBucketOffset lakeBucketOffset = lakeBucketOffsetEntry.getValue(); - Replica replica = getReplicaOrException(tb); - LogTablet logTablet = replica.getLogTablet(); + LogTablet logTablet = getReplicaOrException(tb).getLogTablet(); logTablet.updateLakeTableSnapshotId(lakeBucketOffset.getSnapshotId()); lakeBucketOffset @@ -1405,19 +1402,7 @@ public void notifyLakeTableOffset( lakeBucketOffset .getLogEndOffset() - .ifPresent( - lakeLogEndOffset -> { - logTablet.updateLakeLogEndOffset(lakeLogEndOffset); - if (replica.isHistoricalPartition() - && replica.isKvTable()) { - // Only an explicit log-end-offset notification can - // make historical cleanup eligible. - historicalPartitionManager.onLakeProgress( - replica, - lakeBucketOffset.getSnapshotId(), - lakeLogEndOffset); - } - }); + .ifPresent(logTablet::updateLakeLogEndOffset); lakeBucketOffset .getMaxTimestamp() diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManager.java index ea5e93b2337..39d2f02fcc8 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManager.java @@ -22,7 +22,6 @@ import org.apache.fluss.config.Configuration; import org.apache.fluss.config.TableConfig; import org.apache.fluss.exception.FlussRuntimeException; -import org.apache.fluss.exception.KvStorageException; import org.apache.fluss.exception.LakeStorageNotConfiguredException; import org.apache.fluss.lake.lakestorage.LakeStorage; import org.apache.fluss.lake.lakestorage.LakeStoragePlugin; @@ -590,13 +589,7 @@ private synchronized void acquire(@Nullable Long requiredLakeSnapshotId) { throw new IllegalStateException("Lake table lookuper has been invalidated."); } if (!Objects.equals(lakeSnapshotId, requiredLakeSnapshotId)) { - try { - lookuper.refresh(); - } catch (Exception e) { - throw new KvStorageException( - "Failed to refresh historical lake lookup files for " + tablePath + ".", - e); - } + lookuper.refresh(); lakeSnapshotId = requiredLakeSnapshotId; } activeLookups++; diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java index b12f60c9500..fabb6685f9d 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java @@ -19,12 +19,9 @@ import org.apache.fluss.annotation.Internal; import org.apache.fluss.annotation.VisibleForTesting; -import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; -import org.apache.fluss.exception.ConfigException; import org.apache.fluss.exception.HistoricalPartitionThrottledException; import org.apache.fluss.exception.InvalidPartitionException; -import org.apache.fluss.exception.KvStorageException; import org.apache.fluss.lake.lakestorage.LakeTableLookuper; import org.apache.fluss.metadata.ResolvedPartitionSpec; import org.apache.fluss.metadata.TableBucket; @@ -43,20 +40,14 @@ import org.apache.fluss.server.kv.historical.HistoricalValueLookup; import org.apache.fluss.server.log.LogAppendInfo; import org.apache.fluss.server.replica.Replica; -import org.apache.fluss.server.replica.Replica.HistoricalKvCleanupState; import org.apache.fluss.server.storage.LocalDiskManager; import org.apache.fluss.utils.ByteArraySlice; import org.apache.fluss.utils.ByteArrayWrapper; -import org.apache.fluss.utils.clock.Clock; import org.apache.fluss.utils.concurrent.Scheduler; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import javax.annotation.Nullable; import java.io.File; -import java.time.Duration; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -65,24 +56,14 @@ import java.util.Map; import java.util.concurrent.CompletableFuture; -import static org.apache.fluss.utils.Preconditions.checkArgument; import static org.apache.fluss.utils.Preconditions.checkNotNull; /** Coordinates lookup, write, and lifecycle operations for historical partitions. */ @Internal public final class HistoricalPartitionManager implements AutoCloseable { - private static final Logger LOG = LoggerFactory.getLogger(HistoricalPartitionManager.class); - private static final long MAX_HISTORICAL_KV_SIZE_BYTES = 5L * 1024 * 1024 * 1024; - private final HistoricalPartitionTaskExecutor taskExecutor; private final HistoricalLakeLookupManager lakeLookupManager; - private final Clock clock; - private final @Nullable Scheduler cleanupScheduler; - private final long maxHistoricalKvSizeBytes; - - private volatile long cleanupIdleTimeMs; - private volatile boolean closed; /** Creates a historical partition manager from the tablet server dependencies. */ public HistoricalPartitionManager( @@ -91,10 +72,8 @@ public HistoricalPartitionManager( LocalDiskManager localDiskManager, File dataDir, long dataDirVolumeBytes, - Scheduler scheduler, - Clock clock) { + Scheduler scheduler) { this( - conf, new HistoricalPartitionTaskExecutor(conf), new HistoricalLakeLookupManager( conf, @@ -102,35 +81,16 @@ public HistoricalPartitionManager( localDiskManager, dataDir, dataDirVolumeBytes, - scheduler), - clock, - MAX_HISTORICAL_KV_SIZE_BYTES, - scheduler); + scheduler)); } @VisibleForTesting HistoricalPartitionManager( - Configuration conf, HistoricalPartitionTaskExecutor taskExecutor, - HistoricalLakeLookupManager lakeLookupManager, - Clock clock, - long maxHistoricalKvSizeBytes, - @Nullable Scheduler cleanupScheduler) { - Duration cleanupIdleTime = - conf.get(ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME); - checkArgument( - !cleanupIdleTime.isNegative(), - "%s must not be negative.", - ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME.key()); - checkArgument( - maxHistoricalKvSizeBytes > 0L, "maxHistoricalKvSizeBytes must be greater than 0."); + HistoricalLakeLookupManager lakeLookupManager) { this.taskExecutor = checkNotNull(taskExecutor, "taskExecutor must not be null"); this.lakeLookupManager = checkNotNull(lakeLookupManager, "lakeLookupManager must not be null"); - this.clock = checkNotNull(clock, "clock must not be null"); - this.cleanupScheduler = cleanupScheduler; - this.cleanupIdleTimeMs = cleanupIdleTime.toMillis(); - this.maxHistoricalKvSizeBytes = maxHistoricalKvSizeBytes; } /** Starts the resources used by historical partition operations. */ @@ -138,24 +98,6 @@ public void startup(Scheduler scheduler) { lakeLookupManager.startup(scheduler); } - /** Records new lake progress and schedules any cleanup that it makes eligible. */ - public void onLakeProgress(Replica replica, long lakeSnapshotId, long lakeLogEndOffset) { - if (!replica.isLeader() || !replica.isKvTable()) { - return; - } - int expectedLeaderEpoch = replica.getLeaderEpoch(); - long localLogEndOffset = replica.getLocalLogEndOffset(); - if (lakeLogEndOffset != localLogEndOffset) { - return; - } - HistoricalKvCleanupState cleanupState = replica.getHistoricalKvCleanupState(); - if (cleanupState == null) { - return; - } - cleanupState.updateCleanupCandidate(lakeSnapshotId, expectedLeaderEpoch, lakeLogEndOffset); - tryScheduleCleanup(replica, cleanupState); - } - /** Looks up historical keys from local KV state and then lake storage. */ public CompletableFuture lookup( Replica replica, @@ -200,26 +142,6 @@ public CompletableFuture putKv( checkNotNull( putData.originalPartitionName(), "originalPartitionName must not be null"); - HistoricalKvCleanupState cleanupState = replica.getHistoricalKvCleanupState(); - if (cleanupState == null) { - throw new KvStorageException( - "Local historical KV state is not ready for " - + replica.getTableBucket() - + " because its KV tablet is being initialized or rebuilt."); - } - if (cleanupState.maxSizeReached()) { - return CompletableFuture.completedFuture( - maxSizeThrottledResult( - putData, originalPartitionName, maxHistoricalKvSizeBytes)); - } - long liveSstSize = replica.logicalStorageKvSize(); - if (liveSstSize >= maxHistoricalKvSizeBytes) { - markMaxSizeReached(replica, cleanupState); - return CompletableFuture.completedFuture( - maxSizeThrottledResult( - putData, originalPartitionName, maxHistoricalKvSizeBytes)); - } - cleanupState.recordWrite(clock.milliseconds()); return taskExecutor.submitOrdered( putData.tableBucket(), () -> { @@ -252,33 +174,9 @@ public CompletableFuture putKv( } } - /** Validates dynamic historical partition configuration changes. */ - public void validate(Configuration newConf) throws ConfigException { - Duration newCleanupIdleTime = - newConf.get(ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME); - if (newCleanupIdleTime.isNegative()) { - throw new ConfigException( - String.format( - "Invalid configuration for %s, it must not be negative.", - ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME.key())); - } - } - - /** Applies dynamic historical partition configuration changes. */ + /** Applies dynamic historical lookup configuration changes. */ public void reconfigure(Configuration newConf) { lakeLookupManager.reconfigure(newConf); - long newCleanupIdleTimeMs = - newConf.get(ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME) - .toMillis(); - if (newCleanupIdleTimeMs == cleanupIdleTimeMs) { - return; - } - long oldCleanupIdleTimeMs = cleanupIdleTimeMs; - cleanupIdleTimeMs = newCleanupIdleTimeMs; - LOG.info( - "Historical KV cleanup idle time reconfigured: {} ms -> {} ms.", - oldCleanupIdleTimeMs, - newCleanupIdleTimeMs); } /** Invalidates the cached lake lookuper for the given table. */ @@ -369,6 +267,8 @@ LogAppendInfo processPut( return result.value(); }; + // TODO: Tag historical values and tombstones with WAL offsets for incremental cleanup; see + // https://github.com/apache/fluss/issues/4159. return replica.putHistoricalRecordsToLeader( putData.records(), targetColumns, @@ -379,158 +279,6 @@ LogAppendInfo processPut( requiredAcks); } - private void markMaxSizeReached(Replica replica, HistoricalKvCleanupState cleanupState) { - if (cleanupState.markMaxSizeReached()) { - LOG.warn( - "Pausing historical writes for {} because its live SST size reached the " - + "maximum size {} bytes.", - replica.getTableBucket(), - maxHistoricalKvSizeBytes); - - tryScheduleCleanup(replica, cleanupState); - } - } - - private void tryScheduleCleanup(Replica replica, HistoricalKvCleanupState cleanupState) { - HistoricalKvCleanupState.CleanupCandidate candidate = cleanupState.cleanupCandidate(); - if (candidate == null) { - return; - } - scheduleCleanup( - replica, - cleanupState, - cleanupState.maxSizeReached(), - candidate.lakeSnapshotId(), - candidate.expectedLeaderEpoch(), - candidate.tieredLogEndOffset()); - } - - private void scheduleCleanup( - Replica replica, - HistoricalKvCleanupState cleanupState, - boolean maxSizeReached, - long lakeSnapshotId, - int expectedLeaderEpoch, - long tieredLogEndOffset) { - // Reject cleanup if the replica no longer matches the cleanup state, leader epoch, or - // tiered log end offset used when it was scheduled. - if (closed - || replica.getHistoricalKvCleanupState() != cleanupState - || expectedLeaderEpoch != replica.getLeaderEpoch() - || replica.getLocalLogEndOffset() != tieredLogEndOffset) { - return; - } - - if (!maxSizeReached - && deferIdleCleanupIfNeeded( - replica, - cleanupState, - lakeSnapshotId, - expectedLeaderEpoch, - tieredLogEndOffset)) { - return; - } - - CompletableFuture cleanupFuture; - cleanupFuture = - taskExecutor.submitOrderedMaintenance( - replica.getTableBucket(), - () -> - runCleanup( - replica, - cleanupState, - maxSizeReached, - lakeSnapshotId, - expectedLeaderEpoch, - tieredLogEndOffset)); - cleanupFuture.whenComplete( - (ignored, error) -> { - if (error != null) { - LOG.error( - "Historical KV cleanup failed for {}.", - replica.getTableBucket(), - error); - } - }); - } - - private void runCleanup( - Replica replica, - HistoricalKvCleanupState cleanupState, - boolean maxSizeReached, - long lakeSnapshotId, - int expectedLeaderEpoch, - long tieredLogEndOffset) { - if (replica.getHistoricalKvCleanupState() != cleanupState - || expectedLeaderEpoch != replica.getLeaderEpoch()) { - return; - } - if (!maxSizeReached - && deferIdleCleanupIfNeeded( - replica, - cleanupState, - lakeSnapshotId, - expectedLeaderEpoch, - tieredLogEndOffset)) { - return; - } - - if (replica.cleanupHistoricalKv( - expectedLeaderEpoch, - tieredLogEndOffset, - () -> requireLakeSnapshot(replica.getTableBucket().getTableId(), lakeSnapshotId))) { - LOG.info( - "Cleaned {} local historical KV state for {}.", - maxSizeReached ? "max-size-triggered" : "idle-triggered", - replica.getTableBucket()); - } - } - - /** - * Returns whether idle cleanup must stop now. If the idle window has not elapsed, schedules the - * next check at its deadline. - */ - private boolean deferIdleCleanupIfNeeded( - Replica replica, - HistoricalKvCleanupState cleanupState, - long lakeSnapshotId, - int expectedLeaderEpoch, - long tieredLogEndOffset) { - long idleTimeMs = cleanupIdleTimeMs; - if (idleTimeMs <= 0L) { - return true; - } - long delayMs = remainingIdleCleanupDelayMs(cleanupState, idleTimeMs); - if (delayMs <= 0L) { - return false; - } - checkNotNull(cleanupScheduler, "cleanupScheduler must not be null") - .scheduleOnce( - "historical-kv-idle-cleanup-" + replica.getTableBucket(), - () -> - scheduleCleanup( - replica, - cleanupState, - false, - lakeSnapshotId, - expectedLeaderEpoch, - tieredLogEndOffset), - delayMs); - return true; - } - - /** Returns the remaining delay before idle cleanup is eligible, or {@code 0} if it is due. */ - private long remainingIdleCleanupDelayMs( - HistoricalKvCleanupState cleanupState, long idleTimeMs) { - long now = clock.milliseconds(); - long lastWriteMs = cleanupState.lastWriteMs(); - if (now < lastWriteMs) { - // A backward clock jump restarts the idle window instead of cleaning prematurely. - return idleTimeMs; - } - return Math.max(0L, idleTimeMs - (now - lastWriteMs)); - } - private static PutKvResultForBucket requestLimitThrottledResult( PutKvDataForBucket putData, String originalPartitionName) { return PutKvResultForBucket.historicalFailure( @@ -545,28 +293,8 @@ private static PutKvResultForBucket requestLimitThrottledResult( originalPartitionName); } - private static PutKvResultForBucket maxSizeThrottledResult( - PutKvDataForBucket putData, String originalPartitionName, long maxHistoricalKvSize) { - return PutKvResultForBucket.historicalFailure( - putData.tableBucket(), - ApiError.fromThrowable( - new HistoricalPartitionThrottledException( - "Historical write is throttled for " - + putData.tableBucket() - + " (original partition " - + originalPartitionName - + ") because its local historical KV state reached the live " - + "SST maximum size of " - + maxHistoricalKvSize - + " bytes. New writes are paused until lake tiering " - + "covers all previously accepted writes and cleanup of " - + "the local historical KV state completes.")), - originalPartitionName); - } - @Override public void close() { - closed = true; taskExecutor.close(); lakeLookupManager.close(); } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionTaskExecutor.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionTaskExecutor.java index 763f2cd1807..9bf66848fad 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionTaskExecutor.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionTaskExecutor.java @@ -141,47 +141,24 @@ public CompletableFuture submitOrdered( return CompletableFuture.completedFuture(throttledResult.get()); } + CompletableFuture future; + CompletableFuture tail; try { - return enqueueOrdered(orderingKey, task, true); + synchronized (orderedTasksLock) { + CompletableFuture previousTail = orderedTaskTails.get(orderingKey); + if (previousTail == null) { + future = CompletableFuture.supplyAsync(task, executor); + } else { + future = previousTail.thenApplyAsync(ignored -> task.get(), executor); + } + // Convert success or failure into a normal completion used only for sequencing. + tail = future.handle((ignored, error) -> null); + orderedTaskTails.put(orderingKey, tail); + } } catch (RuntimeException e) { requestPermits.release(); throw e; } - } - - /** - * Submits an internal maintenance task after all accepted tasks with the same ordering key. - * - *

Maintenance work is not a client request and therefore does not consume a request permit. - */ - public CompletableFuture submitOrderedMaintenance( - Object orderingKey, Runnable maintenanceTask) { - checkNotNull(orderingKey, "orderingKey must not be null."); - checkNotNull(maintenanceTask, "maintenanceTask must not be null."); - return enqueueOrdered( - orderingKey, - () -> { - maintenanceTask.run(); - return null; - }, - false); - } - - private CompletableFuture enqueueOrdered( - Object orderingKey, Supplier task, boolean releaseRequestPermit) { - CompletableFuture future; - CompletableFuture tail; - synchronized (orderedTasksLock) { - CompletableFuture previousTail = orderedTaskTails.get(orderingKey); - if (previousTail == null) { - future = CompletableFuture.supplyAsync(task, executor); - } else { - future = previousTail.thenApplyAsync(ignored -> task.get(), executor); - } - // Convert success or failure into a normal completion used only for sequencing. - tail = future.handle((ignored, error) -> null); - orderedTaskTails.put(orderingKey, tail); - } CompletableFuture currentTail = tail; tail.whenComplete( @@ -190,24 +167,17 @@ private CompletableFuture enqueueOrdered( orderedTaskTails.remove(orderingKey, currentTail); } }); - return trackAcceptedRequest(future, releaseRequestPermit); + return trackAcceptedRequest(future); } private CompletableFuture trackAcceptedRequest(CompletableFuture future) { - return trackAcceptedRequest(future, true); - } - - private CompletableFuture trackAcceptedRequest( - CompletableFuture future, boolean releaseRequestPermit) { pendingRequests.add(future); future.whenComplete( (ignored, error) -> { + // Release the permit exactly once when the accepted task reaches a terminal + // state, including exceptional completion and cancellation. pendingRequests.remove(future); - if (releaseRequestPermit) { - // Release the permit exactly once when the accepted request reaches a - // terminal state, including exceptional completion and cancellation. - requestPermits.release(); - } + requestPermits.release(); }); return future; } diff --git a/fluss-server/src/test/java/org/apache/fluss/server/DynamicConfigChangeTest.java b/fluss-server/src/test/java/org/apache/fluss/server/DynamicConfigChangeTest.java index 6beccb61285..5bc4a1fe8ce 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/DynamicConfigChangeTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/DynamicConfigChangeTest.java @@ -191,17 +191,6 @@ void testAlterLakehouseConfigs() throws Exception { } } - @Test - void testAllowsHistoricalKvCleanupIdleTimeToChangeDynamically() { - assertThat( - new DynamicServerConfig(new Configuration()) - .isAllowedConfig( - ConfigOptions - .SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME - .key())) - .isTrue(); - } - @Test void testOverrideConfigs() throws Exception { Configuration configuration = new Configuration(); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/HighWatermarkPersistenceTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/HighWatermarkPersistenceTest.java index a520f9dad2e..ddd0574be5d 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/HighWatermarkPersistenceTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/HighWatermarkPersistenceTest.java @@ -18,25 +18,18 @@ package org.apache.fluss.server.replica; import org.apache.fluss.config.ConfigOptions; -import org.apache.fluss.metadata.PhysicalTablePath; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.server.coordinator.TestCoordinatorGateway; -import org.apache.fluss.server.entity.NotifyLeaderAndIsrData; import org.apache.fluss.server.log.checkpoint.OffsetCheckpointFile; -import org.apache.fluss.server.zk.data.LeaderAndIsr; import org.junit.jupiter.api.Test; import java.io.File; import java.time.Duration; -import java.util.Collections; import static org.apache.fluss.record.TestData.ANOTHER_DATA1; import static org.apache.fluss.record.TestData.DATA1; import static org.apache.fluss.record.TestData.DATA1_TABLE_ID; -import static org.apache.fluss.record.TestData.DATA2_TABLE_ID; -import static org.apache.fluss.record.TestData.DATA2_TABLE_PATH; -import static org.apache.fluss.server.coordinator.CoordinatorContext.INITIAL_COORDINATOR_EPOCH; import static org.apache.fluss.server.replica.ReplicaManager.HIGH_WATERMARK_CHECKPOINT_FILE_NAME; import static org.apache.fluss.testutils.DataTestUtils.genMemoryLogRecordsByObject; import static org.apache.fluss.testutils.common.CommonTestUtils.retry; @@ -92,23 +85,9 @@ void testHighWatermarkPersistenceMultipleReplicas() throws Exception { assertThat(highWatermark0).isEqualTo(10L); assertThat(replica0.getLogTablet().getHighWatermark()).isEqualTo(10L); - // add another replica and set highWatermark. - TableBucket tableBucket1 = new TableBucket(DATA2_TABLE_ID, 0); - replicaManager.becomeLeaderOrFollower( - INITIAL_COORDINATOR_EPOCH, - Collections.singletonList( - new NotifyLeaderAndIsrData( - PhysicalTablePath.of(DATA2_TABLE_PATH), - tableBucket1, - Collections.singletonList(TABLET_SERVER_ID), - new LeaderAndIsr( - TABLET_SERVER_ID, - LeaderAndIsr.INITIAL_LEADER_EPOCH, - Collections.singletonList(TABLET_SERVER_ID), - Collections.emptyList(), - INITIAL_COORDINATOR_EPOCH, - LeaderAndIsr.INITIAL_BUCKET_EPOCH))), - result -> {}); + // Add another append-only replica and set its high watermark. + TableBucket tableBucket1 = new TableBucket(DATA1_TABLE_ID, 1); + makeLogTableAsLeader(tableBucket1.getBucket()); replicaManager.checkpointHighWatermarks(); long highWatermark1 = highWatermarkFor(tableBucket1); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java index 526623b050e..b18f96b3a7a 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java @@ -83,7 +83,6 @@ import com.github.benmanes.caffeine.cache.Scheduler; import com.github.benmanes.caffeine.cache.Ticker; import org.junit.jupiter.api.Test; -import org.rocksdb.FlushOptions; import javax.annotation.Nullable; @@ -96,7 +95,6 @@ import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -125,10 +123,6 @@ class HistoricalPartitionManagerTest extends ReplicaTestBase { private static final String ANOTHER_ORIGINAL_PARTITION = "20240108"; private static final String HISTORICAL_PARTITION = HISTORICAL_PARTITION_VALUE; private static final TableBucket TABLE_BUCKET = new TableBucket(TABLE_ID, PARTITION_ID, 0); - private static final RowType HISTORICAL_KEY_TYPE = - DataTypes.ROW( - new DataField("id", DataTypes.INT()), - new DataField("region", DataTypes.STRING())); @Test void testResolvesMultipleLakeMissesWithoutPrewriteRollback() throws Exception { @@ -139,7 +133,7 @@ void testResolvesMultipleLakeMissesWithoutPrewriteRollback() throws Exception { TestingHistoricalLakeLookupManager lakeLookupManager = new TestingHistoricalLakeLookupManager(lookupConfiguration()); HistoricalPartitionManager historicalPartitionManager = - createNonCleanupManager( + new HistoricalPartitionManager( new HistoricalPartitionTaskExecutor(lookupConfiguration()), lakeLookupManager); @@ -207,7 +201,7 @@ void testWalFullRowUpsertDoesNotLookupLake() throws Exception { TestingHistoricalLakeLookupManager lakeLookupManager = new TestingHistoricalLakeLookupManager(lookupConfiguration()); HistoricalPartitionManager historicalPartitionManager = - createNonCleanupManager( + new HistoricalPartitionManager( new HistoricalPartitionTaskExecutor(lookupConfiguration()), lakeLookupManager); @@ -256,7 +250,7 @@ void testHistoricalInsertUpdateAndDelete() throws Exception { TestingHistoricalLakeLookupManager lakeLookupManager = new TestingHistoricalLakeLookupManager(lookupConfiguration()); HistoricalPartitionManager historicalPartitionManager = - createNonCleanupManager( + new HistoricalPartitionManager( new HistoricalPartitionTaskExecutor(lookupConfiguration()), lakeLookupManager); @@ -268,7 +262,7 @@ void testHistoricalInsertUpdateAndDelete() throws Exception { byte[] primaryKey = new CompactedKeyEncoder(keyType).encodeKey(row(1, "us")); try { - // The first write misses both local KV state and lake, so it creates a local value. + // The first write misses both local state and lake, so it creates a local overlay. KvRecordBatch insertBatch = batch( keyType, @@ -327,7 +321,7 @@ void testHistoricalInsertUpdateAndDelete() throws Exception { row(1, "us", "20240108", "another")); assertThat(lakeLookupManager.lookupCount).hasValue(2); - // Exercise the ReplicaManager entry point; the update should reuse local KV state. + // Exercise the ReplicaManager entry point; the update should reuse the local overlay. KvRecordBatch updateBatch = batch( keyType, @@ -371,7 +365,7 @@ void testHistoricalInsertUpdateAndDelete() throws Exception { row(1, "us", "20240108", "another")); assertThat(lakeLookupManager.lookupCount).hasValue(2); - // Historical lookup should observe the updated value from local KV state. + // Historical lookup should observe the updated value from the local overlay. CompletableFuture> lookupResponse = new CompletableFuture<>(); replicaManager.historicalLookups( @@ -446,7 +440,7 @@ void testUpdateAndDeleteFromLakeFallback() throws Exception { TestingHistoricalLakeLookupManager lakeLookupManager = new TestingHistoricalLakeLookupManager(lookupConfiguration()); HistoricalPartitionManager historicalPartitionManager = - createNonCleanupManager( + new HistoricalPartitionManager( new HistoricalPartitionTaskExecutor(lookupConfiguration()), lakeLookupManager); @@ -547,7 +541,7 @@ void testLeaderChangeDuringLakeLookupFencesHistoricalWrite() throws Exception { TestingHistoricalLakeLookupManager lakeLookupManager = new TestingHistoricalLakeLookupManager(lookupConfiguration()); HistoricalPartitionManager historicalPartitionManager = - createNonCleanupManager( + new HistoricalPartitionManager( new HistoricalPartitionTaskExecutor(lookupConfiguration()), lakeLookupManager); CountDownLatch lakeLookupStarted = new CountDownLatch(1); @@ -598,13 +592,13 @@ void testLeaderChangeDuringLakeLookupFencesHistoricalWrite() throws Exception { } @Test - void testRecoversLocalHistoricalKvFromLakeCommitOffset() throws Exception { + void testRecoversHistoricalOverlayFromLakeCommitOffset() throws Exception { TableInfo tableInfo = registerHistoricalTableAndBecomeLeader(); Replica replica = replicaManager.getReplicaOrException(TABLE_BUCKET); TestingHistoricalLakeLookupManager lakeLookupManager = new TestingHistoricalLakeLookupManager(lookupConfiguration()); HistoricalPartitionManager historicalPartitionManager = - createNonCleanupManager( + new HistoricalPartitionManager( new HistoricalPartitionTaskExecutor(lookupConfiguration()), lakeLookupManager); @@ -685,7 +679,7 @@ void testRecoversLocalHistoricalKvFromLakeCommitOffset() throws Exception { // Persist the exclusive end offset of the first write as the lake recovery point. The // replica has not received this offset locally, so becoming leader must load it before - // creating the local historical KV state. + // creating the historical overlay. long lakeCommitOffset = firstAppend.lastOffset() + 1; new LakeTableHelper(zkClient, DEFAULT_REMOTE_DATA_DIR) .registerLakeTableSnapshotV1( @@ -694,8 +688,8 @@ void testRecoversLocalHistoricalKvFromLakeCommitOffset() throws Exception { 1L, Collections.singletonMap(TABLE_BUCKET, lakeCommitOffset))); assertThat(replica.getLakeLogEndOffset()).isEqualTo(-1L); - // Dropping and recreating the leader KV tablet rebuilds its state only from WAL after - // the lake commit offset. The recovered tombstone must remain + // Dropping and recreating the leader KV tablet forces the overlay to be rebuilt only + // from WAL after the lake commit offset. The recovered tombstone must remain // authoritative over lake fallback. assertThat(replica.makeFollower(followerState())).isTrue(); CompletableFuture> leaderFuture = @@ -734,296 +728,6 @@ void testRecoversLocalHistoricalKvFromLakeCommitOffset() throws Exception { } } - @Test - void testCleansLocalHistoricalKvAfterTieringAndWriteIdleTime() throws Exception { - TableInfo tableInfo = registerHistoricalTableAndBecomeLeader(); - Replica replica = replicaManager.getReplicaOrException(TABLE_BUCKET); - KvTablet originalKvTablet = replica.getKvTablet(); - assertThat(originalKvTablet).isNotNull(); - - Configuration cleanupConf = lookupConfiguration(); - cleanupConf.set( - ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME, - Duration.ofMinutes(1)); - ManuallyTriggeredScheduledExecutorService executor = - new ManuallyTriggeredScheduledExecutorService(); - TestingHistoricalLakeLookupManager lakeLookupManager = - new TestingHistoricalLakeLookupManager(cleanupConf); - HistoricalPartitionManager historicalPartitionManager = - createCleanupManager(cleanupConf, executor, lakeLookupManager, Long.MAX_VALUE); - - byte[] primaryKey = new CompactedKeyEncoder(HISTORICAL_KEY_TYPE).encodeKey(row(1, "us")); - Object[] valueObjects = new Object[] {1, "us", ORIGINAL_PARTITION, "v1"}; - byte[] lakeValue = - ValueEncoder.encodeValue( - (short) tableInfo.getSchemaId(), - compactedRow(tableInfo.getRowType(), valueObjects)); - lakeLookupManager.putLakeValue(ORIGINAL_PARTITION, lakeValue); - - try { - CompletableFuture putFuture = - putHistoricalRecords( - historicalPartitionManager, - replica, - batch( - HISTORICAL_KEY_TYPE, - tableInfo.getRowType(), - Tuple2.of(new Object[] {1, "us"}, valueObjects))); - executor.triggerAll(); - assertThat(putFuture.get(10, TimeUnit.SECONDS).failed()).isFalse(); - flushAndWait(originalKvTablet, Long.MAX_VALUE); - - long tieredOffset = replica.getLocalLogEndOffset(); - historicalPartitionManager.onLakeProgress(replica, 9L, tieredOffset - 1); - assertThat(executor.numQueuedRunnables()).isZero(); - - replica.getLogTablet().updateLakeLogEndOffset(tieredOffset); - historicalPartitionManager.onLakeProgress(replica, 10L, tieredOffset); - // Lake normally catches up before local KV state becomes idle. Cleanup must wake at - // the write deadline even if no further lake progress notification arrives. - assertThat(executor.numQueuedRunnables()).isZero(); - assertThat(executor.getActiveNonPeriodicScheduledTask()).hasSize(1); - assertThat(replica.getKvTablet()).isSameAs(originalKvTablet); - manualClock.advanceTime(Duration.ofMinutes(1)); - executor.triggerNonPeriodicScheduledTasks(); - assertThat(executor.numQueuedRunnables()).isOne(); - assertThat(lakeLookupManager.requiredLakeSnapshotIds).isEmpty(); - executor.triggerAll(); - - KvTablet cleanedKvTablet = replica.getKvTablet(); - assertThat(cleanedKvTablet).isNotNull().isNotSameAs(originalKvTablet); - assertThat(lakeLookupManager.requiredLakeSnapshotIds).containsExactly(10L); - assertThat(cleanedKvTablet.getRocksDBKv().limitScan(10)).isEmpty(); - assertThat(cleanedKvTablet.lookupHistoricalLocal(ORIGINAL_PARTITION, primaryKey)) - .isEqualTo(KvStateLookupResult.notFound()); - - CompletableFuture lookupFuture = - historicalPartitionManager.lookup( - replica, - new LookupDataForBucket( - TABLE_BUCKET, - Collections.singletonList(primaryKey), - ORIGINAL_PARTITION), - (lookupTimeNanos, lookupFileDownloaded) -> {}); - executor.triggerAll(); - assertThat(lookupFuture.get(10, TimeUnit.SECONDS).lookupValues()) - .extracting(ByteArraySlice::toByteArray) - .containsExactly(lakeValue); - } finally { - historicalPartitionManager.close(); - } - } - - @Test - void testCleanupRequiresLakeAndLocalOffsetsToMatch() throws Exception { - TableInfo tableInfo = registerHistoricalTableAndBecomeLeader(); - Replica replica = replicaManager.getReplicaOrException(TABLE_BUCKET); - KvTablet originalKvTablet = replica.getKvTablet(); - assertThat(originalKvTablet).isNotNull(); - - Configuration cleanupConf = lookupConfiguration(); - cleanupConf.set( - ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME, - Duration.ofMinutes(1)); - ManuallyTriggeredScheduledExecutorService executor = - new ManuallyTriggeredScheduledExecutorService(); - TestingHistoricalLakeLookupManager lakeLookupManager = - new TestingHistoricalLakeLookupManager(cleanupConf); - HistoricalPartitionManager historicalPartitionManager = - createCleanupManager(cleanupConf, executor, lakeLookupManager, Long.MAX_VALUE); - - KvRecordBatch records = - batch( - HISTORICAL_KEY_TYPE, - tableInfo.getRowType(), - Tuple2.of( - new Object[] {1, "us"}, - new Object[] {1, "us", ORIGINAL_PARTITION, "v1"})); - - try { - CompletableFuture firstPut = - putHistoricalRecords(historicalPartitionManager, replica, records); - executor.triggerAll(); - assertThat(firstPut.get(10, TimeUnit.SECONDS).failed()).isFalse(); - - long firstTieredOffset = replica.getLocalLogEndOffset(); - replica.getLogTablet().updateLakeLogEndOffset(firstTieredOffset); - historicalPartitionManager.onLakeProgress(replica, 10L, firstTieredOffset); - executor.triggerAll(); - assertThat(replica.getKvTablet()).isSameAs(originalKvTablet); - assertThat(lakeLookupManager.requiredLakeSnapshotIds).isEmpty(); - - CompletableFuture secondPut = - putHistoricalRecords(historicalPartitionManager, replica, records); - manualClock.advanceTime(Duration.ofMinutes(1)); - historicalPartitionManager.onLakeProgress(replica, 10L, firstTieredOffset); - - // The second write runs before the cleanup that captured snapshot 10 / first offset. - executor.trigger(); - assertThat(secondPut.get(10, TimeUnit.SECONDS).failed()).isFalse(); - long secondTieredOffset = replica.getLocalLogEndOffset(); - assertThat(secondTieredOffset).isGreaterThan(firstTieredOffset); - - manualClock.advanceTime(Duration.ofMinutes(1)); - // A lake offset beyond the local end is inconsistent and must not schedule cleanup. - historicalPartitionManager.onLakeProgress(replica, 11L, secondTieredOffset + 1); - executor.trigger(); - - // Snapshot 10 does not cover the second write, so its queued cleanup must be skipped. - assertThat(replica.getKvTablet()).isSameAs(originalKvTablet); - assertThat(lakeLookupManager.requiredLakeSnapshotIds).isEmpty(); - assertThat(executor.numQueuedRunnables()).isZero(); - - replica.getLogTablet().updateLakeLogEndOffset(secondTieredOffset); - historicalPartitionManager.onLakeProgress(replica, 12L, secondTieredOffset); - executor.triggerAll(); - assertThat(replica.getKvTablet()).isNotNull().isNotSameAs(originalKvTablet); - assertThat(lakeLookupManager.requiredLakeSnapshotIds).containsExactly(12L); - } finally { - historicalPartitionManager.close(); - } - } - - @Test - void testMaxSizeBlocksWritesUntilAcceptedWritesAreTieredAndLocalKvIsCleaned() throws Exception { - TableInfo tableInfo = registerHistoricalTableAndBecomeLeader(); - Replica replica = replicaManager.getReplicaOrException(TABLE_BUCKET); - KvTablet originalKvTablet = replica.getKvTablet(); - assertThat(originalKvTablet).isNotNull(); - - Configuration cleanupConf = lookupConfiguration(); - cleanupConf.set( - ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME, Duration.ZERO); - ManuallyTriggeredScheduledExecutorService executor = - new ManuallyTriggeredScheduledExecutorService(); - HistoricalPartitionManager historicalPartitionManager = - createCleanupManager( - cleanupConf, - executor, - new TestingHistoricalLakeLookupManager(cleanupConf), - 1L); - - KvRecordBatch firstBatch = - batch( - HISTORICAL_KEY_TYPE, - tableInfo.getRowType(), - Tuple2.of( - new Object[] {1, "us"}, - new Object[] {1, "us", ORIGINAL_PARTITION, "v1"})); - KvRecordBatch secondBatch = - batch( - HISTORICAL_KEY_TYPE, - tableInfo.getRowType(), - Tuple2.of( - new Object[] {2, "eu"}, - new Object[] {2, "eu", ORIGINAL_PARTITION, "v2"})); - - try { - CompletableFuture firstPut = - putHistoricalRecords(historicalPartitionManager, replica, firstBatch); - executor.triggerAll(); - assertThat(firstPut.get(10, TimeUnit.SECONDS).failed()).isFalse(); - flushAndWait(originalKvTablet, Long.MAX_VALUE); - try (FlushOptions flushOptions = new FlushOptions().setWaitForFlush(true)) { - originalKvTablet.getRocksDBKv().getDb().flush(flushOptions); - } - assertThat(originalKvTablet.liveSstFilesSize()).isPositive(); - - PutKvResultForBucket blockedWrite = - putHistoricalRecords(historicalPartitionManager, replica, secondBatch) - .get(10, TimeUnit.SECONDS); - assertThat(blockedWrite.getError().error()) - .isEqualTo(Errors.HISTORICAL_PARTITION_THROTTLED); - assertThat(blockedWrite.getError().message()) - .contains( - "reached the live SST maximum size of 1 bytes", - "lake tiering covers all previously accepted writes"); - assertThat(executor.numQueuedRunnables()).isZero(); - - long tieredOffset = replica.getLocalLogEndOffset(); - replica.getLogTablet().updateLakeLogEndOffset(tieredOffset); - historicalPartitionManager.onLakeProgress(replica, 11L, tieredOffset); - executor.triggerAll(); - assertThat(replica.getKvTablet()).isNotNull().isNotSameAs(originalKvTablet); - - CompletableFuture resumedWrite = - putHistoricalRecords(historicalPartitionManager, replica, secondBatch); - executor.triggerAll(); - assertThat(resumedWrite.get(10, TimeUnit.SECONDS).failed()).isFalse(); - } finally { - historicalPartitionManager.close(); - } - } - - @Test - void testMaxSizeCleanupWhenLakeCaughtUpBeforeLimitIsObserved() throws Exception { - TableInfo tableInfo = registerHistoricalTableAndBecomeLeader(); - Replica replica = replicaManager.getReplicaOrException(TABLE_BUCKET); - KvTablet originalKvTablet = replica.getKvTablet(); - assertThat(originalKvTablet).isNotNull(); - - Configuration cleanupConf = lookupConfiguration(); - cleanupConf.set( - ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME, Duration.ZERO); - ManuallyTriggeredScheduledExecutorService executor = - new ManuallyTriggeredScheduledExecutorService(); - TestingHistoricalLakeLookupManager lakeLookupManager = - new TestingHistoricalLakeLookupManager(cleanupConf); - HistoricalPartitionManager historicalPartitionManager = - createCleanupManager(cleanupConf, executor, lakeLookupManager, 1L); - - KvRecordBatch firstBatch = - batch( - HISTORICAL_KEY_TYPE, - tableInfo.getRowType(), - Tuple2.of( - new Object[] {1, "us"}, - new Object[] {1, "us", ORIGINAL_PARTITION, "v1"})); - KvRecordBatch secondBatch = - batch( - HISTORICAL_KEY_TYPE, - tableInfo.getRowType(), - Tuple2.of( - new Object[] {2, "eu"}, - new Object[] {2, "eu", ORIGINAL_PARTITION, "v2"})); - - try { - CompletableFuture firstPut = - putHistoricalRecords(historicalPartitionManager, replica, firstBatch); - executor.triggerAll(); - assertThat(firstPut.get(10, TimeUnit.SECONDS).failed()).isFalse(); - flushAndWait(originalKvTablet, Long.MAX_VALUE); - try (FlushOptions flushOptions = new FlushOptions().setWaitForFlush(true)) { - originalKvTablet.getRocksDBKv().getDb().flush(flushOptions); - } - assertThat(originalKvTablet.liveSstFilesSize()).isPositive(); - - // Lake catches up before another request observes that the SST size reached the limit. - long tieredOffset = replica.getLocalLogEndOffset(); - replica.getLogTablet().updateLakeLogEndOffset(tieredOffset); - historicalPartitionManager.onLakeProgress(replica, 12L, tieredOffset); - assertThat(executor.numQueuedRunnables()).isZero(); - - PutKvResultForBucket blockedWrite = - putHistoricalRecords(historicalPartitionManager, replica, secondBatch) - .get(10, TimeUnit.SECONDS); - assertThat(blockedWrite.getError().error()) - .isEqualTo(Errors.HISTORICAL_PARTITION_THROTTLED); - assertThat(executor.numQueuedRunnables()).isOne(); - - executor.triggerAll(); - assertThat(replica.getKvTablet()).isNotNull().isNotSameAs(originalKvTablet); - assertThat(lakeLookupManager.requiredLakeSnapshotIds).containsExactly(12L); - - CompletableFuture resumedWrite = - putHistoricalRecords(historicalPartitionManager, replica, secondBatch); - executor.triggerAll(); - assertThat(resumedWrite.get(10, TimeUnit.SECONDS).failed()).isFalse(); - } finally { - historicalPartitionManager.close(); - } - } - @Test void testHistoricalLookupThrottledWhenPermitsExhausted() throws Exception { registerHistoricalTableAndBecomeLeader(); @@ -1031,7 +735,7 @@ void testHistoricalLookupThrottledWhenPermitsExhausted() throws Exception { ManuallyTriggeredScheduledExecutorService executor = new ManuallyTriggeredScheduledExecutorService(); HistoricalPartitionManager historicalPartitionManager = - createNonCleanupManager( + new HistoricalPartitionManager( new HistoricalPartitionTaskExecutor(lookupConfiguration(), executor), new TestingHistoricalLakeLookupManager(lookupConfiguration())); @@ -1232,68 +936,6 @@ private static void await(CountDownLatch latch) { } } - private HistoricalPartitionManager createCleanupManager( - Configuration configuration, - ManuallyTriggeredScheduledExecutorService executor, - TestingHistoricalLakeLookupManager lakeLookupManager, - long maxHistoricalKvSizeBytes) { - HistoricalPartitionManager manager = - new HistoricalPartitionManager( - configuration, - new HistoricalPartitionTaskExecutor(configuration, executor), - lakeLookupManager, - manualClock, - maxHistoricalKvSizeBytes, - new TestingCleanupScheduler(executor)); - return manager; - } - - private HistoricalPartitionManager createNonCleanupManager( - HistoricalPartitionTaskExecutor taskExecutor, - HistoricalLakeLookupManager lakeLookupManager) { - return new HistoricalPartitionManager( - new Configuration(), - taskExecutor, - lakeLookupManager, - manualClock, - Long.MAX_VALUE, - null); - } - - private static final class TestingCleanupScheduler - implements org.apache.fluss.utils.concurrent.Scheduler { - private final ManuallyTriggeredScheduledExecutorService executor; - - private TestingCleanupScheduler(ManuallyTriggeredScheduledExecutorService executor) { - this.executor = executor; - } - - @Override - public void startup() {} - - @Override - public void shutdown() {} - - @Override - public ScheduledFuture schedule( - String name, Runnable task, long delayMs, long periodMs) { - if (periodMs > 0L) { - return executor.scheduleAtFixedRate(task, delayMs, periodMs, TimeUnit.MILLISECONDS); - } - return executor.schedule(task, delayMs, TimeUnit.MILLISECONDS); - } - } - - private static CompletableFuture putHistoricalRecords( - HistoricalPartitionManager manager, Replica replica, KvRecordBatch records) { - return manager.putKv( - replica, - new PutKvDataForBucket(TABLE_BUCKET, records, ORIGINAL_PARTITION), - null, - MergeMode.DEFAULT, - 1); - } - @SafeVarargs private static KvRecordBatch batch( RowType keyType, RowType rowType, Tuple2... keyAndValues) @@ -1329,7 +971,6 @@ private final class TestingHistoricalLakeLookupManager extends HistoricalLakeLoo private final AtomicInteger lookupCount = new AtomicInteger(); private final AtomicInteger lookupBatchCount = new AtomicInteger(); private final Map lakeValuesByPartition = new HashMap<>(); - private final List requiredLakeSnapshotIds = new ArrayList<>(); private volatile @Nullable Runnable lookupHook; private TestingHistoricalLakeLookupManager(Configuration configuration) { @@ -1351,12 +992,6 @@ private void setLookupHook(Runnable lookupHook) { this.lookupHook = lookupHook; } - @Override - void requireLakeSnapshot(long tableId, long snapshotId) { - requiredLakeSnapshotIds.add(snapshotId); - super.requireLakeSnapshot(tableId, snapshotId); - } - @Override List lookup( LookupDataForBucket lookupData, From f6914503aba659c1f43147e3067fc572dbd59a25 Mon Sep 17 00:00:00 2001 From: luoyuxia Date: Wed, 2 Sep 2026 16:51:18 +0800 Subject: [PATCH 8/8] [server] Incrementally clean historical KV state Preserve historical write lookup results so compaction between lookup and apply cannot change merge, delete, or changelog semantics. Tag historical values and tombstones with their producing WAL offsets, advance an exclusive cleanup offset after publishing the covering lake snapshot, and reuse the RocksDB compaction filter for best-effort cleanup. Perform lake lookups between the local probe and apply lock sections, then expose the complete request-scoped result through an in-memory historical value lookup. Co-Authored-By: Codex AI-Model: gpt-5 AI-Contributed/Feature: 480/480 AI-Contributed/UT: 102/102 --- .../apache/fluss/row/encode/ValueEncoder.java | 46 +- .../fluss/row/encode/KvValueLayoutTest.java | 16 + .../fluss/server/kv/KvRecoverHelper.java | 14 +- .../fluss/server/kv/KvStateAccessor.java | 14 +- .../org/apache/fluss/server/kv/KvTablet.java | 68 ++- .../fluss/server/kv/KvWriteProcessor.java | 133 +++-- .../kv/RowTtlCompactionFilterFactory.java | 42 +- .../kv/historical/HistoricalKvTombstone.java | 43 ++ .../kv/historical/HistoricalValueLookup.java | 11 +- .../LocalPreviousValueLookupResult.java | 128 +++++ .../server/kv/prewrite/KvPreWriteBuffer.java | 3 +- .../apache/fluss/server/replica/Replica.java | 94 +++- .../fluss/server/replica/ReplicaManager.java | 14 +- .../HistoricalPartitionManager.java | 73 +-- .../kv/HistoricalKvCompactionFilterTest.java | 100 ++++ .../server/kv/RowTtlCompactionFilterTest.java | 10 +- .../LocalPreviousValueLookupResultTest.java | 102 ++++ .../HistoricalPartitionManagerTest.java | 470 ++++++++++++++++-- 18 files changed, 1158 insertions(+), 223 deletions(-) create mode 100644 fluss-server/src/main/java/org/apache/fluss/server/kv/historical/HistoricalKvTombstone.java create mode 100644 fluss-server/src/main/java/org/apache/fluss/server/kv/historical/LocalPreviousValueLookupResult.java create mode 100644 fluss-server/src/test/java/org/apache/fluss/server/kv/HistoricalKvCompactionFilterTest.java create mode 100644 fluss-server/src/test/java/org/apache/fluss/server/kv/historical/LocalPreviousValueLookupResultTest.java diff --git a/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueEncoder.java b/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueEncoder.java index 13069a16562..a4054b1ede6 100644 --- a/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueEncoder.java +++ b/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueEncoder.java @@ -20,9 +20,12 @@ import org.apache.fluss.record.BinaryValue; import org.apache.fluss.row.BinaryRow; +import javax.annotation.Nullable; + import java.util.function.ToLongFunction; import static org.apache.fluss.utils.Preconditions.checkNotNull; +import static org.apache.fluss.utils.Preconditions.checkState; /** An encoder to encode {@link BinaryRow} with a schema id as value to be stored in kv store. */ public final class ValueEncoder { @@ -30,21 +33,28 @@ public final class ValueEncoder { private static final ValueEncoder PLAIN_ENCODER = new ValueEncoder(KvValueLayout.PLAIN, null); private final KvValueLayout kvValueLayout; - private final ToLongFunction valueTagProvider; - private ValueEncoder(KvValueLayout kvValueLayout, ToLongFunction valueTagProvider) { + /** + * Generates tags for {@link #encodeValue(BinaryValue)}. {@code null} for plain values or when + * callers supply each tag to {@link #encodeValue(BinaryValue, long)}. + */ + @Nullable private final ToLongFunction valueTagProvider; + + private ValueEncoder( + KvValueLayout kvValueLayout, @Nullable ToLongFunction valueTagProvider) { this.kvValueLayout = kvValueLayout; this.valueTagProvider = valueTagProvider; } - /** Returns an encoder for a layout without an internal value tag. */ + /** + * Returns an encoder for the given layout. Tagged values must supply their tag to {@link + * #encodeValue(BinaryValue, long)}. + */ public static ValueEncoder forLayout(KvValueLayout kvValueLayout) { checkNotNull(kvValueLayout, "kvValueLayout must not be null."); - if (kvValueLayout != KvValueLayout.PLAIN) { - throw new IllegalArgumentException( - "A value tag provider is required for this KV value layout."); - } - return PLAIN_ENCODER; + return kvValueLayout == KvValueLayout.PLAIN + ? PLAIN_ENCODER + : new ValueEncoder(kvValueLayout, null); } /** Returns an encoder for a layout with an internal value tag. */ @@ -66,11 +76,27 @@ public boolean hasValueTag() { /** Encodes a binary value using the layout bound to this encoder. */ public byte[] encodeValue(BinaryValue value) { + checkState( + !kvValueLayout.hasValueTag() || valueTagProvider != null, + "An explicit value tag is required for this KV value encoder."); + return encodeValueInternal( + value, valueTagProvider == null ? 0L : valueTagProvider.applyAsLong(value.row)); + } + + /** Encodes a binary value with the supplied opaque value tag. */ + public byte[] encodeValue(BinaryValue value, long valueTag) { + checkState( + kvValueLayout.hasValueTag(), + "An explicit value tag is not supported for this KV value layout."); + return encodeValueInternal(value, valueTag); + } + + private byte[] encodeValueInternal(BinaryValue value, long valueTag) { int rowPayloadOffset = kvValueLayout.rowPayloadOffset(); byte[] values = new byte[rowPayloadOffset + value.row.getSizeInBytes()]; kvValueLayout.writeSchemaId(values, value.schemaId); - if (valueTagProvider != null) { - kvValueLayout.writeValueTag(values, valueTagProvider.applyAsLong(value.row)); + if (kvValueLayout.hasValueTag()) { + kvValueLayout.writeValueTag(values, valueTag); } value.row.copyTo(values, rowPayloadOffset); return values; diff --git a/fluss-common/src/test/java/org/apache/fluss/row/encode/KvValueLayoutTest.java b/fluss-common/src/test/java/org/apache/fluss/row/encode/KvValueLayoutTest.java index a884fd05d16..98ba190f36f 100644 --- a/fluss-common/src/test/java/org/apache/fluss/row/encode/KvValueLayoutTest.java +++ b/fluss-common/src/test/java/org/apache/fluss/row/encode/KvValueLayoutTest.java @@ -68,6 +68,22 @@ void testLongTagKeepsRpcValueBodyAsSuffix() { assertThat(KvValueLayout.TAGGED.toValueBodySlice(null)).isNull(); } + @Test + void testEncodeValueWithCallerProvidedTag() { + BinaryRow row = compactedRow(DATA1_ROW_TYPE, new Object[] {1, "a"}); + byte[] value = + ValueEncoder.forLayout(KvValueLayout.TAGGED) + .encodeValue(new BinaryValue(DEFAULT_SCHEMA_ID, row), 42L); + + assertThat(KvValueLayout.TAGGED.readValueTag(MemorySegment.wrap(value))).isEqualTo(42L); + assertThatThrownBy( + () -> + ValueEncoder.forLayout(KvValueLayout.TAGGED) + .encodeValue(new BinaryValue(DEFAULT_SCHEMA_ID, row))) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("explicit value tag"); + } + @Test void testTwoArgumentValueDecoderUsesPlainLayout() { BinaryRow row = compactedRow(DATA1_ROW_TYPE, new Object[] {1, "a"}); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvRecoverHelper.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvRecoverHelper.java index a3afeec4bbf..0deed60a418 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvRecoverHelper.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvRecoverHelper.java @@ -39,6 +39,7 @@ import org.apache.fluss.row.indexed.IndexedRow; import org.apache.fluss.server.kv.autoinc.AutoIncIDRange; import org.apache.fluss.server.kv.historical.HistoricalKvKeyEncoder; +import org.apache.fluss.server.kv.historical.HistoricalKvTombstone; import org.apache.fluss.server.log.FetchIsolation; import org.apache.fluss.server.log.LogTablet; import org.apache.fluss.server.zk.ZooKeeperClient; @@ -56,7 +57,6 @@ import java.util.List; import static org.apache.fluss.server.TabletManagerBase.getTableInfo; -import static org.apache.fluss.server.kv.KvStateAccessor.HISTORICAL_TOMBSTONE; import static org.apache.fluss.utils.Preconditions.checkArgument; import static org.apache.fluss.utils.Preconditions.checkNotNull; @@ -149,7 +149,9 @@ public void recover() throws Exception { (resumeRecord) -> { if (resumeRecord.value == null) { if (historicalPartition) { - kvBatchWriter.put(resumeRecord.key, HISTORICAL_TOMBSTONE); + kvBatchWriter.put( + resumeRecord.key, + HistoricalKvTombstone.encode(resumeRecord.logOffset)); } else { kvBatchWriter.delete(resumeRecord.key); } @@ -296,9 +298,13 @@ private long applyLogRecordBatch( // the log row format may not compatible with kv row format, // e.g, arrow vs. compacted, thus needs a conversion here. BinaryRow row = toKvRow(logRow); + BinaryValue binaryValue = + new BinaryValue(currentSchemaId.shortValue(), row); value = - valueEncoder.encodeValue( - new BinaryValue(currentSchemaId.shortValue(), row)); + historicalPartition + ? valueEncoder.encodeValue( + binaryValue, logRecord.logOffset()) + : valueEncoder.encodeValue(binaryValue); } resumeRecordConsumer.accept( new KeyValueAndLogOffset( diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvStateAccessor.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvStateAccessor.java index 1e0cd39962e..daba99a6c84 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvStateAccessor.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvStateAccessor.java @@ -19,6 +19,7 @@ import org.apache.fluss.annotation.Internal; import org.apache.fluss.server.kv.historical.HistoricalKvKeyEncoder; +import org.apache.fluss.server.kv.historical.HistoricalKvTombstone; import org.apache.fluss.server.kv.prewrite.KvPreWriteBuffer; import org.apache.fluss.server.kv.prewrite.KvPreWriteBuffer.Key; import org.apache.fluss.server.kv.prewrite.KvPreWriteBuffer.TruncateReason; @@ -35,9 +36,6 @@ @Internal public final class KvStateAccessor { - /** Encoded RocksDB value marking a deleted key in historical KV state. */ - static final byte[] HISTORICAL_TOMBSTONE = new byte[0]; - private final KvPreWriteBuffer preWriteBuffer; private final RocksDBKv rocksDBKv; private final boolean historicalPartition; @@ -85,9 +83,9 @@ public KvStateLookupResult lookup(Key key) throws IOException { if (value == null) { return KvStateLookupResult.notFound(); } - // Historical KV tablets persist deletes as empty values so that a local miss does not - // expose a stale value from lake storage after the buffered delete has been flushed. - return value.length == 0 + // Historical KV tablets persist deletes as offset-tagged tombstones so that a local miss + // does not expose a stale value from lake storage after the buffered delete is flushed. + return historicalPartition && HistoricalKvTombstone.isTombstone(value) ? KvStateLookupResult.deleted() : KvStateLookupResult.present(value); } @@ -118,4 +116,8 @@ public void delete(Key key, long logOffset) { public void truncateTo(long logOffset, TruncateReason reason) { preWriteBuffer.truncateTo(logOffset, reason); } + + boolean isHistoricalPartition() { + return historicalPartition; + } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java index 6118bb40fb3..760b8cd19c9 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java @@ -41,7 +41,9 @@ import org.apache.fluss.rpc.protocol.MergeMode; import org.apache.fluss.server.kv.autoinc.AutoIncIDRange; import org.apache.fluss.server.kv.autoinc.AutoIncrementManager; +import org.apache.fluss.server.kv.historical.HistoricalKvTombstone; import org.apache.fluss.server.kv.historical.HistoricalValueLookup; +import org.apache.fluss.server.kv.historical.LocalPreviousValueLookupResult; import org.apache.fluss.server.kv.prewrite.KvPreWriteBuffer; import org.apache.fluss.server.kv.prewrite.KvPreWriteBuffer.PreparedFlush; import org.apache.fluss.server.kv.rocksdb.RocksDBKv; @@ -90,11 +92,12 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; -import static org.apache.fluss.server.kv.KvStateAccessor.HISTORICAL_TOMBSTONE; import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; +import static org.apache.fluss.utils.Preconditions.checkArgument; import static org.apache.fluss.utils.Preconditions.checkNotNull; import static org.apache.fluss.utils.Preconditions.checkState; import static org.apache.fluss.utils.concurrent.LockUtils.inReadLock; @@ -147,6 +150,7 @@ public final class KvTablet { private final ReadWriteLock kvLock = new ReentrantReadWriteLock(); private final KvValueLayout kvValueLayout; private final ValueEncoder valueEncoder; + private final AtomicLong historicalCleanupOffset; @Nullable private final RowTtlTimestampProvider rowTtlTimestampProvider; private final boolean rowTtlEnabled; private final AutoIncrementManager autoIncrementManager; @@ -197,6 +201,7 @@ private KvTablet( KvValueLayout kvValueLayout, ValueEncoder valueEncoder, ValueDecoder valueDecoder, + AtomicLong historicalCleanupOffset, @Nullable RocksDBStatistics rocksDBStatistics, KvFlushScheduler kvFlushScheduler, boolean closeFlushScheduler, @@ -221,6 +226,7 @@ private KvTablet( new KvStateAccessor(kvPreWriteBuffer, rocksDBKv, historicalPartition); this.kvValueLayout = kvValueLayout; this.valueEncoder = valueEncoder; + this.historicalCleanupOffset = historicalCleanupOffset; this.rowTtlTimestampProvider = rowTtlTimestampProvider; this.rowTtlEnabled = rowTtlEnabled; this.kvWriteProcessor = @@ -367,11 +373,16 @@ private static KvTablet create( TableConfig tableConfig) throws IOException { checkNotNull(tableConfig, "tableConfig must not be null."); + boolean historicalPartition = + HISTORICAL_PARTITION_VALUE.equals(tablePath.getPartitionName()); Optional rowTtl = tableConfig.getKvTTL(); - KvValueLayout kvValueLayout = KvValueLayout.fromTableConfig(tableConfig); + KvValueLayout kvValueLayout = + historicalPartition + ? KvValueLayout.TAGGED + : KvValueLayout.fromTableConfig(tableConfig); @Nullable RowTtlTimestampProvider rowTtlTimestampProvider = - kvValueLayout.hasValueTag() + !historicalPartition && kvValueLayout.hasValueTag() ? RowTtlTimestampProvider.create( tableConfig, schemaGetter, ZoneId.systemDefault()) : null; @@ -380,13 +391,17 @@ private static KvTablet create( ? ValueEncoder.forLayout(kvValueLayout) : ValueEncoder.forLayout(kvValueLayout, rowTtlTimestampProvider); ValueDecoder valueDecoder = new ValueDecoder(schemaGetter, kvFormat, kvValueLayout); + AtomicLong historicalCleanupOffset = new AtomicLong(0L); @Nullable AbstractCompactionFilterFactory> compactionFilterFactory = - rowTtl.isPresent() + historicalPartition ? RowTtlCompactionFilterFactory.create( - kvValueLayout, rowTtl.get(), clock) - : null; + kvValueLayout, 0L, () -> historicalCleanupOffset.get() - 1L) + : rowTtl.isPresent() + ? RowTtlCompactionFilterFactory.create( + kvValueLayout, rowTtl.get(), clock) + : null; RocksDBKv kv = buildRocksDBKv(serverConf, kvTabletDir, sharedRateLimiter, compactionFilterFactory); @@ -420,6 +435,7 @@ private static KvTablet create( kvValueLayout, valueEncoder, valueDecoder, + historicalCleanupOffset, rocksDBStatistics, kvFlushScheduler, closeFlushScheduler, @@ -655,16 +671,15 @@ public LogAppendInfo putAsLeader( * Puts records for one original partition into this historical KV tablet. * *

The original partition name namespaces the physical primary keys because one historical - * bucket can contain records from multiple original partitions. The supplied fallback may only - * read lake results already resolved for this request; it must not perform lake I/O while the - * tablet lock is held. + * bucket can contain records from multiple original partitions. The supplied lookup must + * contain every previous value required by this batch and must not perform I/O. */ public LogAppendInfo putHistoricalAsLeader( KvRecordBatch kvRecords, @Nullable int[] targetColumns, MergeMode mergeMode, String originalPartitionName, - HistoricalValueLookup memoizedLakeLookup) + HistoricalValueLookup historicalValueLookup) throws Exception { checkState(historicalPartition, "%s is not a historical KV tablet", tableBucket); return putAsLeader( @@ -672,16 +687,16 @@ public LogAppendInfo putHistoricalAsLeader( targetColumns, mergeMode, checkNotNull(originalPartitionName, "originalPartitionName must not be null"), - checkNotNull(memoizedLakeLookup, "memoizedLakeLookup must not be null")); + checkNotNull(historicalValueLookup, "Historical value lookup must not be null")); } /** - * Finds keys whose historical write requires an old value that is absent from local state. + * Probes the local previous values required by a historical write. * *

This method only reads KV entries and uses the tablet read lock. Lake I/O must be * performed by the caller after this method releases the tablet lock. */ - public List findKeysRequiringLakeLookup( + public LocalPreviousValueLookupResult probeLocalPreviousValues( KvRecordBatch kvRecords, @Nullable int[] targetColumns, MergeMode mergeMode, @@ -692,7 +707,7 @@ public List findKeysRequiringLakeLookup( kvLock, () -> { rocksDBKv.checkIfRocksDBClosed(); - return kvWriteProcessor.findKeysRequiringLakeLookup( + return kvWriteProcessor.probeLocalPreviousValues( kvRecords, targetColumns, mergeMode, @@ -708,7 +723,7 @@ private LogAppendInfo putAsLeader( @Nullable int[] targetColumns, MergeMode mergeMode, @Nullable String originalPartitionName, - @Nullable HistoricalValueLookup memoizedLakeLookup) + @Nullable HistoricalValueLookup historicalValueLookup) throws Exception { return inWriteLock( kvLock, @@ -737,7 +752,7 @@ private LogAppendInfo putAsLeader( mergeMode, kvStateAccessor, originalPartitionName, - memoizedLakeLookup); + historicalValueLookup); }); } @@ -761,6 +776,21 @@ public long getFlushedLogOffset() { return flushedLogOffset; } + /** Advances the exclusive historical cleanup offset without allowing it to move backwards. */ + public boolean advanceHistoricalCleanupOffset(long cleanupOffset) { + checkState(historicalPartition, "%s is not a historical KV tablet", tableBucket); + checkArgument(cleanupOffset >= 0L, "Historical cleanup offset must be non-negative."); + long previousCleanupOffset = + historicalCleanupOffset.getAndAccumulate(cleanupOffset, Math::max); + return cleanupOffset > previousCleanupOffset; + } + + /** Returns the current exclusive cleanup offset for a historical overlay. */ + public long getHistoricalCleanupOffset() { + checkState(historicalPartition, "%s is not a historical KV tablet", tableBucket); + return historicalCleanupOffset.get(); + } + @VisibleForTesting FlushState getFlushState() { return inReadLock(kvLock, () -> flushState); @@ -906,7 +936,9 @@ private void writePreparedFlush(PreparedFlush preparedFlush) throws Exception { if (historicalPartition) { // A physical delete would turn a local miss into a lake lookup and // could expose the stale value that this mutation deleted. - kvBatchWriter.put(entry.getKey().get(), HISTORICAL_TOMBSTONE); + kvBatchWriter.put( + entry.getKey().get(), + HistoricalKvTombstone.encode(entry.getLogSequenceNumber())); } else { kvBatchWriter.delete(entry.getKey().get()); } @@ -1119,7 +1151,7 @@ public KvStateLookupResult lookupHistoricalLocal(String originalPartitionName, b if (value == null) { return KvStateLookupResult.notFound(); } - return value.length == 0 + return HistoricalKvTombstone.isTombstone(value) ? KvStateLookupResult.deleted() : KvStateLookupResult.present(value); }); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvWriteProcessor.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvWriteProcessor.java index bc0fb53fff0..9dc46ac53a0 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvWriteProcessor.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvWriteProcessor.java @@ -45,6 +45,7 @@ import org.apache.fluss.server.kv.autoinc.AutoIncrementManager; import org.apache.fluss.server.kv.autoinc.AutoIncrementUpdater; import org.apache.fluss.server.kv.historical.HistoricalValueLookup; +import org.apache.fluss.server.kv.historical.LocalPreviousValueLookupResult; import org.apache.fluss.server.kv.prewrite.KvPreWriteBuffer; import org.apache.fluss.server.kv.prewrite.KvPreWriteBuffer.TruncateReason; import org.apache.fluss.server.kv.rowmerger.DefaultRowMerger; @@ -66,11 +67,12 @@ import javax.annotation.Nullable; import javax.annotation.concurrent.NotThreadSafe; -import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; -import java.util.List; import java.util.Set; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + /** * Processes a KV record batch into local state mutations and the corresponding WAL records. * @@ -81,10 +83,9 @@ * *

The supplied {@link KvStateAccessor} defines how keys and state are accessed. Normal writes * use the original primary key and local state, while historical writes use partition-scoped keys. - * On a historical local miss, the processor can consult a lake result already memoized for the - * current request. Resolving that result from lake storage remains the caller's responsibility and - * must happen outside the tablet lock. The merge and WAL generation path is shared by both write - * kinds. + * Historical writes use previous values resolved before the tablet write lock is acquired. + * Resolving local misses from lake storage remains the caller's responsibility and must happen + * outside the tablet lock. The merge and WAL generation path is shared by both write kinds. */ @Internal @NotThreadSafe @@ -158,7 +159,7 @@ public LogAppendInfo putAsLeader( MergeMode mergeMode, KvStateAccessor stateAccessor, @Nullable String originalPartitionName, - @Nullable HistoricalValueLookup memoizedLakeLookup) + @Nullable HistoricalValueLookup historicalValueLookup) throws Exception { WriteContext writeContext = createWriteContext(kvRecords, targetColumns, mergeMode); RowType latestRowType = writeContext.latestSchema.getRowType(); @@ -185,7 +186,7 @@ public LogAppendInfo putAsLeader( logEndOffsetOfPrevBatch, stateAccessor, originalPartitionName, - memoizedLakeLookup); + historicalValueLookup); // There will be a situation that these batches of kvRecordBatch have not // generated any CDC logs, for example, when client attempts to delete @@ -221,14 +222,13 @@ public LogAppendInfo putAsLeader( } /** - * Finds the original primary keys whose previous values must be loaded from lake storage before - * applying a historical write batch. + * Probes the local previous values required by a historical write batch. * - *

A key is returned only when its previous value is required and its partition-scoped key is - * absent from local state. Records that can establish their result without a previous value are - * skipped, and each key is returned at most once. + *

Every key whose previous value is required is probed at most once. Local values and + * deletes are decoded into the returned collection, while true local misses are exposed for + * lake lookup. Records that establish their result without a previous value are skipped. */ - List findKeysRequiringLakeLookup( + LocalPreviousValueLookupResult probeLocalPreviousValues( KvRecordBatch kvRecords, @Nullable int[] targetColumns, MergeMode mergeMode, @@ -237,9 +237,9 @@ List findKeysRequiringLakeLookup( throws Exception { WriteContext writeContext = createWriteContext(kvRecords, targetColumns, mergeMode); - List keysRequiringLakeLookup = new ArrayList<>(); - // Track keys whose lake-lookup requirement has already been evaluated. - Set keysEvaluatedForLakeLookup = new HashSet<>(); + LocalPreviousValueLookupResult localLookupResult = + new LocalPreviousValueLookupResult(valueDecoder, lakeValueDecoder); + Set probedKeys = new HashSet<>(); KvRecordBatch.ReadContext readContext = KvRecordReadContext.createReadContext(kvFormat, schemaGetter); for (KvRecord kvRecord : kvRecords.records(readContext)) { @@ -252,23 +252,20 @@ List findKeysRequiringLakeLookup( } else if (canSkipOldValueLookup( writeContext.rowMerger, writeContext.autoIncrementUpdater)) { // A full-row WAL upsert establishes the state without reading its previous value. - keysEvaluatedForLakeLookup.add(wrappedKey); + probedKeys.add(wrappedKey); continue; } // Probe local state only for the first record that needs a previous value. A true local // miss schedules one lake lookup shared by all records for this key in the batch. - if (keysEvaluatedForLakeLookup.add(wrappedKey) - && stateAccessor - .lookup( - stateAccessor.encodeKey( - primaryKey, originalPartitionName)) - .status() - == KvStateLookupResult.Status.NOT_FOUND) { - keysRequiringLakeLookup.add(primaryKey); + if (probedKeys.add(wrappedKey)) { + localLookupResult.add( + primaryKey, + stateAccessor.lookup( + stateAccessor.encodeKey(primaryKey, originalPartitionName))); } } - return keysRequiringLakeLookup; + return localLookupResult; } private WriteContext createWriteContext( @@ -313,9 +310,13 @@ private void processKvRecords( long startLogOffset, KvStateAccessor stateAccessor, @Nullable String originalPartitionName, - @Nullable HistoricalValueLookup memoizedLakeLookup) + @Nullable HistoricalValueLookup historicalValueLookup) throws Exception { long logOffset = startLogOffset; + // Once this batch mutates a historical key, later same-key records must read the staged + // value from local state instead of reusing the value memoized before apply. + Set stagedHistoricalKeys = + historicalValueLookup == null ? Collections.emptySet() : new HashSet<>(); // TODO: reuse the read context KvRecordBatch.ReadContext readContext = @@ -326,6 +327,11 @@ private void processKvRecords( KvPreWriteBuffer.Key key = stateAccessor.encodeKey(keyBytes, originalPartitionName); BinaryRow row = kvRecord.getRow(); BinaryValue currentValue = row == null ? null : new BinaryValue(schemaIdOfNewData, row); + long previousLogOffset = logOffset; + ByteArrayWrapper historicalKey = + historicalValueLookup == null ? null : new ByteArrayWrapper(keyBytes); + boolean useMemoizedPreviousValue = + historicalKey != null && !stagedHistoricalKeys.contains(historicalKey); if (currentValue == null) { logOffset = @@ -337,7 +343,8 @@ private void processKvRecords( logOffset, stateAccessor, keyBytes, - memoizedLakeLookup); + historicalValueLookup, + useMemoizedPreviousValue); } else { logOffset = processUpsert( @@ -350,7 +357,11 @@ private void processKvRecords( logOffset, stateAccessor, keyBytes, - memoizedLakeLookup); + historicalValueLookup, + useMemoizedPreviousValue); + } + if (historicalKey != null && logOffset > previousLogOffset) { + stagedHistoricalKeys.add(historicalKey); } } } @@ -363,13 +374,20 @@ private long processDeletion( long logOffset, KvStateAccessor stateAccessor, byte[] primaryKey, - @Nullable HistoricalValueLookup memoizedLakeLookup) + @Nullable HistoricalValueLookup historicalValueLookup, + boolean useMemoizedPreviousValue) throws Exception { if (shouldIgnoreDeletion(currentMerger)) { return logOffset; } - BinaryValue oldValue = getPreviousValue(key, primaryKey, stateAccessor, memoizedLakeLookup); + BinaryValue oldValue = + getPreviousValue( + key, + primaryKey, + stateAccessor, + historicalValueLookup, + useMemoizedPreviousValue); if (oldValue == null) { LOG.debug( "The specific key can't be found in kv tablet although the kv record is for deletion, " @@ -399,14 +417,21 @@ private long processUpsert( long logOffset, KvStateAccessor stateAccessor, byte[] primaryKey, - @Nullable HistoricalValueLookup memoizedLakeLookup) + @Nullable HistoricalValueLookup historicalValueLookup, + boolean useMemoizedPreviousValue) throws Exception { if (canSkipOldValueLookup(currentMerger, autoIncrementUpdater)) { return applyUpdate( key, null, currentValue, walBuilder, latestSchemaRow, logOffset, stateAccessor); } - BinaryValue oldValue = getPreviousValue(key, primaryKey, stateAccessor, memoizedLakeLookup); + BinaryValue oldValue = + getPreviousValue( + key, + primaryKey, + stateAccessor, + historicalValueLookup, + useMemoizedPreviousValue); if (oldValue == null) { BinaryValue valueToInsert = currentMerger.merge(null, currentValue); return applyInsert( @@ -454,7 +479,7 @@ private long applyInsert( throws Exception { BinaryValue newValue = autoIncrementUpdater.updateAutoIncrementColumns(currentValue); walBuilder.append(ChangeType.INSERT, latestSchemaRow.replaceRow(newValue.row)); - stateAccessor.insert(key, valueEncoder.encodeValue(newValue), logOffset); + stateAccessor.insert(key, encodeStateValue(newValue, logOffset, stateAccessor), logOffset); return logOffset + 1; } @@ -469,42 +494,52 @@ private long applyUpdate( throws Exception { if (changelogImage == ChangelogImage.WAL) { walBuilder.append(ChangeType.UPDATE_AFTER, latestSchemaRow.replaceRow(newValue.row)); - stateAccessor.update(key, valueEncoder.encodeValue(newValue), logOffset); + stateAccessor.update( + key, encodeStateValue(newValue, logOffset, stateAccessor), logOffset); return logOffset + 1; } else { walBuilder.append(ChangeType.UPDATE_BEFORE, latestSchemaRow.replaceRow(oldValue.row)); walBuilder.append(ChangeType.UPDATE_AFTER, latestSchemaRow.replaceRow(newValue.row)); - stateAccessor.update(key, valueEncoder.encodeValue(newValue), logOffset + 1); + long updateAfterOffset = logOffset + 1; + stateAccessor.update( + key, + encodeStateValue(newValue, updateAfterOffset, stateAccessor), + updateAfterOffset); return logOffset + 2; } } + private byte[] encodeStateValue( + BinaryValue value, long logOffset, KvStateAccessor stateAccessor) { + return stateAccessor.isHistoricalPartition() + ? valueEncoder.encodeValue(value, logOffset) + : valueEncoder.encodeValue(value); + } + /** - * Returns the previous value from local state, falling back to the memoized lake result only on - * a genuine local miss. + * Returns the previous value from request-scoped historical state or the current local state. * * @param localStateKey the key used by the local prewrite buffer and RocksDB; it wraps {@code * primaryKey} for a normal partition and adds the original partition namespace for a * historical partition * @param primaryKey the encoded bytes of the logical primary key, without the historical - * partition namespace; used by the lake lookup + * partition namespace * @return the previous value, or null if the key is absent or locally marked as deleted */ private BinaryValue getPreviousValue( KvPreWriteBuffer.Key localStateKey, byte[] primaryKey, KvStateAccessor stateAccessor, - @Nullable HistoricalValueLookup memoizedLakeLookup) + @Nullable HistoricalValueLookup historicalValueLookup, + boolean useMemoizedPreviousValue) throws Exception { - KvStateLookupResult localResult = stateAccessor.lookup(localStateKey); - if (localResult.status() != KvStateLookupResult.Status.NOT_FOUND - || memoizedLakeLookup == null) { - return localResult.value() == null - ? null - : valueDecoder.decodeValue(localResult.value()); + if (useMemoizedPreviousValue) { + return checkNotNull(historicalValueLookup, "Historical value lookup must not be null") + .lookup(primaryKey); } - byte[] lakeValue = memoizedLakeLookup.lookup(primaryKey); - return lakeValue == null ? null : lakeValueDecoder.decodeValue(lakeValue); + + KvStateLookupResult localResult = stateAccessor.lookup(localStateKey); + return localResult.value() == null ? null : valueDecoder.decodeValue(localResult.value()); } private boolean canSkipOldValueLookup( diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/RowTtlCompactionFilterFactory.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/RowTtlCompactionFilterFactory.java index 8cb01d0c1f2..e77e5b9b468 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/kv/RowTtlCompactionFilterFactory.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/RowTtlCompactionFilterFactory.java @@ -17,7 +17,6 @@ package org.apache.fluss.server.kv; -import org.apache.fluss.annotation.VisibleForTesting; import org.apache.fluss.row.encode.KvValueLayout; import org.apache.fluss.server.utils.RowTtlUtils; import org.apache.fluss.utils.clock.Clock; @@ -26,6 +25,7 @@ import org.rocksdb.RocksDB; import java.time.Duration; +import java.util.function.LongSupplier; import static org.apache.fluss.utils.Preconditions.checkArgument; import static org.apache.fluss.utils.Preconditions.checkNotNull; @@ -40,29 +40,47 @@ private RowTtlCompactionFilterFactory() {} /** Creates a configured native compaction filter factory for row TTL cleanup. */ public static FlinkCompactionFilter.FlinkCompactionFilterFactory create( KvValueLayout kvValueLayout, Duration ttl, Clock clock) { - return create(kvValueLayout, ttl, QUERY_TIME_AFTER_NUM_ENTRIES, clock); + long ttlMillis = RowTtlUtils.validateAndConvertTtlDurationToMillis(ttl); + checkNotNull(clock, "clock must not be null."); + return create(kvValueLayout, ttlMillis, clock::milliseconds); } - @VisibleForTesting + /** Removes values using the default interval for refreshing the supplied current value. */ static FlinkCompactionFilter.FlinkCompactionFilterFactory create( - KvValueLayout kvValueLayout, Duration ttl, long queryTimeAfterNumEntries, Clock clock) { - long ttlMillis = RowTtlUtils.validateAndConvertTtlDurationToMillis(ttl); + KvValueLayout kvValueLayout, + long expirationDistance, + LongSupplier currentValueSupplier) { + return create( + kvValueLayout, + expirationDistance, + QUERY_TIME_AFTER_NUM_ENTRIES, + currentValueSupplier); + } + + /** Removes a value when {@code valueTag + expirationDistance <= currentValue}. */ + static FlinkCompactionFilter.FlinkCompactionFilterFactory create( + KvValueLayout kvValueLayout, + long expirationDistance, + long queryCurrentValueAfterNumEntries, + LongSupplier currentValueSupplier) { checkNotNull(kvValueLayout, "kvValueLayout must not be null."); - checkNotNull(clock, "clock must not be null."); - checkArgument(kvValueLayout.hasValueTag(), "Row TTL requires a tagged KV value layout."); + checkNotNull(currentValueSupplier, "currentValueSupplier must not be null."); + checkArgument(kvValueLayout.hasValueTag(), "Compaction filter requires a tagged layout."); + checkArgument(expirationDistance >= 0L, "Expiration distance must be non-negative."); checkArgument( - queryTimeAfterNumEntries > 0, - "queryTimeAfterNumEntries must be greater than zero."); + queryCurrentValueAfterNumEntries > 0L, + "queryCurrentValueAfterNumEntries must be greater than zero."); RocksDB.loadLibrary(); FlinkCompactionFilter.FlinkCompactionFilterFactory factory = - new FlinkCompactionFilter.FlinkCompactionFilterFactory(clock::milliseconds); + new FlinkCompactionFilter.FlinkCompactionFilterFactory( + currentValueSupplier::getAsLong); factory.configure( FlinkCompactionFilter.Config.createNotList( FlinkCompactionFilter.StateType.Value, kvValueLayout.valueTagOffset(), - ttlMillis, - queryTimeAfterNumEntries)); + expirationDistance, + queryCurrentValueAfterNumEntries)); return factory; } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/historical/HistoricalKvTombstone.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/historical/HistoricalKvTombstone.java new file mode 100644 index 00000000000..6191978d67b --- /dev/null +++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/historical/HistoricalKvTombstone.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.kv.historical; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.row.encode.KvValueLayout; + +import static org.apache.fluss.utils.Preconditions.checkArgument; + +/** Utilities for tombstones stored in a historical KV overlay. */ +@Internal +public final class HistoricalKvTombstone { + + private HistoricalKvTombstone() {} + + /** Encodes a tombstone tagged with the WAL offset that produced the delete. */ + public static byte[] encode(long logOffset) { + checkArgument(logOffset >= 0L, "Historical KV log offset must be non-negative."); + byte[] tombstone = new byte[KvValueLayout.TAGGED.valueTagLength()]; + KvValueLayout.TAGGED.writeValueTag(tombstone, logOffset); + return tombstone; + } + + /** Returns whether the raw historical value is an offset-tagged tombstone. */ + public static boolean isTombstone(byte[] rawValue) { + return rawValue.length == KvValueLayout.TAGGED.valueTagLength(); + } +} diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/historical/HistoricalValueLookup.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/historical/HistoricalValueLookup.java index bdc723df96e..e632fd2a213 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/kv/historical/HistoricalValueLookup.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/historical/HistoricalValueLookup.java @@ -2,7 +2,7 @@ * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 + * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * @@ -18,20 +18,21 @@ package org.apache.fluss.server.kv.historical; import org.apache.fluss.annotation.Internal; +import org.apache.fluss.record.BinaryValue; import javax.annotation.Nullable; -/** Resolves a lake value already memoized for the current historical write request. */ +/** Looks up a previous value already memoized for the current historical write request. */ @Internal @FunctionalInterface public interface HistoricalValueLookup { /** - * Returns the encoded value for the primary key, or null when it does not exist. + * Returns the decoded previous value, or null when the key was absent or deleted. * - *

This method is invoked while the KV write lock is held and must not perform lake or file + *

This method is invoked while the KV write lock is held and must not perform local or lake * I/O. */ @Nullable - byte[] lookup(byte[] primaryKey); + BinaryValue lookup(byte[] primaryKey); } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/historical/LocalPreviousValueLookupResult.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/historical/LocalPreviousValueLookupResult.java new file mode 100644 index 00000000000..96b6eabc652 --- /dev/null +++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/historical/LocalPreviousValueLookupResult.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.kv.historical; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.record.BinaryValue; +import org.apache.fluss.row.encode.ValueDecoder; +import org.apache.fluss.server.kv.KvStateLookupResult; +import org.apache.fluss.utils.ByteArrayWrapper; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.NotThreadSafe; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; +import static org.apache.fluss.utils.Preconditions.checkState; + +/** Previous-value lookup results captured from local state for one historical write request. */ +@Internal +@NotThreadSafe +public final class LocalPreviousValueLookupResult { + + /** Decodes tagged values returned by local historical KV state. */ + private final ValueDecoder localValueDecoder; + + /** Decodes plain values returned by lake storage. */ + private final ValueDecoder lakeValueDecoder; + + /** Previous values found locally, including tombstones represented by an empty optional. */ + private final Map> localValuesByKey = new HashMap<>(); + + /** True local misses in lake request order. */ + private final Set keysMissingLocally = new LinkedHashSet<>(); + + /** Creates a local lookup result with decoders for the local and lake value layouts. */ + public LocalPreviousValueLookupResult( + ValueDecoder localValueDecoder, ValueDecoder lakeValueDecoder) { + this.localValueDecoder = localValueDecoder; + this.lakeValueDecoder = lakeValueDecoder; + } + + /** Records the local result for one key whose previous value is required. */ + public void add(byte[] primaryKey, KvStateLookupResult localResult) { + ByteArrayWrapper wrappedKey = new ByteArrayWrapper(primaryKey); + checkState( + !localValuesByKey.containsKey(wrappedKey) + && !keysMissingLocally.contains(wrappedKey), + "Historical write key has already been probed"); + + if (localResult.status() == KvStateLookupResult.Status.NOT_FOUND) { + keysMissingLocally.add(wrappedKey); + } else { + localValuesByKey.put(wrappedKey, decode(localResult.value(), localValueDecoder)); + } + } + + /** Returns whether at least one previous value was not found in local state. */ + public boolean hasLocalMisses() { + return !keysMissingLocally.isEmpty(); + } + + /** Returns a snapshot of true local misses in lake request order. */ + public List keysMissingLocally() { + List primaryKeys = new ArrayList<>(keysMissingLocally.size()); + for (ByteArrayWrapper keyMissingLocally : keysMissingLocally) { + primaryKeys.add(keyMissingLocally.getData()); + } + return Collections.unmodifiableList(primaryKeys); + } + + /** + * Combines lake results with the local lookup results and creates an in-memory lookup for + * apply. + * + *

Lake values must have the same order as {@link #keysMissingLocally()}. + */ + public HistoricalValueLookup createValueLookup(List lakeValues) { + checkNotNull(lakeValues, "Historical lake values must not be null"); + checkArgument( + lakeValues.size() == keysMissingLocally.size(), + "Expected %s historical lake values, but received %s", + keysMissingLocally.size(), + lakeValues.size()); + Map> previousValuesByKey = + new HashMap<>(localValuesByKey); + Iterator missingKeyIterator = keysMissingLocally.iterator(); + for (byte[] lakeValue : lakeValues) { + previousValuesByKey.put(missingKeyIterator.next(), decode(lakeValue, lakeValueDecoder)); + } + return primaryKey -> + checkNotNull( + previousValuesByKey.get(new ByteArrayWrapper(primaryKey)), + "No previous value for a historical write key") + .orElse(null); + } + + private static Optional decode( + @Nullable byte[] encodedValue, ValueDecoder valueDecoder) { + return encodedValue == null + ? Optional.empty() + : Optional.of(valueDecoder.decodeValue(encodedValue)); + } +} diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/prewrite/KvPreWriteBuffer.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/prewrite/KvPreWriteBuffer.java index 98320697685..f248b8271b9 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/kv/prewrite/KvPreWriteBuffer.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/prewrite/KvPreWriteBuffer.java @@ -391,7 +391,8 @@ public Value getValue() { return value; } - long getLogSequenceNumber() { + /** Returns the WAL offset that produced this mutation. */ + public long getLogSequenceNumber() { return logSequenceNumber; } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java index fd37583db2c..516cf5b0747 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java @@ -66,6 +66,7 @@ import org.apache.fluss.server.kv.RemoteLogFetcher; import org.apache.fluss.server.kv.autoinc.AutoIncIDRange; import org.apache.fluss.server.kv.historical.HistoricalValueLookup; +import org.apache.fluss.server.kv.historical.LocalPreviousValueLookupResult; import org.apache.fluss.server.kv.rocksdb.RocksDBKvBuilder; import org.apache.fluss.server.kv.scan.OpenScanResult; import org.apache.fluss.server.kv.scan.ScannerContext; @@ -114,6 +115,7 @@ import org.apache.fluss.utils.FlussPaths; import org.apache.fluss.utils.IOUtils; import org.apache.fluss.utils.clock.Clock; +import org.apache.fluss.utils.function.FunctionWithException; import org.apache.fluss.utils.types.Tuple2; import org.slf4j.Logger; @@ -908,6 +910,10 @@ private Optional initKvTablet() { } logTablet.updateMinRetainOffset(restoreStartOffset); + if (isHistoricalPartition()) { + checkNotNull(kvTablet, "kv tablet should not be null.") + .advanceHistoricalCleanupOffset(restoreStartOffset); + } recoverKvTablet(restoreStartOffset, rowCount, autoIncIDRange); } catch (Exception e) { throw new KvStorageException( @@ -1270,40 +1276,39 @@ public LogAppendInfo putRecordsToLeader( } /** - * Finds historical write keys that require lake fallback without mutating local KV state. - * - *

The caller must keep historical writes for this table bucket ordered until the subsequent - * {@link #putHistoricalRecordsToLeader} call completes. + * Looks up previous values without holding replica or KV locks during lake I/O, then writes the + * records to the local historical KV state. */ - public List findKeysRequiringLakeLookup( - KvRecordBatch kvRecords, - @Nullable int[] targetColumns, - MergeMode mergeMode, - String originalPartitionName, - int expectedLeaderEpoch, - int requiredAcks) - throws Exception { - return inReadLock( - leaderIsrUpdateLock, - () -> { - validateHistoricalWrite(expectedLeaderEpoch, requiredAcks); - KvTablet kv = this.kvTablet; - checkNotNull(kv, "KvTablet for the historical replica shouldn't be null."); - return kv.findKeysRequiringLakeLookup( - kvRecords, targetColumns, mergeMode, originalPartitionName); - }); - } - - /** Writes records to the local historical KV state of the leader replica. */ public LogAppendInfo putHistoricalRecordsToLeader( KvRecordBatch kvRecords, @Nullable int[] targetColumns, MergeMode mergeMode, String originalPartitionName, - HistoricalValueLookup memoizedLakeLookup, + FunctionWithException, List, Exception> lakeLookup, int expectedLeaderEpoch, int requiredAcks) throws Exception { + checkNotNull(lakeLookup, "Historical lake lookup must not be null"); + LocalPreviousValueLookupResult localLookupResult = + inReadLock( + leaderIsrUpdateLock, + () -> { + validateHistoricalWrite(expectedLeaderEpoch, requiredAcks); + KvTablet kv = this.kvTablet; + checkNotNull( + kv, "KvTablet for the historical replica shouldn't be null."); + return kv.probeLocalPreviousValues( + kvRecords, targetColumns, mergeMode, originalPartitionName); + }); + + // Both the replica read lock and the KV read lock have been released before lake I/O. + List lakeValues = + localLookupResult.hasLocalMisses() + ? lakeLookup.apply(localLookupResult.keysMissingLocally()) + : Collections.emptyList(); + HistoricalValueLookup historicalValueLookup = + localLookupResult.createValueLookup(lakeValues); + return inReadLock( leaderIsrUpdateLock, () -> { @@ -1316,12 +1321,49 @@ public LogAppendInfo putHistoricalRecordsToLeader( targetColumns, mergeMode, originalPartitionName, - memoizedLakeLookup); + historicalValueLookup); maybeIncrementLeaderHW(logTablet, clock.milliseconds()); return appendInfo; }); } + /** + * Updates the historical cleanup offset if it is valid. + * + *

{@code beforeUpdate} runs after validation and before the new cleanup offset becomes + * visible to the RocksDB compaction filter. + */ + public void tryUpdateHistoricalCleanupOffset(long newCleanupOffset, Runnable beforeUpdate) { + checkNotNull(beforeUpdate, "beforeUpdate must not be null."); + inWriteLock( + leaderIsrUpdateLock, + () -> { + if (!isLeader() || !historicalPartition) { + return; + } + KvTablet currentKvTablet = kvTablet; + if (currentKvTablet == null) { + return; + } + long currentCleanupOffset = currentKvTablet.getHistoricalCleanupOffset(); + long localLogEndOffset = logTablet.localLogEndOffset(); + if (newCleanupOffset < currentCleanupOffset + || newCleanupOffset > localLogEndOffset) { + LOG.warn( + "Ignore invalid historical cleanup offset {} for {} with " + + "current cleanup offset {} and local log end " + + "offset {}.", + newCleanupOffset, + tableBucket, + currentCleanupOffset, + localLogEndOffset); + return; + } + beforeUpdate.run(); + currentKvTablet.advanceHistoricalCleanupOffset(newCleanupOffset); + }); + } + private void validateHistoricalWrite(int expectedLeaderEpoch, int requiredAcks) { if (!isLeader()) { throw new NotLeaderOrFollowerException( diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java index 327c086ba24..b69bee9eed0 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java @@ -1393,7 +1393,8 @@ public void notifyLakeTableOffset( lakeBucketOffsets.entrySet()) { TableBucket tb = lakeBucketOffsetEntry.getKey(); LakeBucketOffset lakeBucketOffset = lakeBucketOffsetEntry.getValue(); - LogTablet logTablet = getReplicaOrException(tb).getLogTablet(); + Replica replica = getReplicaOrException(tb); + LogTablet logTablet = replica.getLogTablet(); logTablet.updateLakeTableSnapshotId(lakeBucketOffset.getSnapshotId()); lakeBucketOffset @@ -1402,7 +1403,16 @@ public void notifyLakeTableOffset( lakeBucketOffset .getLogEndOffset() - .ifPresent(logTablet::updateLakeLogEndOffset); + .ifPresent( + lakeLogEndOffset -> { + logTablet.updateLakeLogEndOffset(lakeLogEndOffset); + if (replica.isHistoricalPartition()) { + historicalPartitionManager.onLakeProgress( + replica, + lakeBucketOffset.getSnapshotId(), + lakeLogEndOffset); + } + }); lakeBucketOffset .getMaxTimestamp() diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java index fabb6685f9d..d62e49ba1c0 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java @@ -37,12 +37,10 @@ import org.apache.fluss.server.entity.PutKvDataForBucket; import org.apache.fluss.server.kv.KvStateLookupResult; import org.apache.fluss.server.kv.KvStateLookupResult.Status; -import org.apache.fluss.server.kv.historical.HistoricalValueLookup; import org.apache.fluss.server.log.LogAppendInfo; import org.apache.fluss.server.replica.Replica; import org.apache.fluss.server.storage.LocalDiskManager; import org.apache.fluss.utils.ByteArraySlice; -import org.apache.fluss.utils.ByteArrayWrapper; import org.apache.fluss.utils.concurrent.Scheduler; import javax.annotation.Nullable; @@ -50,10 +48,8 @@ import java.io.File; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.concurrent.CompletableFuture; import static org.apache.fluss.utils.Preconditions.checkNotNull; @@ -189,6 +185,16 @@ public void requireLakeSnapshot(long tableId, long lakeSnapshotId) { lakeLookupManager.requireLakeSnapshot(tableId, lakeSnapshotId); } + /** Publishes lake coverage used by natural RocksDB compaction for a historical leader. */ + public void onLakeProgress(Replica replica, long lakeSnapshotId, long lakeLogEndOffset) { + replica.tryUpdateHistoricalCleanupOffset( + lakeLogEndOffset, + () -> + // Future fallback lookups must require the covering snapshot before local + // entries become eligible for physical removal. + requireLakeSnapshot(replica.getTableBucket().getTableId(), lakeSnapshotId)); + } + /** Returns the number of accepted historical operations that have not completed. */ public int numInflightRequests() { return taskExecutor.numInflightRequests(); @@ -225,56 +231,23 @@ LogAppendInfo processPut( ResolvedPartitionSpec.fromPartitionName( tableInfo.getPartitionKeys(), originalPartitionName); // The public put path holds the TableBucket ordering slot until processPut returns, so - // local state cannot be changed by a later historical write between resolve and apply. + // local state cannot be changed by a later historical write between lookup and apply. int expectedLeaderEpoch = replica.getLeaderEpoch(); - List keysRequiringLakeLookup = - replica.findKeysRequiringLakeLookup( - putData.records(), - targetColumns, - mergeMode, - originalPartitionName, - expectedLeaderEpoch, - requiredAcks); - - Map lakeResults = new HashMap<>(); - if (!keysRequiringLakeLookup.isEmpty()) { - List lakeValues = - lakeLookupManager.lookup( - new LookupDataForBucket( - putData.tableBucket(), - keysRequiringLakeLookup, - originalPartitionName), - tableInfo, - replica.getLatestSchemaInfo(), - originalPartitionSpec, - replica.tableMetrics()::recordHistoricalLakeLookup); - for (int i = 0; i < keysRequiringLakeLookup.size(); i++) { - byte[] lakeValue = lakeValues.get(i); - lakeResults.put( - new ByteArrayWrapper(keysRequiringLakeLookup.get(i)), - lakeValue == null - ? KvStateLookupResult.notFound() - : KvStateLookupResult.present(lakeValue)); - } - } - - HistoricalValueLookup memoizedLakeLookup = - primaryKey -> { - KvStateLookupResult result = - checkNotNull( - lakeResults.get(new ByteArrayWrapper(primaryKey)), - "No resolved lake value for a historical write key"); - return result.value(); - }; - - // TODO: Tag historical values and tombstones with WAL offsets for incremental cleanup; see - // https://github.com/apache/fluss/issues/4159. return replica.putHistoricalRecordsToLeader( putData.records(), targetColumns, mergeMode, originalPartitionName, - memoizedLakeLookup, + lakeLookupKeys -> + lakeLookupManager.lookup( + new LookupDataForBucket( + putData.tableBucket(), + lakeLookupKeys, + originalPartitionName), + tableInfo, + replica.getLatestSchemaInfo(), + originalPartitionSpec, + replica.tableMetrics()::recordHistoricalLakeLookup), expectedLeaderEpoch, requiredAcks); } @@ -342,15 +315,13 @@ private LookupResultForBucket lookupInternal( Iterator lakeValueIterator = lakeValues.iterator(); List values = new ArrayList<>(localResults.size()); - KvValueLayout localValueLayout = - KvValueLayout.fromTableConfig(tableInfo.getTableConfig()); for (KvStateLookupResult localResult : localResults) { // Consume one lake value for each NOT_FOUND result; local values and tombstones // keep their original positions without advancing the lake iterator. if (localResult.status() == Status.NOT_FOUND) { values.add(KvValueLayout.PLAIN.toValueBodySlice(lakeValueIterator.next())); } else { - values.add(localValueLayout.toValueBodySlice(localResult.value())); + values.add(KvValueLayout.TAGGED.toValueBodySlice(localResult.value())); } } return new LookupResultForBucket(tableBucket, values, originalPartitionName); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/HistoricalKvCompactionFilterTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/HistoricalKvCompactionFilterTest.java new file mode 100644 index 00000000000..9a17e2bf2ae --- /dev/null +++ b/fluss-server/src/test/java/org/apache/fluss/server/kv/HistoricalKvCompactionFilterTest.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.kv; + +import org.apache.fluss.record.BinaryValue; +import org.apache.fluss.rocksdb.RocksDBHandle; +import org.apache.fluss.row.BinaryRow; +import org.apache.fluss.row.encode.KvValueLayout; +import org.apache.fluss.row.encode.ValueEncoder; +import org.apache.fluss.server.kv.historical.HistoricalKvTombstone; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.rocksdb.ColumnFamilyOptions; +import org.rocksdb.DBOptions; +import org.rocksdb.FlinkCompactionFilter; +import org.rocksdb.FlushOptions; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.concurrent.atomic.AtomicLong; + +import static org.apache.fluss.record.TestData.DATA1_ROW_TYPE; +import static org.apache.fluss.record.TestData.DEFAULT_SCHEMA_ID; +import static org.apache.fluss.testutils.DataTestUtils.compactedRow; +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests exclusive-offset cleanup for offset-tagged historical values and tombstones. */ +class HistoricalKvCompactionFilterTest { + + @TempDir private Path tempDir; + + @Test + void testRemovesOnlyOffsetsBelowCleanupOffset() throws Exception { + AtomicLong cleanupOffset = new AtomicLong(0L); + byte[] valueBeforeKey = bytes("value-before"); + byte[] tombstoneBeforeKey = bytes("tombstone-before"); + byte[] valueAtKey = bytes("value-at"); + byte[] tombstoneAtKey = bytes("tombstone-at"); + byte[] valueAfterKey = bytes("value-after"); + byte[] tombstoneAfterKey = bytes("tombstone-after"); + BinaryRow row = compactedRow(DATA1_ROW_TYPE, new Object[] {1, "a"}); + + try (FlinkCompactionFilter.FlinkCompactionFilterFactory filterFactory = + RowTtlCompactionFilterFactory.create( + KvValueLayout.TAGGED, 0L, 1L, () -> cleanupOffset.get() - 1L); + DBOptions dbOptions = new DBOptions().setCreateIfMissing(true); + ColumnFamilyOptions cfOptions = + new ColumnFamilyOptions().setCompactionFilterFactory(filterFactory); + RocksDBHandle handle = new RocksDBHandle(tempDir.toFile(), dbOptions, cfOptions); + FlushOptions flushOptions = new FlushOptions().setWaitForFlush(true)) { + handle.openDB(); + handle.getDb().put(valueBeforeKey, encodeValue(row, 4L)); + handle.getDb().put(tombstoneBeforeKey, HistoricalKvTombstone.encode(4L)); + handle.getDb().put(valueAtKey, encodeValue(row, 5L)); + handle.getDb().put(tombstoneAtKey, HistoricalKvTombstone.encode(5L)); + handle.getDb().put(valueAfterKey, encodeValue(row, 6L)); + handle.getDb().put(tombstoneAfterKey, HistoricalKvTombstone.encode(6L)); + handle.getDb().flush(flushOptions); + + handle.getDb().compactRange(); + assertThat(handle.getDb().get(valueBeforeKey)).isNotNull(); + assertThat(handle.getDb().get(tombstoneBeforeKey)).isNotNull(); + + cleanupOffset.set(5L); + handle.getDb().compactRange(); + + assertThat(handle.getDb().get(valueBeforeKey)).isNull(); + assertThat(handle.getDb().get(tombstoneBeforeKey)).isNull(); + assertThat(handle.getDb().get(valueAtKey)).isNotNull(); + assertThat(handle.getDb().get(tombstoneAtKey)).isNotNull(); + assertThat(handle.getDb().get(valueAfterKey)).isNotNull(); + assertThat(handle.getDb().get(tombstoneAfterKey)).isNotNull(); + } + } + + private static byte[] encodeValue(BinaryRow row, long logOffset) { + return ValueEncoder.forLayout(KvValueLayout.TAGGED) + .encodeValue(new BinaryValue(DEFAULT_SCHEMA_ID, row), logOffset); + } + + private static byte[] bytes(String value) { + return value.getBytes(StandardCharsets.UTF_8); + } +} diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/RowTtlCompactionFilterTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/RowTtlCompactionFilterTest.java index 762b3756c2e..644922c6d5a 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/kv/RowTtlCompactionFilterTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/kv/RowTtlCompactionFilterTest.java @@ -56,10 +56,7 @@ void testFlinkCompactionFilterReadsTimestampFromTaggedValue() throws Exception { try (FlinkCompactionFilter.FlinkCompactionFilterFactory filterFactory = RowTtlCompactionFilterFactory.create( - KvValueLayout.TAGGED, - Duration.ofHours(1L), - 1L, - new ManualClock(now)); + KvValueLayout.TAGGED, Duration.ofHours(1L), new ManualClock(now)); DBOptions dbOptions = new DBOptions().setCreateIfMissing(true); ColumnFamilyOptions cfOptions = new ColumnFamilyOptions().setCompactionFilterFactory(filterFactory); @@ -83,10 +80,7 @@ void testCreateRejectsInvalidTtlDuration() { assertThatThrownBy( () -> RowTtlCompactionFilterFactory.create( - KvValueLayout.TAGGED, - Duration.ZERO, - 1L, - new ManualClock(0L))) + KvValueLayout.TAGGED, Duration.ZERO, new ManualClock(0L))) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining(ConfigOptions.TABLE_KV_TTL.key()); } diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/historical/LocalPreviousValueLookupResultTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/historical/LocalPreviousValueLookupResultTest.java new file mode 100644 index 00000000000..6622152162d --- /dev/null +++ b/fluss-server/src/test/java/org/apache/fluss/server/kv/historical/LocalPreviousValueLookupResultTest.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.kv.historical; + +import org.apache.fluss.metadata.KvFormat; +import org.apache.fluss.record.BinaryValue; +import org.apache.fluss.record.TestingSchemaGetter; +import org.apache.fluss.row.BinaryRow; +import org.apache.fluss.row.encode.KvValueLayout; +import org.apache.fluss.row.encode.ValueDecoder; +import org.apache.fluss.row.encode.ValueEncoder; +import org.apache.fluss.server.kv.KvStateLookupResult; + +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; + +import static org.apache.fluss.record.TestData.DATA1_ROW_TYPE; +import static org.apache.fluss.record.TestData.DATA1_SCHEMA; +import static org.apache.fluss.record.TestData.DEFAULT_SCHEMA_ID; +import static org.apache.fluss.testutils.DataTestUtils.compactedRow; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for {@link LocalPreviousValueLookupResult}. */ +class LocalPreviousValueLookupResultTest { + + @Test + void testCreatesLookupFromLocalAndLakeResults() { + LocalPreviousValueLookupResult localLookupResult = createLookupResult(); + byte[] localValueKey = new byte[] {1}; + byte[] localDeleteKey = new byte[] {2}; + byte[] lakeValueKey = new byte[] {3}; + byte[] lakeMissKey = new byte[] {4}; + BinaryValue localValue = binaryValue(1, "local"); + BinaryValue lakeValue = binaryValue(3, "lake"); + + localLookupResult.add( + localValueKey, + KvStateLookupResult.present( + ValueEncoder.forLayout(KvValueLayout.TAGGED).encodeValue(localValue, 10L))); + localLookupResult.add(localDeleteKey, KvStateLookupResult.deleted()); + localLookupResult.add(lakeValueKey, KvStateLookupResult.notFound()); + localLookupResult.add(lakeMissKey, KvStateLookupResult.notFound()); + + assertThat(localLookupResult.hasLocalMisses()).isTrue(); + assertThat(localLookupResult.keysMissingLocally()) + .containsExactly(lakeValueKey, lakeMissKey); + + HistoricalValueLookup valueLookup = + localLookupResult.createValueLookup( + Arrays.asList( + ValueEncoder.forLayout(KvValueLayout.PLAIN).encodeValue(lakeValue), + null)); + + assertThat(valueLookup.lookup(localValueKey)).isEqualTo(localValue); + assertThat(valueLookup.lookup(localDeleteKey)).isNull(); + assertThat(valueLookup.lookup(lakeValueKey)).isEqualTo(lakeValue); + assertThat(valueLookup.lookup(lakeMissKey)).isNull(); + assertThatThrownBy(() -> valueLookup.lookup(new byte[] {5})) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("No previous value for a historical write key"); + } + + @Test + void testValidatesLakeResultCountBeforeCreatingLookup() { + LocalPreviousValueLookupResult localLookupResult = createLookupResult(); + localLookupResult.add(new byte[] {1}, KvStateLookupResult.notFound()); + + assertThatThrownBy(() -> localLookupResult.createValueLookup(Collections.emptyList())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Expected 1 historical lake values, but received 0"); + } + + private static LocalPreviousValueLookupResult createLookupResult() { + TestingSchemaGetter schemaGetter = new TestingSchemaGetter(DEFAULT_SCHEMA_ID, DATA1_SCHEMA); + return new LocalPreviousValueLookupResult( + new ValueDecoder(schemaGetter, KvFormat.COMPACTED, KvValueLayout.TAGGED), + new ValueDecoder(schemaGetter, KvFormat.COMPACTED, KvValueLayout.PLAIN)); + } + + private static BinaryValue binaryValue(int id, String value) { + BinaryRow row = compactedRow(DATA1_ROW_TYPE, new Object[] {id, value}); + return new BinaryValue(DEFAULT_SCHEMA_ID, row); + } +} diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java index b18f96b3a7a..2de12ea77e9 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java @@ -25,6 +25,7 @@ import org.apache.fluss.exception.HistoricalPartitionThrottledException; import org.apache.fluss.exception.InvalidPartitionException; import org.apache.fluss.lake.lakestorage.LakeTableLookuper; +import org.apache.fluss.memory.MemorySegment; import org.apache.fluss.metadata.ChangelogImage; import org.apache.fluss.metadata.DataLakeFormat; import org.apache.fluss.metadata.PhysicalTablePath; @@ -50,16 +51,21 @@ import org.apache.fluss.rpc.entity.FetchLogResultForBucket; import org.apache.fluss.rpc.entity.LookupResultForBucket; import org.apache.fluss.rpc.entity.PutKvResultForBucket; +import org.apache.fluss.rpc.messages.NotifyLakeTableOffsetResponse; import org.apache.fluss.rpc.protocol.ApiKeys; import org.apache.fluss.rpc.protocol.Errors; import org.apache.fluss.rpc.protocol.MergeMode; import org.apache.fluss.server.entity.FetchReqInfo; +import org.apache.fluss.server.entity.LakeBucketOffset; import org.apache.fluss.server.entity.LookupDataForBucket; +import org.apache.fluss.server.entity.NotifyLakeTableOffsetData; import org.apache.fluss.server.entity.NotifyLeaderAndIsrData; import org.apache.fluss.server.entity.NotifyLeaderAndIsrResultForBucket; import org.apache.fluss.server.entity.PutKvDataForBucket; import org.apache.fluss.server.kv.KvStateLookupResult; import org.apache.fluss.server.kv.KvTablet; +import org.apache.fluss.server.kv.historical.HistoricalKvKeyEncoder; +import org.apache.fluss.server.kv.historical.HistoricalKvTombstone; import org.apache.fluss.server.log.FetchParams; import org.apache.fluss.server.log.LogAppendInfo; import org.apache.fluss.server.metadata.BucketMetadata; @@ -83,9 +89,11 @@ import com.github.benmanes.caffeine.cache.Scheduler; import com.github.benmanes.caffeine.cache.Ticker; import org.junit.jupiter.api.Test; +import org.rocksdb.FlushOptions; import javax.annotation.Nullable; +import java.io.IOException; import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; @@ -144,6 +152,7 @@ void testResolvesMultipleLakeMissesWithoutPrewriteRollback() throws Exception { CompactedKeyEncoder keyEncoder = new CompactedKeyEncoder(keyType); byte[] firstKey = keyEncoder.encodeKey(row(1, "us")); byte[] secondKey = keyEncoder.encodeKey(row(2, "eu")); + byte[] thirdKey = keyEncoder.encodeKey(row(3, "ap")); KvRecordBatch insertBatch = batch( keyType, @@ -158,7 +167,13 @@ void testResolvesMultipleLakeMissesWithoutPrewriteRollback() throws Exception { new Object[] {1, "us", ORIGINAL_PARTITION, "v1-updated"}), Tuple2.of( new Object[] {2, "eu"}, - new Object[] {2, "eu", ORIGINAL_PARTITION, "v2"})); + new Object[] {2, "eu", ORIGINAL_PARTITION, "v2"}), + // The absent delete is a no-op. The following upsert must reuse the + // resolved absence instead of treating this key as already staged. + Tuple2.of(new Object[] {3, "ap"}, null), + Tuple2.of( + new Object[] {3, "ap"}, + new Object[] {3, "ap", ORIGINAL_PARTITION, "v3"})); long truncateCount = replicaManager.getServerMetricGroup().kvTruncateAsErrorCount().getCount(); @@ -170,7 +185,7 @@ void testResolvesMultipleLakeMissesWithoutPrewriteRollback() throws Exception { MergeMode.DEFAULT, 1); - assertThat(lakeLookupManager.lookupCount).hasValue(2); + assertThat(lakeLookupManager.lookupCount).hasValue(3); assertThat(lakeLookupManager.lookupBatchCount).hasValue(1); assertThat(replicaManager.getServerMetricGroup().kvTruncateAsErrorCount().getCount()) .isEqualTo(truncateCount); @@ -187,6 +202,118 @@ void testResolvesMultipleLakeMissesWithoutPrewriteRollback() throws Exception { secondKey, tableInfo, row(2, "eu", ORIGINAL_PARTITION, "v2")); + assertHistoricalValue( + kvTablet, + ORIGINAL_PARTITION, + thirdKey, + tableInfo, + row(3, "ap", ORIGINAL_PARTITION, "v3")); + } finally { + historicalPartitionManager.close(); + } + } + + @Test + void testReusesLocalProbeResultsWhenEntriesDisappearBeforeApply() throws Exception { + TableInfo tableInfo = registerHistoricalTableAndBecomeLeader(); + Replica replica = replicaManager.getReplicaOrException(TABLE_BUCKET); + KvTablet kvTablet = replica.getKvTablet(); + assertThat(kvTablet).isNotNull(); + TestingHistoricalLakeLookupManager lakeLookupManager = + new TestingHistoricalLakeLookupManager(lookupConfiguration()); + HistoricalPartitionManager historicalPartitionManager = + new HistoricalPartitionManager( + new HistoricalPartitionTaskExecutor(lookupConfiguration()), + lakeLookupManager); + + RowType keyType = + DataTypes.ROW( + new DataField("id", DataTypes.INT()), + new DataField("region", DataTypes.STRING())); + RowType rowType = tableInfo.getRowType(); + CompactedKeyEncoder keyEncoder = new CompactedKeyEncoder(keyType); + byte[] valueKey = keyEncoder.encodeKey(row(1, "us")); + byte[] tombstoneKey = keyEncoder.encodeKey(row(2, "eu")); + byte[] missingKey = keyEncoder.encodeKey(row(3, "ap")); + + try { + // Seed exactly the two local states that compaction can remove between phases. + kvTablet.getRocksDBKv() + .put( + HistoricalKvKeyEncoder.encode(ORIGINAL_PARTITION, valueKey), + ValueEncoder.forLayout(KvValueLayout.TAGGED) + .encodeValue( + new BinaryValue( + (short) tableInfo.getSchemaId(), + compactedRow( + rowType, + new Object[] { + 1, "us", ORIGINAL_PARTITION, "v1" + })), + 0L)); + kvTablet.getRocksDBKv() + .put( + HistoricalKvKeyEncoder.encode(ORIGINAL_PARTITION, tombstoneKey), + HistoricalKvTombstone.encode(1L)); + + // The third key forces lake I/O after all three keys have been probed. Remove the + // value and tombstone during that I/O to model compaction between probe and apply. + lakeLookupManager.setLookupHook( + () -> { + deleteHistoricalRocksDbKey(kvTablet, valueKey); + deleteHistoricalRocksDbKey(kvTablet, tombstoneKey); + }); + + LogAppendInfo appendInfo = + historicalPartitionManager.processPut( + replica, + new PutKvDataForBucket( + TABLE_BUCKET, + batch( + keyType, + rowType, + Tuple2.of( + new Object[] {1, "us"}, + new Object[] { + 1, "us", ORIGINAL_PARTITION, "v2" + }), + Tuple2.of( + new Object[] {2, "eu"}, + new Object[] { + 2, "eu", ORIGINAL_PARTITION, "recreated" + }), + Tuple2.of( + new Object[] {3, "ap"}, + new Object[] { + 3, "ap", ORIGINAL_PARTITION, "inserted" + })), + ORIGINAL_PARTITION), + null, + MergeMode.DEFAULT, + 1); + + assertThat(appendInfo.numMessages()).isEqualTo(4); + assertThat(lakeLookupManager.lookupCount).hasValue(1); + assertThat(lakeLookupManager.lookupBatchCount).hasValue(1); + flushAndWait(kvTablet, Long.MAX_VALUE); + assertHistoricalValue( + kvTablet, + ORIGINAL_PARTITION, + valueKey, + tableInfo, + row(1, "us", ORIGINAL_PARTITION, "v2")); + assertHistoricalValue( + kvTablet, + ORIGINAL_PARTITION, + tombstoneKey, + tableInfo, + row(2, "eu", ORIGINAL_PARTITION, "recreated")); + assertHistoricalValue( + kvTablet, + ORIGINAL_PARTITION, + missingKey, + tableInfo, + row(3, "ap", ORIGINAL_PARTITION, "inserted")); } finally { historicalPartitionManager.close(); } @@ -430,6 +557,221 @@ void testHistoricalInsertUpdateAndDelete() throws Exception { } } + @Test + void testHistoricalMutationsUseProducingWalOffsetsAsTags() throws Exception { + TableInfo tableInfo = registerHistoricalTableAndBecomeLeader(); + Replica replica = replicaManager.getReplicaOrException(TABLE_BUCKET); + KvTablet kvTablet = replica.getKvTablet(); + assertThat(kvTablet).isNotNull(); + HistoricalPartitionManager historicalPartitionManager = + new HistoricalPartitionManager( + new HistoricalPartitionTaskExecutor(lookupConfiguration()), + new TestingHistoricalLakeLookupManager(lookupConfiguration())); + RowType keyType = + DataTypes.ROW( + new DataField("id", DataTypes.INT()), + new DataField("region", DataTypes.STRING())); + RowType rowType = tableInfo.getRowType(); + byte[] primaryKey = new CompactedKeyEncoder(keyType).encodeKey(row(1, "us")); + + try { + LogAppendInfo insertAppend = + historicalPartitionManager.processPut( + replica, + new PutKvDataForBucket( + TABLE_BUCKET, + batch( + keyType, + rowType, + Tuple2.of( + new Object[] {1, "us"}, + new Object[] { + 1, "us", ORIGINAL_PARTITION, "v1" + })), + ORIGINAL_PARTITION), + null, + MergeMode.DEFAULT, + 1); + flushAndWait(kvTablet, Long.MAX_VALUE); + assertHistoricalValueTag( + kvTablet, ORIGINAL_PARTITION, primaryKey, insertAppend.lastOffset()); + + LogAppendInfo updateAppend = + historicalPartitionManager.processPut( + replica, + new PutKvDataForBucket( + TABLE_BUCKET, + batch( + keyType, + rowType, + Tuple2.of( + new Object[] {1, "us"}, + new Object[] { + 1, "us", ORIGINAL_PARTITION, "v2" + })), + ORIGINAL_PARTITION), + null, + MergeMode.DEFAULT, + 1); + assertThat(updateAppend.numMessages()).isEqualTo(2); + flushAndWait(kvTablet, Long.MAX_VALUE); + // In FULL changelog mode, UPDATE_AFTER is the second WAL record and produces state. + assertHistoricalValueTag( + kvTablet, ORIGINAL_PARTITION, primaryKey, updateAppend.lastOffset()); + + LogAppendInfo deleteAppend = + historicalPartitionManager.processPut( + replica, + new PutKvDataForBucket( + TABLE_BUCKET, + batch( + keyType, + rowType, + Tuple2.of(new Object[] {1, "us"}, null)), + ORIGINAL_PARTITION), + null, + MergeMode.DEFAULT, + 1); + flushAndWait(kvTablet, Long.MAX_VALUE); + byte[] tombstone = historicalRawValue(kvTablet, ORIGINAL_PARTITION, primaryKey); + assertThat(tombstone).hasSize(Long.BYTES); + assertHistoricalValueTag( + kvTablet, ORIGINAL_PARTITION, primaryKey, deleteAppend.lastOffset()); + } finally { + historicalPartitionManager.close(); + } + } + + @Test + void testCompactionUsesPublishedLakeSnapshot() throws Exception { + TableInfo tableInfo = registerHistoricalTableAndBecomeLeader(); + Replica replica = replicaManager.getReplicaOrException(TABLE_BUCKET); + KvTablet kvTablet = replica.getKvTablet(); + assertThat(kvTablet).isNotNull(); + TestingHistoricalLakeLookupManager lakeLookupManager = + new TestingHistoricalLakeLookupManager(lookupConfiguration()); + HistoricalPartitionManager historicalPartitionManager = + new HistoricalPartitionManager( + new HistoricalPartitionTaskExecutor(lookupConfiguration()), + lakeLookupManager); + RowType keyType = + DataTypes.ROW( + new DataField("id", DataTypes.INT()), + new DataField("region", DataTypes.STRING())); + byte[] coveredKey = new CompactedKeyEncoder(keyType).encodeKey(row(1, "us")); + byte[] retainedKey = new CompactedKeyEncoder(keyType).encodeKey(row(2, "eu")); + + try { + historicalPartitionManager.processPut( + replica, + new PutKvDataForBucket( + TABLE_BUCKET, + batch( + keyType, + tableInfo.getRowType(), + Tuple2.of( + new Object[] {1, "us"}, + new Object[] {1, "us", ORIGINAL_PARTITION, "covered"})), + ORIGINAL_PARTITION), + null, + MergeMode.DEFAULT, + 1); + historicalPartitionManager.processPut( + replica, + new PutKvDataForBucket( + TABLE_BUCKET, + batch( + keyType, + tableInfo.getRowType(), + Tuple2.of( + new Object[] {2, "eu"}, + new Object[] { + 2, "eu", ORIGINAL_PARTITION, "retained" + })), + ORIGINAL_PARTITION), + null, + MergeMode.DEFAULT, + 1); + flushAndWait(kvTablet, Long.MAX_VALUE); + assertHistoricalValueTag(kvTablet, ORIGINAL_PARTITION, coveredKey, 0L); + assertHistoricalValueTag(kvTablet, ORIGINAL_PARTITION, retainedKey, 1L); + + lakeLookupManager.setRequireSnapshotHook( + () -> assertThat(kvTablet.getHistoricalCleanupOffset()).isZero()); + historicalPartitionManager.onLakeProgress(replica, 7L, 1L); + + assertThat(lakeLookupManager.requiredSnapshotCount).hasValue(1); + assertThat(kvTablet.getHistoricalCleanupOffset()).isOne(); + compactHistoricalKv(kvTablet); + assertThat(historicalRawValue(kvTablet, ORIGINAL_PARTITION, coveredKey)).isNull(); + assertHistoricalValueTag(kvTablet, ORIGINAL_PARTITION, retainedKey, 1L); + + // A miss caused by cleanup must enter lake lookup only after the covering snapshot + // token has been published. + lakeLookupManager.putLakeValue( + ORIGINAL_PARTITION, + ValueEncoder.encodeValue( + (short) tableInfo.getSchemaId(), + compactedRow( + tableInfo.getRowType(), + new Object[] { + 1, "us", ORIGINAL_PARTITION, "covered-from-lake" + }))); + lakeLookupManager.setLookupHook( + () -> { + assertThat(lakeLookupManager.requiredSnapshotCount).hasValue(1); + assertThat(kvTablet.getHistoricalCleanupOffset()).isOne(); + }); + LookupResultForBucket fallbackResult = + historicalPartitionManager + .lookup( + replica, + new LookupDataForBucket( + TABLE_BUCKET, + Collections.singletonList(coveredKey), + ORIGINAL_PARTITION), + (lookupTimeNanos, lookupFileDownloaded) -> {}) + .get(10, TimeUnit.SECONDS); + assertThat(fallbackResult.failed()).isFalse(); + BinaryValue fallbackValue = + new ValueDecoder( + schemaGetter(tableInfo), + tableInfo.getTableConfig().getKvFormat(), + KvValueLayout.PLAIN) + .decodeValue(fallbackResult.lookupValues().get(0).toByteArray()); + assertThat(fallbackValue.row.getString(3)) + .isEqualTo(BinaryString.fromString("covered-from-lake")); + + // A new opaque snapshot token at the same offset is still published to the lookuper. + lakeLookupManager.setRequireSnapshotHook( + () -> assertThat(kvTablet.getHistoricalCleanupOffset()).isOne()); + historicalPartitionManager.onLakeProgress(replica, 8L, 1L); + assertThat(lakeLookupManager.requiredSnapshotCount).hasValue(2); + assertThat(kvTablet.getHistoricalCleanupOffset()).isOne(); + + // A regressing cleanup offset is ignored before it can publish an older required + // snapshot. + historicalPartitionManager.onLakeProgress(replica, 9L, 0L); + assertThat(lakeLookupManager.requiredSnapshotCount).hasValue(2); + assertThat(kvTablet.getHistoricalCleanupOffset()).isOne(); + + CompletableFuture notifyFuture = + new CompletableFuture<>(); + replicaManager.notifyLakeTableOffset( + new NotifyLakeTableOffsetData( + 1, + Collections.singletonMap( + TABLE_BUCKET, new LakeBucketOffset(10L, null, 2L, null))), + notifyFuture::complete); + notifyFuture.get(10, TimeUnit.SECONDS); + assertThat(kvTablet.getHistoricalCleanupOffset()).isEqualTo(2L); + compactHistoricalKv(kvTablet); + assertThat(historicalRawValue(kvTablet, ORIGINAL_PARTITION, retainedKey)).isNull(); + } finally { + historicalPartitionManager.close(); + } + } + @Test void testUpdateAndDeleteFromLakeFallback() throws Exception { TableInfo tableInfo = registerHistoricalTableAndBecomeLeader(); @@ -628,22 +970,26 @@ void testRecoversHistoricalOverlayFromLakeCommitOffset() throws Exception { null, MergeMode.DEFAULT, 1); - historicalPartitionManager.processPut( - replica, - new PutKvDataForBucket( - TABLE_BUCKET, - batch( - keyType, - tableInfo.getRowType(), - Tuple2.of( - new Object[] {1, "us"}, - new Object[] { - 1, "us", ANOTHER_ORIGINAL_PARTITION, "another" - })), - ANOTHER_ORIGINAL_PARTITION), - null, - MergeMode.DEFAULT, - 1); + LogAppendInfo anotherPartitionAppend = + historicalPartitionManager.processPut( + replica, + new PutKvDataForBucket( + TABLE_BUCKET, + batch( + keyType, + tableInfo.getRowType(), + Tuple2.of( + new Object[] {1, "us"}, + new Object[] { + 1, + "us", + ANOTHER_ORIGINAL_PARTITION, + "another" + })), + ANOTHER_ORIGINAL_PARTITION), + null, + MergeMode.DEFAULT, + 1); historicalPartitionManager.processPut( replica, new PutKvDataForBucket( @@ -660,18 +1006,19 @@ void testRecoversHistoricalOverlayFromLakeCommitOffset() throws Exception { null, MergeMode.DEFAULT, 1); - historicalPartitionManager.processPut( - replica, - new PutKvDataForBucket( - TABLE_BUCKET, - batch( - keyType, - tableInfo.getRowType(), - Tuple2.of(new Object[] {2, "eu"}, null)), - ORIGINAL_PARTITION), - null, - MergeMode.DEFAULT, - 1); + LogAppendInfo deleteAppend = + historicalPartitionManager.processPut( + replica, + new PutKvDataForBucket( + TABLE_BUCKET, + batch( + keyType, + tableInfo.getRowType(), + Tuple2.of(new Object[] {2, "eu"}, null)), + ORIGINAL_PARTITION), + null, + MergeMode.DEFAULT, + 1); KvTablet kvTabletBeforeFollower = replica.getKvTablet(); assertThat(kvTabletBeforeFollower).isNotNull(); flushAndWait(kvTabletBeforeFollower, Long.MAX_VALUE); @@ -704,6 +1051,10 @@ void testRecoversHistoricalOverlayFromLakeCommitOffset() throws Exception { KvTablet recoveredKvTablet = replica.getKvTablet(); assertThat(recoveredKvTablet).isNotNull(); assertThat(replica.getLakeLogEndOffset()).isEqualTo(lakeCommitOffset); + assertThat(recoveredKvTablet.getHistoricalCleanupOffset()).isEqualTo(lakeCommitOffset); + // Recovery retries can reuse the same tablet and restore offset. + assertThat(recoveredKvTablet.advanceHistoricalCleanupOffset(lakeCommitOffset)) + .isFalse(); assertThat(replica.getKvSnapshotManager()).isNull(); assertThat(recoveredKvTablet.getFlushedLogOffset()) .isEqualTo(replica.getLogHighWatermark()); @@ -723,6 +1074,16 @@ void testRecoversHistoricalOverlayFromLakeCommitOffset() throws Exception { tieredPrimaryKey, tableInfo, row(1, "us", ANOTHER_ORIGINAL_PARTITION, "another")); + assertHistoricalValueTag( + recoveredKvTablet, + ANOTHER_ORIGINAL_PARTITION, + tieredPrimaryKey, + anotherPartitionAppend.lastOffset()); + assertHistoricalValueTag( + recoveredKvTablet, + ORIGINAL_PARTITION, + deletedPrimaryKey, + deleteAppend.lastOffset()); } finally { historicalPartitionManager.close(); } @@ -957,11 +1318,42 @@ private static void assertHistoricalValue( new ValueDecoder( schemaGetter(tableInfo), tableInfo.getTableConfig().getKvFormat(), - KvValueLayout.fromTableConfig(tableInfo.getTableConfig())) + KvValueLayout.TAGGED) .decodeValue(result.value()); assertThatRow(value.row).withSchema(tableInfo.getRowType()).isEqualTo(expectedRow); } + private static void assertHistoricalValueTag( + KvTablet kvTablet, String originalPartition, byte[] primaryKey, long expectedLogOffset) + throws IOException { + byte[] rawValue = historicalRawValue(kvTablet, originalPartition, primaryKey); + assertThat(rawValue).isNotNull(); + assertThat(KvValueLayout.TAGGED.readValueTag(MemorySegment.wrap(rawValue))) + .isEqualTo(expectedLogOffset); + } + + private static byte[] historicalRawValue( + KvTablet kvTablet, String originalPartition, byte[] primaryKey) throws IOException { + return kvTablet.getRocksDBKv() + .get(HistoricalKvKeyEncoder.encode(originalPartition, primaryKey)); + } + + private static void compactHistoricalKv(KvTablet kvTablet) throws Exception { + try (FlushOptions flushOptions = new FlushOptions().setWaitForFlush(true)) { + kvTablet.getRocksDBKv().getDb().flush(flushOptions); + kvTablet.getRocksDBKv().getDb().compactRange(); + } + } + + private static void deleteHistoricalRocksDbKey(KvTablet kvTablet, byte[] primaryKey) { + try { + kvTablet.getRocksDBKv() + .delete(HistoricalKvKeyEncoder.encode(ORIGINAL_PARTITION, primaryKey)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + private static SchemaGetter schemaGetter(TableInfo tableInfo) { return new TestingSchemaGetter( new SchemaInfo(tableInfo.getSchema(), tableInfo.getSchemaId())); @@ -970,8 +1362,10 @@ private static SchemaGetter schemaGetter(TableInfo tableInfo) { private final class TestingHistoricalLakeLookupManager extends HistoricalLakeLookupManager { private final AtomicInteger lookupCount = new AtomicInteger(); private final AtomicInteger lookupBatchCount = new AtomicInteger(); + private final AtomicInteger requiredSnapshotCount = new AtomicInteger(); private final Map lakeValuesByPartition = new HashMap<>(); private volatile @Nullable Runnable lookupHook; + private volatile @Nullable Runnable requireSnapshotHook; private TestingHistoricalLakeLookupManager(Configuration configuration) { super( @@ -992,6 +1386,20 @@ private void setLookupHook(Runnable lookupHook) { this.lookupHook = lookupHook; } + private void setRequireSnapshotHook(Runnable requireSnapshotHook) { + this.requireSnapshotHook = requireSnapshotHook; + } + + @Override + void requireLakeSnapshot(long tableId, long snapshotId) { + super.requireLakeSnapshot(tableId, snapshotId); + requiredSnapshotCount.incrementAndGet(); + Runnable hook = requireSnapshotHook; + if (hook != null) { + hook.run(); + } + } + @Override List lookup( LookupDataForBucket lookupData,