diff --git a/fluss-client/src/main/java/org/apache/fluss/client/utils/ClientRpcMessageUtils.java b/fluss-client/src/main/java/org/apache/fluss/client/utils/ClientRpcMessageUtils.java index 3c7512945dc..1a40afc92be 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/utils/ClientRpcMessageUtils.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/utils/ClientRpcMessageUtils.java @@ -147,6 +147,10 @@ public static ProduceLogRequest makeProduceLogRequest( if (tableBucket.getPartitionId() != null) { pbProduceLogReqForBucket.setPartitionId(tableBucket.getPartitionId()); } + if (readyBatch.writeBatch().getOriginalPartitionName() != null) { + pbProduceLogReqForBucket.setOriginalPartitionName( + readyBatch.writeBatch().getOriginalPartitionName()); + } }); return request; } @@ -202,6 +206,11 @@ public static PutKvRequest makePutKvRequest( if (tableBucket.getPartitionId() != null) { pbPutKvReqForBucket.setPartitionId(tableBucket.getPartitionId()); } + KvWriteBatch kvWriteBatch = (KvWriteBatch) readyBatch.writeBatch(); + if (kvWriteBatch.getOriginalPartitionName() != null) { + pbPutKvReqForBucket.setOriginalPartitionName( + kvWriteBatch.getOriginalPartitionName()); + } }); return request; } diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/AbstractRowLogWriteBatch.java b/fluss-client/src/main/java/org/apache/fluss/client/write/AbstractRowLogWriteBatch.java index 104f8f29e33..d9afe697600 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/AbstractRowLogWriteBatch.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/AbstractRowLogWriteBatch.java @@ -26,6 +26,8 @@ import org.apache.fluss.record.bytesview.BytesView; import org.apache.fluss.row.InternalRow; +import javax.annotation.Nullable; + import java.io.IOException; import java.util.List; @@ -49,11 +51,19 @@ protected AbstractRowLogWriteBatch( PhysicalTablePath physicalTablePath, int schemaId, WriteFormat writeFormat, + @Nullable String originalPartitionName, long createdMs, AbstractPagedOutputView outputView, MemoryLogRecordsRowBuilder recordsBuilder, String buildErrorMessage) { - super(tableId, bucketId, physicalTablePath, schemaId, writeFormat, createdMs); + super( + tableId, + bucketId, + physicalTablePath, + schemaId, + writeFormat, + originalPartitionName, + createdMs); this.outputView = outputView; this.recordsBuilder = recordsBuilder; this.buildErrorMessage = buildErrorMessage; diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/ArrowLogWriteBatch.java b/fluss-client/src/main/java/org/apache/fluss/client/write/ArrowLogWriteBatch.java index a6894b98b2e..4c4f293aa8f 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/ArrowLogWriteBatch.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/ArrowLogWriteBatch.java @@ -58,9 +58,17 @@ public ArrowLogWriteBatch( int schemaId, ArrowWriter arrowWriter, AbstractPagedOutputView outputView, + @Nullable String originalPartitionName, long createdMs, @Nullable LogRecordBatchStatisticsCollector statisticsCollector) { - super(tableId, bucketId, physicalTablePath, schemaId, WriteFormat.ARROW_LOG, createdMs); + super( + tableId, + bucketId, + physicalTablePath, + schemaId, + WriteFormat.ARROW_LOG, + originalPartitionName, + createdMs); this.outputView = outputView; this.recordsBuilder = MemoryLogRecordsArrowBuilder.builder( diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/CompactedLogWriteBatch.java b/fluss-client/src/main/java/org/apache/fluss/client/write/CompactedLogWriteBatch.java index 81bedff0fa0..1e3fc0b8dbc 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/CompactedLogWriteBatch.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/CompactedLogWriteBatch.java @@ -25,6 +25,7 @@ import org.apache.fluss.row.compacted.CompactedRow; import org.apache.fluss.rpc.messages.ProduceLogRequest; +import javax.annotation.Nullable; import javax.annotation.concurrent.NotThreadSafe; import static org.apache.fluss.utils.Preconditions.checkArgument; @@ -46,6 +47,7 @@ public CompactedLogWriteBatch( int schemaId, int writeLimit, AbstractPagedOutputView outputView, + @Nullable String originalPartitionName, long createdMs) { super( tableId, @@ -53,6 +55,7 @@ public CompactedLogWriteBatch( physicalTablePath, schemaId, WriteFormat.COMPACTED_LOG, + originalPartitionName, createdMs, outputView, MemoryLogRecordsCompactedBuilder.builder(schemaId, writeLimit, outputView, true), diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/IndexedLogWriteBatch.java b/fluss-client/src/main/java/org/apache/fluss/client/write/IndexedLogWriteBatch.java index 2bb496cbfe3..2b50402218b 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/IndexedLogWriteBatch.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/IndexedLogWriteBatch.java @@ -24,6 +24,7 @@ import org.apache.fluss.row.indexed.IndexedRow; import org.apache.fluss.rpc.messages.ProduceLogRequest; +import javax.annotation.Nullable; import javax.annotation.concurrent.NotThreadSafe; import static org.apache.fluss.utils.Preconditions.checkArgument; @@ -45,6 +46,7 @@ public IndexedLogWriteBatch( int schemaId, int writeLimit, AbstractPagedOutputView outputView, + @Nullable String originalPartitionName, long createdMs) { super( tableId, @@ -52,6 +54,7 @@ public IndexedLogWriteBatch( physicalTablePath, schemaId, WriteFormat.INDEXED_LOG, + originalPartitionName, createdMs, outputView, MemoryLogRecordsIndexedBuilder.builder(schemaId, writeLimit, outputView, true), diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/KvWriteBatch.java b/fluss-client/src/main/java/org/apache/fluss/client/write/KvWriteBatch.java index 4ca01e133b2..0a5c16db688 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/KvWriteBatch.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/KvWriteBatch.java @@ -63,6 +63,7 @@ public KvWriteBatch( AbstractPagedOutputView outputView, @Nullable int[] targetColumns, MergeMode mergeMode, + @Nullable String originalPartitionName, long createdMs) { super( tableId, @@ -70,6 +71,7 @@ public KvWriteBatch( physicalTablePath, schemaId, WriteFormat.fromKvFormat(kvFormat), + originalPartitionName, createdMs); this.outputView = outputView; this.recordsBuilder = diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java b/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java index ffa18ad9455..9d78ae5da89 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java @@ -32,6 +32,7 @@ import org.apache.fluss.metadata.PhysicalTablePath; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; import org.apache.fluss.metrics.MetricNames; import org.apache.fluss.record.LogRecordBatchStatisticsCollector; import org.apache.fluss.row.arrow.ArrowWriter; @@ -67,6 +68,7 @@ import static org.apache.fluss.record.LogRecordBatchFormat.NO_BATCH_SEQUENCE; import static org.apache.fluss.record.LogRecordBatchFormat.NO_WRITER_ID; import static org.apache.fluss.shaded.arrow.org.apache.arrow.memory.BufferAllocatorUtil.createBufferAllocator; +import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; import static org.apache.fluss.utils.Preconditions.checkNotNull; /* This file is based on source code of Apache Kafka Project (https://kafka.apache.org/), licensed by the Apache @@ -115,6 +117,10 @@ public final class RecordAccumulator { private final ConcurrentMap writeBatches = new CopyOnWriteMap<>(); + /** Whether tables observed by this writer have historical partition support enabled. */ + private final ConcurrentMap historicalPartitionEnabledByTable = + new ConcurrentHashMap<>(); + private final IncompleteBatches incomplete; private final Map nodesDrainIndex; @@ -206,7 +212,9 @@ public RecordAppendResult append( physicalTablePath, k -> new BucketAndWriteBatches( - partitionIdOpt.orElse(null), tableInfo.isPartitioned())); + partitionIdOpt.orElse(null), + tableInfo.isPartitioned(), + physicalTablePath)); // We keep track of the number of appending thread to make sure we do not miss batches in // abortIncompleteBatches(). @@ -273,12 +281,10 @@ public ReadyCheckResult ready(Cluster cluster) { // Go table by table so that we can get queue sizes for buckets in a table and calculate // cumulative frequency table (used in bucket assigner). - for (Map.Entry writeBatchesEntry : - writeBatches.entrySet()) { + for (BucketAndWriteBatches bucketAndWriteBatches : writeBatches.values()) { nextReadyCheckDelayMs = bucketReady( - writeBatchesEntry.getKey(), - writeBatchesEntry.getValue(), + bucketAndWriteBatches, readyNodes, unknownLeaderTables, cluster, @@ -319,6 +325,9 @@ public Map> drain( public void reEnqueue(ReadyWriteBatch readyWriteBatch) { WriteBatch batch = readyWriteBatch.writeBatch(); + if (batch.isDone()) { + return; + } batch.reEnqueued(); Deque deque = getOrCreateDeque(readyWriteBatch.tableBucket(), batch.physicalTablePath()); @@ -331,6 +340,156 @@ public void reEnqueue(ReadyWriteBatch readyWriteBatch) { } } + /** + * Routes writes for an original partition path to the given physical target. + * + *

The accumulator keeps queues keyed by {@code originalPath}, while metadata lookup, leader + * discovery, and RPC sending use {@code targetPath}. The target may therefore be either the + * original partition itself or the shared historical partition. + * + *

A normal target can be replaced by the historical target only while the original path has + * no incomplete batch. This method never moves queued or inflight batches between physical + * partitions. + * + * @throws FlussRuntimeException if a different target was fixed previously + */ + void routeWritesTo( + PhysicalTablePath originalPath, PhysicalTablePath targetPath, long targetPartitionId) { + BucketAndWriteBatches resolvedTarget = + new BucketAndWriteBatches(targetPartitionId, true, targetPath); + // Install the route atomically before append can create the first queue for this path. + BucketAndWriteBatches existing = writeBatches.putIfAbsent(originalPath, resolvedTarget); + if (existing == null) { + return; + } + + // Pair with appendNewBatch(). For a historical route change, either a normal batch is + // registered as incomplete first and rejects the switch, or batch creation observes the + // historical target. + synchronized (existing) { + if (existing.targetPath.equals(targetPath)) { + existing.partitionId = targetPartitionId; + return; + } + if (!existing.isHistoricalWriteTarget() && resolvedTarget.isHistoricalWriteTarget()) { + if (hasIncompleteBatchFor(originalPath)) { + throw new FlussRuntimeException( + String.format( + "Cannot route writes for %s to %s while this writer has " + + "incomplete writes to %s.", + originalPath, targetPath, existing.targetPath)); + } + existing.switchToHistoricalTarget(targetPath, targetPartitionId); + return; + } + + throw new FlussRuntimeException( + String.format( + "Cannot route writes for %s to %s because this writer already routed " + + "the partition to %s.", + originalPath, targetPath, existing.targetPath)); + } + } + + private boolean hasIncompleteBatchFor(PhysicalTablePath physicalTablePath) { + for (WriteBatch batch : incomplete.copyAll()) { + if (batch.physicalTablePath().equals(physicalTablePath)) { + return true; + } + } + return false; + } + + /** Returns whether this original path is already routed to the historical partition. */ + boolean hasHistoricalWriteTarget(PhysicalTablePath originalPath) { + BucketAndWriteBatches writeTarget = writeBatches.get(originalPath); + return writeTarget != null && writeTarget.isHistoricalWriteTarget(); + } + + /** + * Checks whether a table has historical partition support enabled and caches the result. + * + *

This method is called from the per-record write path. Reading the value from {@link + * TableInfo#getTableConfig()} for every record would repeatedly enter the synchronized {@code + * Configuration} lookup path. Caching both enabled and disabled results keeps subsequent checks + * to a concurrent-map read. The cache is shared here because WriterClient, batch creation, and + * Sender must use the same table classification. + */ + boolean checkAndCacheHistoricalPartitionEnabled(TableInfo tableInfo) { + TablePath tablePath = tableInfo.getTablePath(); + Boolean cached = historicalPartitionEnabledByTable.get(tablePath); + if (cached != null) { + return cached; + } + + boolean historicalPartitionEnabled = + tableInfo.getTableConfig().isHistoricalPartitionEnabled(); + // A writer observes stable table configuration, so concurrent initializers compute the + // same value even if another thread wins putIfAbsent. + historicalPartitionEnabledByTable.putIfAbsent(tablePath, historicalPartitionEnabled); + return historicalPartitionEnabled; + } + + /** Returns the cached historical partition setting for a table. */ + boolean isHistoricalPartitionEnabled(TablePath tablePath) { + return Boolean.TRUE.equals(historicalPartitionEnabledByTable.get(tablePath)); + } + + /** Reroutes queued batches for {@code originalPath} to the historical target. */ + void rerouteQueuedWritesToHistorical( + PhysicalTablePath originalPath, + PhysicalTablePath historicalPath, + long historicalPartitionId) { + BucketAndWriteBatches writeTarget = + checkNotNull( + writeBatches.get(originalPath), + "Write target for %s must exist.", + originalPath); + String originalPartitionName = checkNotNull(originalPath.getPartitionName()); + long originalPartitionId; + synchronized (writeTarget) { + if (writeTarget.isHistoricalWriteTarget()) { + writeTarget.partitionId = historicalPartitionId; + return; + } + // New appends observe the historical route and are created with the original partition + // name. Existing queued batches are converted below before the Sender can drain again. + originalPartitionId = + writeTarget.switchToHistoricalTarget(historicalPath, historicalPartitionId); + } + // The caller has confirmed that no request to the original target remains in flight. + // Convert every queued bucket under its deque lock so append and drain cannot observe a + // batch between detaching its original idempotence state and marking it as historical. + for (Map.Entry> entry : writeTarget.batches.entrySet()) { + Deque deque = entry.getValue(); + synchronized (deque) { + for (WriteBatch batch : deque) { + if (idempotenceManager.idempotenceEnabled() && batch.hasBatchSequence()) { + idempotenceManager.removeInFlightBatch( + new ReadyWriteBatch( + new TableBucket( + batch.tableId(), + originalPartitionId, + entry.getKey()), + batch)); + batch.resetWriterState(NO_WRITER_ID, NO_BATCH_SEQUENCE); + } + batch.rerouteToHistoricalPartition(originalPartitionName); + } + } + } + } + + /** Aborts incomplete batches whose current RPC target is {@code targetPath}. */ + void abortBatches(PhysicalTablePath targetPath, Exception reason) { + for (WriteBatch batch : incomplete.copyAll()) { + BucketAndWriteBatches writeTarget = writeBatches.get(batch.physicalTablePath()); + if (writeTarget != null && writeTarget.targetPath.equals(targetPath)) { + abortBatch(reason, batch); + } + } + } + /** Abort all incomplete batches (whether they have been sent or not). */ public void abortAllBatches(final Exception reason) { for (WriteBatch batch : incomplete.copyAll()) { @@ -357,7 +516,8 @@ private Deque getOrCreateDeque( k -> new BucketAndWriteBatches( tableBucket.getPartitionId(), - physicalTablePath.getPartitionName() != null)); + physicalTablePath.getPartitionName() != null, + physicalTablePath)); return bucketAndWriteBatches.batches.computeIfAbsent( tableBucket.getBucket(), k -> new ArrayDeque<>()); } @@ -478,24 +638,23 @@ private List allocateMemorySegments( /** Check whether there are bucket ready for input table. */ private long bucketReady( - PhysicalTablePath physicalTablePath, BucketAndWriteBatches bucketAndWriteBatches, Set readyNodes, Set unknownLeaderTables, Cluster cluster, long nextReadyCheckDelayMs) { // first check this table has partitionId. + PhysicalTablePath targetPath = bucketAndWriteBatches.targetPath; if (bucketAndWriteBatches.isPartitionedTable && bucketAndWriteBatches.partitionId == null) { - Optional optionIdOpt = cluster.getPartitionId(physicalTablePath); + Optional optionIdOpt = cluster.getPartitionId(targetPath); if (optionIdOpt.isPresent()) { bucketAndWriteBatches.partitionId = optionIdOpt.get(); } else { LOG.debug( - "Partition not exists for {}, bucket will not be set to ready", - physicalTablePath); + "Partition not exists for {}, bucket will not be set to ready", targetPath); // TODO: we shouldn't add unready partitions to unknownLeaderTables, // because it cases PartitionNotExistException later - unknownLeaderTables.add(physicalTablePath); + unknownLeaderTables.add(targetPath); return nextReadyCheckDelayMs; } } @@ -529,12 +688,12 @@ private long bucketReady( } int bucketId = entry.getKey(); - Optional tableIdOpt = cluster.getTableId(physicalTablePath.getTablePath()); + Optional tableIdOpt = cluster.getTableId(targetPath.getTablePath()); if (!tableIdOpt.isPresent()) { - unknownLeaderTables.add(physicalTablePath); + unknownLeaderTables.add(targetPath); } else { TableBucket tableBucket = - cluster.getTableBucket(tableIdOpt.get(), physicalTablePath, bucketId); + cluster.getTableBucket(tableIdOpt.get(), targetPath, bucketId); // If this bucket is throttled, don't mark its node as ready. // Instead, factor the remaining throttle time into the next check delay. @@ -556,7 +715,7 @@ private long bucketReady( // This is a bucket for which leader is not known, but messages are // available to send. Note that entries are currently not removed from // batches when deque is empty. - unknownLeaderTables.add(physicalTablePath); + unknownLeaderTables.add(targetPath); } else { nextReadyCheckDelayMs = batchReady( @@ -616,31 +775,49 @@ private RecordAppendResult appendNewBatch( Deque deque, List segments) throws Exception { - RecordAppendResult appendResult = tryAppend(writeRecord, callback, deque); - if (appendResult != null) { - // Somebody else found us a batch, return the one we waited for! Hopefully this doesn't - // happen often... - return appendResult; - } - PhysicalTablePath physicalTablePath = writeRecord.getPhysicalTablePath(); - PreAllocatedPagedOutputView outputView = new PreAllocatedPagedOutputView(segments); - int schemaId = tableInfo.getSchemaId(); - WriteFormat writeFormat = writeRecord.getWriteFormat(); - final WriteBatch batch = - createWriteBatch( - writeRecord, - bucketId, - tableInfo, - writeFormat, - physicalTablePath, - outputView, - schemaId); + BucketAndWriteBatches bucketAndWriteBatches = + checkNotNull( + writeBatches.get(physicalTablePath), + "Write batches for %s must exist.", + physicalTablePath); + // Historical-enabled tables coordinate with routeWritesTo(). Other tables reuse the + // deque monitor already held by the caller, avoiding cross-bucket serialization. + Object routeLock = + isHistoricalPartitionEnabled(physicalTablePath.getTablePath()) + ? bucketAndWriteBatches + : deque; + synchronized (routeLock) { + RecordAppendResult appendResult = tryAppend(writeRecord, callback, deque); + if (appendResult != null) { + // Somebody else found us a batch, return the one we waited for! Hopefully this + // doesn't happen often... + return appendResult; + } - batch.tryAppend(writeRecord, callback); - deque.addLast(batch); - incomplete.add(batch); - return new RecordAppendResult(deque.size() > 1 || batch.isClosed(), true, false); + PreAllocatedPagedOutputView outputView = new PreAllocatedPagedOutputView(segments); + int schemaId = tableInfo.getSchemaId(); + WriteFormat writeFormat = writeRecord.getWriteFormat(); + String originalPartitionName = + bucketAndWriteBatches.isHistoricalWriteTarget() + ? checkNotNull(physicalTablePath.getPartitionName()) + : null; + final WriteBatch batch = + createWriteBatch( + writeRecord, + bucketId, + tableInfo, + writeFormat, + physicalTablePath, + outputView, + schemaId, + originalPartitionName); + + batch.tryAppend(writeRecord, callback); + deque.addLast(batch); + incomplete.add(batch); + return new RecordAppendResult(deque.size() > 1 || batch.isClosed(), true, false); + } } private WriteBatch createWriteBatch( @@ -650,7 +827,8 @@ private WriteBatch createWriteBatch( WriteFormat writeFormat, PhysicalTablePath physicalTablePath, PreAllocatedPagedOutputView outputView, - int schemaId) { + int schemaId, + @Nullable String originalPartitionName) { // If the table is kv table we need to create a kv batch, otherwise we create a log batch. switch (writeFormat) { case COMPACTED_KV: @@ -665,6 +843,7 @@ private WriteBatch createWriteBatch( outputView, writeRecord.getTargetColumns(), writeRecord.getMergeMode(), + originalPartitionName, clock.milliseconds()); case ARROW_LOG: @@ -688,6 +867,7 @@ private WriteBatch createWriteBatch( tableInfo.getSchemaId(), arrowWriter, outputView, + originalPartitionName, clock.milliseconds(), statisticsCollector); @@ -699,6 +879,7 @@ private WriteBatch createWriteBatch( schemaId, outputView.getPreAllocatedSize(), outputView, + originalPartitionName, clock.milliseconds()); case INDEXED_LOG: @@ -709,6 +890,7 @@ private WriteBatch createWriteBatch( tableInfo.getSchemaId(), outputView.getPreAllocatedSize(), outputView, + originalPartitionName, clock.milliseconds()); default: @@ -1013,6 +1195,12 @@ private List getAllBucketsInCurrentNode(Integer currentNode, Clu List buckets = new ArrayList<>(); Set physicalTablePaths = cluster.getBucketLocationsByPath().keySet(); for (PhysicalTablePath path : physicalTablePaths) { + BucketAndWriteBatches bucketAndWriteBatches = writeBatches.get(path); + // A historical route uses the original path only as the accumulator queue key. Its + // actual bucket locations come from the historical target and are added below. + if (bucketAndWriteBatches != null && bucketAndWriteBatches.isHistoricalWriteTarget()) { + continue; + } List bucketsForTable = cluster.getAvailableBucketsForPhysicalTablePath(path); for (BucketLocation bucket : bucketsForTable) { @@ -1023,6 +1211,31 @@ private List getAllBucketsInCurrentNode(Integer currentNode, Clu } } } + + // Historical queues remain keyed by their original partition path. Add a location using + // that queue key while retaining the historical bucket as the RPC target. + for (Map.Entry entry : writeBatches.entrySet()) { + BucketAndWriteBatches bucketAndWriteBatches = entry.getValue(); + PhysicalTablePath originalPath = entry.getKey(); + if (!bucketAndWriteBatches.isHistoricalWriteTarget()) { + continue; + } + for (BucketLocation bucketLocation : + cluster.getAvailableBucketsForPhysicalTablePath( + bucketAndWriteBatches.targetPath)) { + if (bucketLocation.getLeader() != null + && Objects.equals(currentNode, bucketLocation.getLeader())) { + // Keep the original path so drain can find its original-keyed queue. The + // TableBucket, leader, and replicas still describe the historical RPC target. + buckets.add( + new BucketLocation( + originalPath, + bucketLocation.getTableBucket(), + bucketLocation.getLeader(), + bucketLocation.getReplicas())); + } + } + } return buckets; } @@ -1162,13 +1375,42 @@ public void destroyResources() { /** Per table bucket and write batches. */ private static class BucketAndWriteBatches { public final boolean isPartitionedTable; + /** The physical partition used for metadata lookup, leader discovery, and write RPCs. */ + private volatile PhysicalTablePath targetPath; + public volatile @Nullable Long partitionId; // Write batches for each bucket in queue. public final Map> batches = new CopyOnWriteMap<>(); - public BucketAndWriteBatches(@Nullable Long partitionId, boolean isPartitionedTable) { + private BucketAndWriteBatches( + @Nullable Long partitionId, + boolean isPartitionedTable, + PhysicalTablePath targetPath) { this.partitionId = partitionId; this.isPartitionedTable = isPartitionedTable; + this.targetPath = targetPath; + } + + public boolean isHistoricalWriteTarget() { + return HISTORICAL_PARTITION_VALUE.equals(targetPath.getPartitionName()); + } + + /** + * Atomically switches the target path and partition ID to the historical partition. + * + *

This method must be called while the current target is still the original partition. + * The returned original partition ID is used to detach queued batches from their original + * idempotence state before rerouting them. + */ + private synchronized long switchToHistoricalTarget( + PhysicalTablePath historicalPath, long historicalPartitionId) { + long originalPartitionId = + checkNotNull( + partitionId, + "Original partition ID must be resolved before rerouting."); + targetPath = historicalPath; + partitionId = historicalPartitionId; + return originalPartitionId; } } } diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java b/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java index f4f0c6eedf5..544a78668ab 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java @@ -45,6 +45,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nullable; import javax.annotation.concurrent.GuardedBy; import java.util.ArrayList; @@ -52,10 +53,12 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import static org.apache.fluss.client.utils.ClientRpcMessageUtils.makeProduceLogRequest; import static org.apache.fluss.client.utils.ClientRpcMessageUtils.makePutKvRequest; +import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; import static org.apache.fluss.utils.Preconditions.checkArgument; import static org.apache.fluss.utils.Preconditions.checkNotNull; @@ -228,8 +231,8 @@ private void sendWriteData() throws Exception { // TODO: this try-catch is not needed when we don't update metadata for // unready partitions Throwable t = ExceptionUtils.stripExecutionException(e); - if (t.getCause() instanceof PartitionNotExistException) { - // ignore this exception, this is probably happen because the partition + if (t instanceof PartitionNotExistException) { + handlePartitionNotExistException(readyCheckResult.unknownLeaderTables); } else { throw e; } @@ -250,10 +253,8 @@ private void sendWriteData() throws Exception { // get the list of batches prepare to send. Map> batches = accumulator.drain(clusterSnapshot, readyNodes, maxRequestSize); - if (!batches.isEmpty()) { addToInflightBatches(batches); - // TODO add logic for batch expire. sendWriteRequests(batches); @@ -409,26 +410,81 @@ private void sendWriteRequest(int destination, short acks, List batches); } else { writeBatchByTable.forEach( - (tableId, writeBatches) -> { - if (isLogBatches(writeBatches)) { - sendProduceLogRequestAndHandleResponse( - gateway, - makeProduceLogRequest( - tableId, acks, maxRequestTimeoutMs, writeBatches), - tableId, - writeBatches); - } else { - sendPutKvRequestAndHandleResponse( - gateway, - makePutKvRequest( - tableId, acks, maxRequestTimeoutMs, writeBatches), - tableId, - writeBatches); - } - }); + (tableId, writeBatches) -> + sendWriteRequestsForTable(gateway, tableId, acks, writeBatches)); + } + } + + /** + * Sends normal and historical batches in separate requests. + * + *

Normal and historical writes cannot share a request. Both write protocols correlate + * historical responses by {@link TableBucket} and original partition name, so different + * original partitions targeting the same historical table bucket can remain in one request. + */ + private void sendWriteRequestsForTable( + TabletServerGateway gateway, + long tableId, + short acks, + List writeBatches) { + boolean logBatches = isLogBatches(writeBatches); + List normalBatches = new ArrayList<>(); + List historicalBatches = new ArrayList<>(); + + for (ReadyWriteBatch readyWriteBatch : writeBatches) { + if (readyWriteBatch.writeBatch().getOriginalPartitionName() == null) { + normalBatches.add(readyWriteBatch); + } else { + historicalBatches.add(readyWriteBatch); + } + } + + sendBatchesInRequest(gateway, tableId, acks, logBatches, normalBatches); + sendBatchesInRequest(gateway, tableId, acks, logBatches, historicalBatches); + } + + private void sendBatchesInRequest( + TabletServerGateway gateway, + long tableId, + short acks, + boolean logBatches, + List writeBatches) { + if (writeBatches.isEmpty()) { + return; + } + if (logBatches) { + sendProduceLogRequestAndHandleResponse( + gateway, + makeProduceLogRequest(tableId, acks, maxRequestTimeoutMs, writeBatches), + tableId, + writeBatches); + } else { + sendPutKvRequestAndHandleResponse( + gateway, + makePutKvRequest(tableId, acks, maxRequestTimeoutMs, writeBatches), + tableId, + writeBatches); } } + private static Map toWriteBatchesByKey( + List writeBatches) { + Map writeBatchesByKey = new HashMap<>(); + for (ReadyWriteBatch readyWriteBatch : writeBatches) { + WriteBatch writeBatch = readyWriteBatch.writeBatch(); + WriteBatchKey key = + new WriteBatchKey( + readyWriteBatch.tableBucket(), writeBatch.getOriginalPartitionName()); + ReadyWriteBatch previous = writeBatchesByKey.put(key, readyWriteBatch); + checkArgument( + previous == null, + "A write request contains duplicate table bucket %s and original partition %s.", + readyWriteBatch.tableBucket(), + writeBatch.getOriginalPartitionName()); + } + return writeBatchesByKey; + } + /** * Check whether the given batches are log batches. We assume all the batches are of the same * type. @@ -447,8 +503,7 @@ private void sendProduceLogRequestAndHandleResponse( ProduceLogRequest request, long tableId, List writeBatches) { - Map recordsByBucket = new HashMap<>(); - writeBatches.forEach(batch -> recordsByBucket.put(batch.tableBucket(), batch)); + Map writeBatchesByKey = toWriteBatchesByKey(writeBatches); long startTime = System.currentTimeMillis(); gateway.produceLog(request) .whenComplete( @@ -459,7 +514,7 @@ private void sendProduceLogRequestAndHandleResponse( handleWriteRequestException(e, writeBatches); } else { handleProduceLogResponse( - produceLogResponse, tableId, recordsByBucket); + produceLogResponse, tableId, writeBatchesByKey); } }); } @@ -469,8 +524,7 @@ private void sendPutKvRequestAndHandleResponse( PutKvRequest request, long tableId, List writeBatches) { - Map recordsByBucket = new HashMap<>(); - writeBatches.forEach(batch -> recordsByBucket.put(batch.tableBucket(), batch)); + Map writeBatchesByKey = toWriteBatchesByKey(writeBatches); long startTime = System.currentTimeMillis(); gateway.putKv(request) .whenComplete( @@ -480,7 +534,7 @@ private void sendPutKvRequestAndHandleResponse( if (e != null) { handleWriteRequestException(e, writeBatches); } else { - handlePutKvResponse(putKvResponse, tableId, recordsByBucket); + handlePutKvResponse(putKvResponse, tableId, writeBatchesByKey); } }); } @@ -488,7 +542,7 @@ private void sendPutKvRequestAndHandleResponse( private void handleProduceLogResponse( ProduceLogResponse response, long tableId, - Map recordsByBucket) { + Map writeBatchesByKey) { Set invalidMetadataTablesSet = new HashSet<>(); for (PbProduceLogRespForBucket logRespForBucket : response.getBucketsRespsList()) { TableBucket tb = @@ -498,7 +552,13 @@ private void handleProduceLogResponse( ? logRespForBucket.getPartitionId() : null, logRespForBucket.getBucketId()); - ReadyWriteBatch writeBatch = recordsByBucket.get(tb); + ReadyWriteBatch writeBatch = + writeBatchesByKey.get( + new WriteBatchKey( + tb, + logRespForBucket.hasOriginalPartitionName() + ? logRespForBucket.getOriginalPartitionName() + : null)); if (logRespForBucket.hasErrorCode()) { Set invalidMetadataTables = handleWriteBatchException( @@ -514,7 +574,7 @@ private void handleProduceLogResponse( private void handlePutKvResponse( PutKvResponse putKvResponse, long tableId, - Map recordsByBucket) { + Map writeBatchesByKey) { Set invalidMetadataTablesSet = new HashSet<>(); for (PbPutKvRespForBucket respForBucket : putKvResponse.getBucketsRespsList()) { TableBucket tb = @@ -528,7 +588,13 @@ private void handlePutKvResponse( accumulator.updateThrottle(tb, respForBucket.getPressure()); } - ReadyWriteBatch writeBatch = recordsByBucket.get(tb); + ReadyWriteBatch writeBatch = + writeBatchesByKey.get( + new WriteBatchKey( + tb, + respForBucket.hasOriginalPartitionName() + ? respForBucket.getOriginalPartitionName() + : null)); if (writeBatch == null) { continue; } @@ -566,6 +632,9 @@ private Set handleWriteBatchException( ReadyWriteBatch readyWriteBatch, ApiError error) { Set invalidMetadataTables = new HashSet<>(); WriteBatch writeBatch = readyWriteBatch.writeBatch(); + // Historical queues use the original path as their accumulator key, so capture the actual + // RPC target before any retry handling. + PhysicalTablePath writeTargetPath = writeBatch.writeTargetPath(); if (error.exception() instanceof StorageBackpressureException) { // Hard rejection: the storage engine reached its slowdown trigger and rejected the // write. Map it to full pressure (internal hard-rejection value 1.0f) so the bucket is @@ -634,7 +703,10 @@ private Set handleWriteBatchException( readyWriteBatch.tableBucket(), error.exception()); } - invalidMetadataTables.add(writeBatch.physicalTablePath()); + // A historical batch remains keyed by its original partition path in the + // accumulator, but its RPC is sent to the internal historical partition. Invalidate + // the actual RPC target so the retry refreshes the historical bucket metadata. + invalidMetadataTables.add(writeTargetPath); } } else { LOG.warn( @@ -649,6 +721,138 @@ private Set handleWriteBatchException( return invalidMetadataTables; } + /** + * Rechecks unknown-leader partitions after a bulk metadata update reports {@link + * PartitionNotExistException}, and handles missing partitions for tables with historical + * partition support. + */ + private void handlePartitionNotExistException(Set unknownLeaderTables) { + for (PhysicalTablePath targetPath : unknownLeaderTables) { + if (!accumulator.isHistoricalPartitionEnabled(targetPath.getTablePath())) { + continue; + } + try { + metadataUpdater.checkAndUpdatePartitionMetadata(targetPath); + } catch (Exception e) { + Throwable t = ExceptionUtils.stripExecutionException(e); + if (t instanceof PartitionNotExistException) { + handleMissingPartition(targetPath, t); + continue; + } + throw e; + } + } + } + + /** + * Handles a write target after refreshed metadata confirms that it no longer exists. + * + *

A missing historical target has no further fallback and is aborted. An original target is + * rerouted only after all requests to it have left the in-flight set, so no response from the + * old physical target can race with resetting its queued batches to the historical target. + */ + private void handleMissingPartition(PhysicalTablePath targetPath, Throwable cause) { + if (HISTORICAL_PARTITION_VALUE.equals(targetPath.getPartitionName())) { + abortBatches( + targetPath, + newPartitionNotExistException( + "Cannot continue historical writes to " + + targetPath + + " because refreshed metadata confirms that the partition " + + "no longer exists.", + cause)); + return; + } + + if (hasInFlightBatches(targetPath)) { + // Supporting this case requires waiting for the outstanding responses, detaching the + // batches from the original TableBucket's idempotence state, preserving their order, + // and assigning new sequences for the historical TableBucket. Keep this transition + // simple by failing only the affected target. + // TODO: If this race occurs frequently in practice, implement the coordinated + // in-flight handoff instead of aborting the batches. + abortBatches( + targetPath, + newPartitionNotExistException( + "Cannot safely reroute writes from missing partition " + + targetPath + + " while requests to it are still in flight because rerouting " + + "them to the historical partition could break idempotence.", + cause)); + return; + } + + // TODO: No current in-flight request does not rule out an earlier attempt that was + // accepted while its response was lost. Rerouting such a batch can duplicate the write; + // target-scoped abort only exposes the ambiguity and cannot prevent an upper-layer replay + // from duplicating it. Resolve this with the durable FREEZING/RETIRED handoff; see + // https://github.com/apache/fluss/issues/4161. + PhysicalTablePath historicalPath = + PhysicalTablePath.of(targetPath.getTablePath(), HISTORICAL_PARTITION_VALUE); + @Nullable Throwable historicalTargetCause = null; + try { + if (metadataUpdater.checkAndUpdatePartitionMetadata(historicalPath)) { + accumulator.rerouteQueuedWritesToHistorical( + targetPath, + historicalPath, + metadataUpdater.getPartitionIdOrElseThrow(historicalPath)); + return; + } + } catch (PartitionNotExistException e) { + historicalTargetCause = e; + } + + abortBatches( + targetPath, + newPartitionNotExistException( + "Cannot reroute writes from " + + targetPath + + " because historical write target " + + historicalPath + + " does not exist according to refreshed metadata.", + historicalTargetCause)); + } + + /** Returns whether a request to {@code targetPath} is still awaiting a response. */ + private boolean hasInFlightBatches(PhysicalTablePath targetPath) { + return !getInFlightBatches(targetPath).isEmpty(); + } + + private List getInFlightBatches(PhysicalTablePath targetPath) { + List matchedBatches = new ArrayList<>(); + synchronized (inFlightBatchesLock) { + for (List batches : inFlightBatches.values()) { + for (ReadyWriteBatch batch : batches) { + if (batch.writeBatch().writeTargetPath().equals(targetPath)) { + matchedBatches.add(batch); + } + } + } + } + return matchedBatches; + } + + private void abortBatches(PhysicalTablePath targetPath, PartitionNotExistException exception) { + List matchingInFlightBatches = getInFlightBatches(targetPath); + // Make the batches terminal before detaching their in-flight bookkeeping. + accumulator.abortBatches(targetPath, exception); + for (ReadyWriteBatch batch : matchingInFlightBatches) { + maybeRemoveFromInflightBatches(batch); + if (idempotenceManager.idempotenceEnabled()) { + idempotenceManager.removeInFlightBatch(batch); + } + } + } + + private static PartitionNotExistException newPartitionNotExistException( + String message, @Nullable Throwable cause) { + PartitionNotExistException exception = new PartitionNotExistException(message); + if (cause != null) { + exception.initCause(cause); + } + return exception; + } + private void updateWriterMetrics(Map> batches) { batches.values() .forEach( @@ -719,4 +923,35 @@ private void awaitNextReadyCheck(long delayMs) throws InterruptedException { void destroyResources() { accumulator.destroyResources(); } + + private static final class WriteBatchKey { + private final TableBucket tableBucket; + + // Distinguishes historical writes from different original partitions that share a target + // bucket. Normal writes have no original partition name. + private final @Nullable String originalPartitionName; + + private WriteBatchKey(TableBucket tableBucket, @Nullable String originalPartitionName) { + this.tableBucket = tableBucket; + this.originalPartitionName = originalPartitionName; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof WriteBatchKey)) { + return false; + } + WriteBatchKey that = (WriteBatchKey) o; + return tableBucket.equals(that.tableBucket) + && Objects.equals(originalPartitionName, that.originalPartitionName); + } + + @Override + public int hashCode() { + return Objects.hash(tableBucket, originalPartitionName); + } + } } diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/WriteBatch.java b/fluss-client/src/main/java/org/apache/fluss/client/write/WriteBatch.java index 9cbfaa6af2e..2ea1c3bbc40 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/WriteBatch.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/WriteBatch.java @@ -36,6 +36,7 @@ import java.util.concurrent.atomic.AtomicReference; import static org.apache.fluss.record.LogRecordBatchFormat.NO_BATCH_SEQUENCE; +import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; import static org.apache.fluss.utils.Preconditions.checkNotNull; /** The abstract write batch contains write callback object to wait write request feedback. */ @@ -54,6 +55,14 @@ public abstract class WriteBatch { protected final List callbacks = new ArrayList<>(); private final AtomicReference finalState = new AtomicReference<>(null); private final AtomicInteger attempts = new AtomicInteger(0); + /** + * The original partition name for a batch targeting the historical system partition. + * + *

It is null for a normal write and contains the logical partition namespace for a + * historical write. + */ + private volatile @Nullable String originalPartitionName; + protected boolean reopened; protected int recordCount; private long drainedMs; @@ -68,12 +77,14 @@ public WriteBatch( PhysicalTablePath physicalTablePath, int schemaId, WriteFormat writeFormat, + @Nullable String originalPartitionName, long createdMs) { this.physicalTablePath = physicalTablePath; this.createdMs = createdMs; this.tableId = tableId; this.schemaId = schemaId; this.writeFormat = checkNotNull(writeFormat, "write format must be not null"); + this.originalPartitionName = originalPartitionName; this.bucketId = bucketId; this.requestFuture = new RequestFuture(); this.recordCount = 0; @@ -205,6 +216,24 @@ public PhysicalTablePath physicalTablePath() { return physicalTablePath; } + /** Returns the physical partition path used as the write RPC target. */ + public PhysicalTablePath writeTargetPath() { + return originalPartitionName == null + ? physicalTablePath + : PhysicalTablePath.of( + physicalTablePath.getTablePath(), HISTORICAL_PARTITION_VALUE); + } + + /** Returns the original partition name for a historical write, or null for a normal write. */ + public @Nullable String getOriginalPartitionName() { + return originalPartitionName; + } + + /** Marks this batch as targeting the historical partition. */ + void rerouteToHistoricalPartition(String partitionName) { + originalPartitionName = checkNotNull(partitionName); + } + public RequestFuture getRequestFuture() { return requestFuture; } diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/WriterClient.java b/fluss-client/src/main/java/org/apache/fluss/client/write/WriterClient.java index ad8c7870547..2239b98d95e 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/WriterClient.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/WriterClient.java @@ -28,10 +28,12 @@ import org.apache.fluss.config.Configuration; import org.apache.fluss.exception.FlussRuntimeException; import org.apache.fluss.exception.IllegalConfigurationException; +import org.apache.fluss.exception.PartitionNotExistException; import org.apache.fluss.metadata.PhysicalTablePath; import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.rpc.gateway.TabletServerGateway; import org.apache.fluss.rpc.metrics.ClientMetricGroup; +import org.apache.fluss.utils.AutoPartitionStrategy; import org.apache.fluss.utils.CopyOnWriteMap; import org.apache.fluss.utils.clock.SystemClock; import org.apache.fluss.utils.concurrent.ExecutorThreadFactory; @@ -42,6 +44,9 @@ import javax.annotation.concurrent.ThreadSafe; import java.time.Duration; +import java.time.Instant; +import java.time.ZonedDateTime; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.concurrent.ExecutorService; @@ -51,6 +56,8 @@ import static org.apache.fluss.config.ConfigOptions.NoKeyAssigner.ROUND_ROBIN; import static org.apache.fluss.config.ConfigOptions.NoKeyAssigner.STICKY; import static org.apache.fluss.utils.ExceptionUtils.toException; +import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; +import static org.apache.fluss.utils.PartitionUtils.generateAutoPartitionTime; /** * A client that write records to server. @@ -194,7 +201,16 @@ private void doSend(WriteRecord record, WriteCallback callback) { PhysicalTablePath physicalTablePath = record.getPhysicalTablePath(); // Skip the call entirely on non-partitioned tables; there is no partition to create. if (tableInfo.isPartitioned()) { - dynamicPartitionCreator.checkAndCreatePartitionAsync(physicalTablePath, tableInfo); + boolean historicalPartitionEnabled = + accumulator.checkAndCacheHistoricalPartitionEnabled(tableInfo); + if (historicalPartitionEnabled + && mayBeExpiredHistoricalPartition( + physicalTablePath, tableInfo, Instant.now())) { + resolveHistoricalWriteTarget(physicalTablePath); + } else { + dynamicPartitionCreator.checkAndCreatePartitionAsync( + physicalTablePath, tableInfo); + } } // maybe create bucket assigner. @@ -240,6 +256,76 @@ private void doSend(WriteRecord record, WriteCallback callback) { } } + /** + * Returns whether a partition of a historical-partition-enabled table is old enough that it may + * have expired under its retention policy. + * + *

This client-side precheck uses the time zone resolved from the table configuration. A + * {@code true} result does not confirm that the partition is missing; the caller must refresh + * metadata before routing the write to a historical partition. If the table does not explicitly + * configure a time zone and the Client and Coordinator use different defaults, they may + * classify partitions near the retention boundary differently. Late classification may fail a + * write to an already removed original partition, while early classification only causes an + * extra metadata refresh. + */ + static boolean mayBeExpiredHistoricalPartition( + PhysicalTablePath physicalTablePath, TableInfo tableInfo, Instant now) { + // TODO: Move this per-record configuration and time calculation off the hot path by using + // periodically refreshed, server-authoritative partition status; see + // https://github.com/apache/fluss/issues/4161. + String partitionName = physicalTablePath.getPartitionName(); + if (partitionName == null) { + return false; + } + + AutoPartitionStrategy strategy = tableInfo.getTableConfig().getAutoPartitionStrategy(); + if (strategy.numToRetain() < 0) { + return false; + } + + ZonedDateTime currentDateTime = + ZonedDateTime.ofInstant(now, strategy.timeZone().toZoneId()); + String earliestRetainedPartition = + generateAutoPartitionTime( + currentDateTime, -strategy.numToRetain(), strategy.timeUnit(), strategy); + return partitionName.compareTo(earliestRetainedPartition) < 0; + } + + private void resolveHistoricalWriteTarget(PhysicalTablePath originalPath) { + // Keep refreshing while the target is still the original partition so its retirement can + // be detected before more records are appended to the stale route. Ideally, the Client + // should learn the server-authoritative partition status without synchronously refreshing + // metadata on the per-record path; see https://github.com/apache/fluss/issues/4161. + if (accumulator.hasHistoricalWriteTarget(originalPath)) { + return; + } + + PhysicalTablePath targetPath = originalPath; + // The time check only limits metadata traffic. Invalidate a potentially stale cached route + // and authoritatively choose the target before the record enters the queue. + metadataUpdater.invalidPhysicalTableBucketAndPartitionMeta( + Collections.singleton(originalPath)); + try { + if (!metadataUpdater.checkAndUpdatePartitionMetadata(originalPath)) { + throw new FlussRuntimeException( + "Failed to resolve write target for " + originalPath + '.'); + } + } catch (PartitionNotExistException ignored) { + targetPath = + PhysicalTablePath.of(originalPath.getTablePath(), HISTORICAL_PARTITION_VALUE); + // TODO: Activate this target only after the lake-aware partition retirement protocol + // guarantees that all accepted original writes are readable from the lake; see + // https://github.com/apache/fluss/pull/3820. + if (!metadataUpdater.checkAndUpdatePartitionMetadata(targetPath)) { + throw new PartitionNotExistException( + "Historical partition " + targetPath + " does not exist."); + } + } + + accumulator.routeWritesTo( + originalPath, targetPath, metadataUpdater.getPartitionIdOrElseThrow(targetPath)); + } + private void maybeAbortBatches(Throwable t) { if (accumulator.hasIncomplete()) { LOG.error("Aborting all pending write batches due to fatal error", t); diff --git a/fluss-client/src/test/java/org/apache/fluss/client/utils/ClientRpcMessageUtilsTest.java b/fluss-client/src/test/java/org/apache/fluss/client/utils/ClientRpcMessageUtilsTest.java index 3ed17da7da5..153c3dc13f0 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/utils/ClientRpcMessageUtilsTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/utils/ClientRpcMessageUtilsTest.java @@ -140,6 +140,7 @@ private KvWriteBatch createKvWriteBatch(int bucketId, MergeMode mergeMode) throw outputView, null, mergeMode, + null, System.currentTimeMillis()); } } diff --git a/fluss-client/src/test/java/org/apache/fluss/client/write/ArrowLogWriteBatchTest.java b/fluss-client/src/test/java/org/apache/fluss/client/write/ArrowLogWriteBatchTest.java index f87f0c01a35..c9b98e1dfca 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/write/ArrowLogWriteBatchTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/write/ArrowLogWriteBatchTest.java @@ -137,6 +137,7 @@ void testAppendWithPreAllocatedMemorySegments() throws Exception { DATA1_ROW_TYPE, DEFAULT_COMPRESSION), new PreAllocatedPagedOutputView(memorySegmentList), + null, System.currentTimeMillis(), null); assertThat(arrowLogWriteBatch.pooledMemorySegments()).isEqualTo(memorySegmentList); @@ -213,6 +214,7 @@ void testArrowCompressionRatioEstimated() throws Exception { DATA1_TABLE_INFO.getSchemaId(), arrowWriter, new PreAllocatedPagedOutputView(memorySegmentList), + null, System.currentTimeMillis(), null); @@ -315,6 +317,7 @@ private ArrowLogWriteBatch createArrowLogWriteBatch(TableBucket tb, int maxSizeI DATA1_ROW_TYPE, DEFAULT_COMPRESSION), new UnmanagedPagedOutputView(128), + null, System.currentTimeMillis(), null); } diff --git a/fluss-client/src/test/java/org/apache/fluss/client/write/CompactedLogWriteBatchTest.java b/fluss-client/src/test/java/org/apache/fluss/client/write/CompactedLogWriteBatchTest.java index dc8fbce9d32..2461c2a7134 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/write/CompactedLogWriteBatchTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/write/CompactedLogWriteBatchTest.java @@ -254,6 +254,7 @@ private CompactedLogWriteBatch createLogWriteBatch( DATA1_TABLE_INFO.getSchemaId(), writeLimit, new PreAllocatedPagedOutputView(Collections.singletonList(memorySegment)), + null, System.currentTimeMillis()); } diff --git a/fluss-client/src/test/java/org/apache/fluss/client/write/IndexedLogWriteBatchTest.java b/fluss-client/src/test/java/org/apache/fluss/client/write/IndexedLogWriteBatchTest.java index 331be4209f6..a6269ad64b3 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/write/IndexedLogWriteBatchTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/write/IndexedLogWriteBatchTest.java @@ -216,6 +216,7 @@ private IndexedLogWriteBatch createLogWriteBatch( DATA1_TABLE_INFO.getSchemaId(), writeLimit, new PreAllocatedPagedOutputView(Collections.singletonList(memorySegment)), + null, System.currentTimeMillis()); } diff --git a/fluss-client/src/test/java/org/apache/fluss/client/write/KvWriteBatchTest.java b/fluss-client/src/test/java/org/apache/fluss/client/write/KvWriteBatchTest.java index 7b61976a038..b0faf06e679 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/write/KvWriteBatchTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/write/KvWriteBatchTest.java @@ -230,6 +230,7 @@ private KvWriteBatch createKvWriteBatch( outputView, null, MergeMode.DEFAULT, + null, System.currentTimeMillis()); } @@ -326,6 +327,7 @@ private KvWriteBatch createKvWriteBatchWithMergeMode(TableBucket tb, MergeMode m outputView, null, mergeMode, + null, System.currentTimeMillis()); } diff --git a/fluss-client/src/test/java/org/apache/fluss/client/write/SenderTest.java b/fluss-client/src/test/java/org/apache/fluss/client/write/SenderTest.java index 3e99500c317..132da6cc112 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/write/SenderTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/write/SenderTest.java @@ -19,18 +19,24 @@ import org.apache.fluss.client.metadata.TestingMetadataUpdater; import org.apache.fluss.client.metrics.TestingWriterMetricGroup; +import org.apache.fluss.cluster.BucketLocation; import org.apache.fluss.cluster.Cluster; import org.apache.fluss.cluster.ServerNode; +import org.apache.fluss.config.AutoPartitionTimeUnit; import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; import org.apache.fluss.config.MemorySize; import org.apache.fluss.exception.AuthorizationException; import org.apache.fluss.exception.NetworkException; import org.apache.fluss.exception.OutOfOrderSequenceException; +import org.apache.fluss.exception.PartitionNotExistException; import org.apache.fluss.exception.TableNotExistException; import org.apache.fluss.exception.TimeoutException; +import org.apache.fluss.metadata.DataLakeFormat; import org.apache.fluss.metadata.PhysicalTablePath; +import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.metadata.TablePath; import org.apache.fluss.record.MemoryLogRecords; @@ -45,7 +51,10 @@ import org.apache.fluss.rpc.messages.PutKvRequest; import org.apache.fluss.rpc.messages.PutKvResponse; import org.apache.fluss.rpc.protocol.Errors; +import org.apache.fluss.server.entity.ProduceLogDataForBucket; +import org.apache.fluss.server.entity.PutKvDataForBucket; import org.apache.fluss.server.tablet.TestTabletServerGateway; +import org.apache.fluss.types.DataTypes; import org.apache.fluss.utils.clock.SystemClock; import org.junit.jupiter.api.AfterEach; @@ -53,10 +62,13 @@ import org.junit.jupiter.api.Test; import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Deque; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -77,12 +89,14 @@ import static org.apache.fluss.record.TestData.DATA2_TABLE_ID; import static org.apache.fluss.record.TestData.DEFAULT_REMOTE_DATA_DIR; import static org.apache.fluss.rpc.protocol.Errors.SCHEMA_NOT_EXIST; -import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getProduceLogData; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.makeProduceLogResponse; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.makePutKvResponse; +import static org.apache.fluss.server.utils.ServerRpcMessageUtils.toProduceLogDataForBuckets; +import static org.apache.fluss.server.utils.ServerRpcMessageUtils.toPutKvDataForBuckets; import static org.apache.fluss.testutils.DataTestUtils.compactedRow; import static org.apache.fluss.testutils.DataTestUtils.row; import static org.apache.fluss.testutils.common.CommonTestUtils.retry; +import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; import static org.assertj.core.api.Assertions.assertThat; /** ITCase for {@link Sender}. */ @@ -115,6 +129,213 @@ public void teardown() throws Exception { sender.destroyResources(); } + @Test + void testPotentialExpirationUsesAutoPartitionTimeUnit() { + TableInfo tableInfo = createHistoricalTableInfo(AutoPartitionTimeUnit.HOUR, 48); + Instant now = Instant.parse("2026-08-24T00:00:00Z"); + + assertThat( + WriterClient.mayBeExpiredHistoricalPartition( + PhysicalTablePath.of(tableInfo.getTablePath(), "2026082123"), + tableInfo, + now)) + .isTrue(); + assertThat( + WriterClient.mayBeExpiredHistoricalPartition( + PhysicalTablePath.of(tableInfo.getTablePath(), "2026082200"), + tableInfo, + now)) + .isFalse(); + } + + @Test + void testReroutesWriteAfterExplicitMissingPartitionResponse() throws Exception { + sender.destroyResources(); + TableInfo tableInfo = createHistoricalTableInfo(); + PhysicalTablePath originalPath = PhysicalTablePath.of(tableInfo.getTablePath(), "20990101"); + PhysicalTablePath historicalPath = + PhysicalTablePath.of(tableInfo.getTablePath(), HISTORICAL_PARTITION_VALUE); + TableBucket originalBucket = new TableBucket(tableInfo.getTableId(), 21L, 0); + TableBucket historicalBucket = new TableBucket(tableInfo.getTableId(), 22L, 0); + metadataUpdater = missingPartitionMetadataUpdater(tableInfo, originalPath); + Map tableBucketsByPath = new HashMap<>(); + tableBucketsByPath.put(originalPath, originalBucket); + tableBucketsByPath.put(historicalPath, historicalBucket); + metadataUpdater.updateCluster(partitionedCluster(tableInfo, tableBucketsByPath)); + IdempotenceManager idempotenceManager = createIdempotenceManager(true); + idempotenceManager.setWriterId(0L); + sender = setupWithIdempotenceState(idempotenceManager); + + CompletableFuture future = + appendKvRecord(tableInfo, originalPath, 1, metadataUpdater.getCluster()); + // Send the first attempt to the original partition. + sender.runOnce(); + + TestTabletServerGateway gateway = node1Gateway(); + // The explicit rejection re-enqueues the batch and invalidates the original metadata. + gateway.response( + 0, createPutKvResponse(originalBucket, Errors.UNKNOWN_TABLE_OR_BUCKET_EXCEPTION)); + assertThat(future).isNotDone(); + + // With no request left in flight, the missing original target can be safely rerouted. + sender.runOnce(); + assertThat(future).isNotDone(); + assertThat(idempotenceManager.inflightBatchSize(originalBucket)).isZero(); + + // The retry targets the historical bucket while preserving the original partition name. + sender.runOnce(); + PutKvRequest historicalRequest = (PutKvRequest) gateway.getRequest(0); + assertThat(historicalRequest.getBucketsReqsCount()).isOne(); + assertThat(historicalRequest.getBucketsReqAt(0).getPartitionId()) + .isEqualTo(historicalBucket.getPartitionId()); + assertThat(historicalRequest.getBucketsReqAt(0).getOriginalPartitionName()) + .isEqualTo(originalPath.getPartitionName()); + List historicalData = toPutKvDataForBuckets(historicalRequest); + assertThat(historicalData).hasSize(1); + assertThat(historicalData.get(0).records().writerId()).isEqualTo(0L); + assertThat(historicalData.get(0).records().batchSequence()).isZero(); + assertThat(idempotenceManager.inflightBatchSize(historicalBucket)).isOne(); + + gateway.response( + 0, + makePutKvResponse( + Collections.singletonList( + PutKvResultForBucket.historicalSuccess( + historicalBucket, 1L, originalPath.getPartitionName())))); + assertThat(future.get()).isNull(); + } + + @Test + void testAbortsOnlyMissingPartitionWhenRerouteIsUnsafe() throws Exception { + sender.destroyResources(); + TableInfo tableInfo = createHistoricalTableInfo(); + PhysicalTablePath missingPath = PhysicalTablePath.of(tableInfo.getTablePath(), "20990101"); + PhysicalTablePath activePath = PhysicalTablePath.of(tableInfo.getTablePath(), "20990102"); + PhysicalTablePath historicalPath = + PhysicalTablePath.of(tableInfo.getTablePath(), HISTORICAL_PARTITION_VALUE); + TableBucket missingBucket = new TableBucket(tableInfo.getTableId(), 21L, 0); + TableBucket historicalBucket = new TableBucket(tableInfo.getTableId(), 22L, 0); + TableBucket activeBucket = new TableBucket(tableInfo.getTableId(), 23L, 0); + metadataUpdater = missingPartitionMetadataUpdater(tableInfo, missingPath); + Map tableBucketsByPath = new HashMap<>(); + tableBucketsByPath.put(missingPath, missingBucket); + tableBucketsByPath.put(activePath, activeBucket); + tableBucketsByPath.put(historicalPath, historicalBucket); + metadataUpdater.updateCluster(partitionedCluster(tableInfo, tableBucketsByPath)); + sender = setupWithIdempotenceState(); + + CompletableFuture firstMissingFuture = + appendKvRecord(tableInfo, missingPath, 1, metadataUpdater.getCluster()); + sender.runOnce(); + // Keep a second request to the same original target in flight. + CompletableFuture secondMissingFuture = + appendKvRecord(tableInfo, missingPath, 2, metadataUpdater.getCluster()); + sender.runOnce(); + + TestTabletServerGateway gateway = node1Gateway(); + // Reject the first request while the second one is still awaiting a response. + gateway.response( + 0, createPutKvResponse(missingBucket, Errors.UNKNOWN_TABLE_OR_BUCKET_EXCEPTION)); + assertThat(firstMissingFuture).isNotDone(); + assertThat(secondMissingFuture).isNotDone(); + + // A write to another partition verifies that abort remains scoped to the missing target. + CompletableFuture activeFuture = + appendKvRecord(tableInfo, activePath, 3, metadataUpdater.getCluster()); + sender.runOnce(); + + assertThat(firstMissingFuture.get()) + .isInstanceOf(PartitionNotExistException.class) + .hasMessageContaining(missingPath.toString()); + assertThat(secondMissingFuture.get()) + .isInstanceOf(PartitionNotExistException.class) + .hasMessageContaining(missingPath.toString()); + assertThat(activeFuture).isNotDone(); + + // Complete the request that was still in flight when its batches were aborted. + gateway.response( + 0, createPutKvResponse(missingBucket, Errors.UNKNOWN_TABLE_OR_BUCKET_EXCEPTION)); + gateway.response(0, createPutKvResponse(activeBucket, 1L)); + assertThat(activeFuture.get()).isNull(); + } + + @Test + void testNormalAndHistoricalPutRequests() throws Exception { + sender.destroyResources(); + TableInfo tableInfo = createHistoricalTableInfo(); + PhysicalTablePath activePath = PhysicalTablePath.of(tableInfo.getTablePath(), "20990101"); + PhysicalTablePath firstOriginalPath = + PhysicalTablePath.of(tableInfo.getTablePath(), "20000101"); + PhysicalTablePath secondOriginalPath = + PhysicalTablePath.of(tableInfo.getTablePath(), "20000102"); + PhysicalTablePath historicalPath = + PhysicalTablePath.of(tableInfo.getTablePath(), HISTORICAL_PARTITION_VALUE); + TableBucket activeBucket = new TableBucket(tableInfo.getTableId(), 21L, 0); + TableBucket historicalBucket = new TableBucket(tableInfo.getTableId(), 22L, 0); + + Map tableBucketsByPath = new HashMap<>(); + tableBucketsByPath.put(activePath, activeBucket); + tableBucketsByPath.put(historicalPath, historicalBucket); + metadataUpdater = + new TestingMetadataUpdater( + Collections.singletonMap(tableInfo.getTablePath(), tableInfo)); + metadataUpdater.updateCluster(partitionedCluster(tableInfo, tableBucketsByPath)); + sender = setupWithIdempotenceState(); + + CompletableFuture activeFuture = + appendKvRecord(tableInfo, activePath, 1, metadataUpdater.getCluster()); + accumulator.routeWritesTo( + firstOriginalPath, historicalPath, historicalBucket.getPartitionId()); + accumulator.routeWritesTo( + secondOriginalPath, historicalPath, historicalBucket.getPartitionId()); + CompletableFuture firstHistoricalFuture = + appendKvRecord(tableInfo, firstOriginalPath, 2, metadataUpdater.getCluster()); + CompletableFuture secondHistoricalFuture = + appendKvRecord(tableInfo, secondOriginalPath, 3, metadataUpdater.getCluster()); + + sender.runOnce(); + + TestTabletServerGateway gateway = node1Gateway(); + assertThat(gateway.pendingRequestSize()).isEqualTo(2); + + PutKvRequest normalRequest = (PutKvRequest) gateway.getRequest(0); + assertThat(normalRequest.getBucketsReqsCount()).isOne(); + assertThat(normalRequest.getBucketsReqAt(0).getPartitionId()) + .isEqualTo(activeBucket.getPartitionId()); + assertThat(normalRequest.getBucketsReqAt(0).hasOriginalPartitionName()).isFalse(); + + PutKvRequest historicalRequest = (PutKvRequest) gateway.getRequest(1); + assertThat(historicalRequest.getBucketsReqsCount()).isEqualTo(2); + Set originalPartitionNames = new HashSet<>(); + for (int i = 0; i < historicalRequest.getBucketsReqsCount(); i++) { + assertThat(historicalRequest.getBucketsReqAt(i).getPartitionId()) + .isEqualTo(historicalBucket.getPartitionId()); + originalPartitionNames.add( + historicalRequest.getBucketsReqAt(i).getOriginalPartitionName()); + } + assertThat(originalPartitionNames) + .containsExactlyInAnyOrder( + firstOriginalPath.getPartitionName(), + secondOriginalPath.getPartitionName()); + + gateway.response(0, createPutKvResponse(activeBucket, 1L)); + gateway.response( + 0, + makePutKvResponse( + Arrays.asList( + PutKvResultForBucket.historicalSuccess( + historicalBucket, + 1L, + secondOriginalPath.getPartitionName()), + PutKvResultForBucket.historicalSuccess( + historicalBucket, + 1L, + firstOriginalPath.getPartitionName())))); + assertThat(activeFuture.get()).isNull(); + assertThat(firstHistoricalFuture.get()).isNull(); + assertThat(secondHistoricalFuture.get()).isNull(); + } + @Test void testSimple() throws Exception { long offset = 0; @@ -1263,6 +1484,116 @@ private void resetTableInfosWith(TableInfo tableInfo) { metadataUpdater.updateTableInfos(tableInfos); } + private static TableInfo createHistoricalTableInfo() { + return createHistoricalTableInfo(AutoPartitionTimeUnit.DAY, 7); + } + + private static TableInfo createHistoricalTableInfo( + AutoPartitionTimeUnit timeUnit, int numToRetain) { + Schema schema = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("dt", DataTypes.STRING()) + .primaryKey("id", "dt") + .build(); + TableDescriptor descriptor = + TableDescriptor.builder() + .schema(schema) + .partitionedBy("dt") + .distributedBy(1) + .property(ConfigOptions.TABLE_AUTO_PARTITION_ENABLED, true) + .property(ConfigOptions.TABLE_AUTO_PARTITION_TIME_UNIT, timeUnit) + .property(ConfigOptions.TABLE_AUTO_PARTITION_NUM_RETENTION, numToRetain) + .property(ConfigOptions.TABLE_AUTO_PARTITION_TIMEZONE, "UTC") + .property(ConfigOptions.TABLE_DATALAKE_ENABLED, true) + .property(ConfigOptions.TABLE_DATALAKE_FORMAT, DataLakeFormat.PAIMON) + .property(ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_ENABLED, true) + .build(); + return TableInfo.of( + DATA1_TABLE_PATH_PK, + DATA1_TABLE_ID_PK, + 1, + descriptor, + DEFAULT_REMOTE_DATA_DIR, + 1L, + 1L); + } + + private static TestingMetadataUpdater missingPartitionMetadataUpdater( + TableInfo tableInfo, PhysicalTablePath missingPath) { + return new TestingMetadataUpdater( + Collections.singletonMap(tableInfo.getTablePath(), tableInfo)) { + @Override + public void updatePhysicalTableMetadata(Set physicalTablePaths) { + throw new PartitionNotExistException("Partition does not exist."); + } + + @Override + public boolean checkAndUpdatePartitionMetadata(PhysicalTablePath physicalTablePath) { + if (physicalTablePath.equals(missingPath)) { + throw new PartitionNotExistException("Partition does not exist."); + } + return getCluster().getPartitionId(physicalTablePath).isPresent(); + } + }; + } + + private static Cluster partitionedCluster( + TableInfo tableInfo, Map tableBucketsByPath) { + int[] replicas = new int[] {TestingMetadataUpdater.NODE1.id()}; + Map> bucketLocationsByPath = new HashMap<>(); + Map partitionIdsByPath = new HashMap<>(); + tableBucketsByPath.forEach( + (physicalTablePath, tableBucket) -> { + bucketLocationsByPath.put( + physicalTablePath, + Collections.singletonList( + new BucketLocation( + physicalTablePath, + tableBucket, + TestingMetadataUpdater.NODE1.id(), + replicas))); + partitionIdsByPath.put(physicalTablePath, tableBucket.getPartitionId()); + }); + return new Cluster( + Collections.singletonMap( + TestingMetadataUpdater.NODE1.id(), TestingMetadataUpdater.NODE1), + TestingMetadataUpdater.COORDINATOR, + bucketLocationsByPath, + Collections.singletonMap(tableInfo.getTablePath(), tableInfo.getTableId()), + partitionIdsByPath); + } + + private CompletableFuture appendKvRecord( + TableInfo tableInfo, PhysicalTablePath physicalTablePath, int id, Cluster cluster) + throws Exception { + accumulator.checkAndCacheHistoricalPartitionEnabled(tableInfo); + BinaryRow row = + compactedRow( + tableInfo.getRowType(), + new Object[] {id, physicalTablePath.getPartitionName()}); + byte[] key = + new CompactedKeyEncoder( + tableInfo.getRowType(), + tableInfo.getSchema().getPrimaryKeyIndexes()) + .encodeKey(row); + CompletableFuture future = new CompletableFuture<>(); + accumulator.append( + WriteRecord.forUpsert( + tableInfo, + physicalTablePath, + row, + key, + key, + WriteFormat.COMPACTED_KV, + null), + (tableBucket, logEndOffset, error) -> future.complete(error), + cluster, + 0, + false); + return future; + } + private void appendToAccumulator(TableBucket tb, GenericRow row, WriteCallback writeCallback) throws Exception { appendToAccumulator(DATA1_TABLE_INFO, tb, row, writeCallback); @@ -1308,6 +1639,11 @@ private void appendKvToAccumulator( false); } + private TestTabletServerGateway node1Gateway() { + return (TestTabletServerGateway) + metadataUpdater.newTabletServerClientForNode(TestingMetadataUpdater.NODE1.id()); + } + private ApiMessage getRequest(TableBucket tb, int index) { TestTabletServerGateway gateway = (TestTabletServerGateway) @@ -1448,14 +1784,24 @@ private IdempotenceManager createIdempotenceManager(boolean idempotenceEnabled) } private static boolean hasIdempotentRecords(TableBucket tb, ProduceLogRequest request) { - MemoryLogRecords memoryLogRecords = getProduceLogData(request).get(tb); + MemoryLogRecords memoryLogRecords = getProduceLogRecords(request, tb); return memoryLogRecords.batchIterator().next().writerId() != NO_WRITER_ID; } private static void assertBatchSequenceEquals( TableBucket tb, ProduceLogRequest request, int expectedBatchSequence) { - MemoryLogRecords memoryLogRecords = getProduceLogData(request).get(tb); + MemoryLogRecords memoryLogRecords = getProduceLogRecords(request, tb); assertThat(memoryLogRecords.batchIterator().next().batchSequence()) .isEqualTo(expectedBatchSequence); } + + private static MemoryLogRecords getProduceLogRecords( + ProduceLogRequest request, TableBucket tableBucket) { + for (ProduceLogDataForBucket bucketData : toProduceLogDataForBuckets(request)) { + if (bucketData.tableBucket().equals(tableBucket)) { + return bucketData.records(); + } + } + throw new IllegalArgumentException("No records found for table bucket " + tableBucket); + } } diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java index 19e71dfee21..d3157eeb798 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java @@ -1924,11 +1924,12 @@ public class ConfigOptions { .booleanType() .defaultValue(false) .withDescription( - "Whether to enable historical partition lookup for the table. " + "Whether to enable historical partition access for the table. " + "When enabled, the coordinator creates and retains a system partition " - + "for routing lookups of expired partitions to lake storage. " - + "Currently, this option only supports auto-partitioned Paimon primary " - + "key tables with a single partition key. Disabled by default. " + + "for routing writes to expired partitions and, for primary-key tables, " + + "lookups of expired partitions to lake storage. Currently, this option " + + "only supports auto-partitioned Paimon tables with a single partition " + + "key. Disabled by default. " + "After changing this option, restart existing lookup jobs that need " + "to look up historical partition data so that their clients load the " + "updated table configuration."); diff --git a/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java b/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java index ea046e3b11d..d1b41983768 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java @@ -129,7 +129,7 @@ public boolean isDataLakeEnabled() { return config.get(ConfigOptions.TABLE_DATALAKE_ENABLED); } - /** Whether historical partition lookup is enabled. */ + /** Whether historical partition access is enabled. */ public boolean isHistoricalPartitionEnabled() { return config.get(ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_ENABLED); } diff --git a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeTableLookuper.java b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeTableLookuper.java index ab2d201b72b..1389e294d48 100644 --- a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeTableLookuper.java +++ b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeTableLookuper.java @@ -60,6 +60,17 @@ interface LookupMetricRecorder { @Nullable byte[] lookup(byte[] key, LookupContext context) throws Exception; + /** + * Requests that registered lake data files be refreshed before the next lookup. + * + *

This method must not perform I/O and may be called concurrently with {@link + * #lookup(byte[], LookupContext)}. Implementations must ensure that it is thread-safe. + */ + default void refresh() { + throw new UnsupportedOperationException( + "Refreshing registered files is not supported by this lake table lookuper."); + } + /** Context for a lake table point lookup. */ final class LookupContext { private final ResolvedPartitionSpec partitionSpec; diff --git a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/PluginLakeStorageWrapper.java b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/PluginLakeStorageWrapper.java index 2f778bc0a90..7e2932dd168 100644 --- a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/PluginLakeStorageWrapper.java +++ b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/PluginLakeStorageWrapper.java @@ -166,6 +166,13 @@ private ClassLoaderFixingLakeTableLookuper(LakeTableLookuper inner, ClassLoader } } + @Override + public void refresh() { + try (TemporaryClassLoaderContext ignored = TemporaryClassLoaderContext.of(loader)) { + inner.refresh(); + } + } + @Override public void close() throws Exception { try (TemporaryClassLoaderContext ignored = TemporaryClassLoaderContext.of(loader)) { diff --git a/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueEncoder.java b/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueEncoder.java index 13069a16562..a4054b1ede6 100644 --- a/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueEncoder.java +++ b/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueEncoder.java @@ -20,9 +20,12 @@ import org.apache.fluss.record.BinaryValue; import org.apache.fluss.row.BinaryRow; +import javax.annotation.Nullable; + import java.util.function.ToLongFunction; import static org.apache.fluss.utils.Preconditions.checkNotNull; +import static org.apache.fluss.utils.Preconditions.checkState; /** An encoder to encode {@link BinaryRow} with a schema id as value to be stored in kv store. */ public final class ValueEncoder { @@ -30,21 +33,28 @@ public final class ValueEncoder { private static final ValueEncoder PLAIN_ENCODER = new ValueEncoder(KvValueLayout.PLAIN, null); private final KvValueLayout kvValueLayout; - private final ToLongFunction valueTagProvider; - private ValueEncoder(KvValueLayout kvValueLayout, ToLongFunction valueTagProvider) { + /** + * Generates tags for {@link #encodeValue(BinaryValue)}. {@code null} for plain values or when + * callers supply each tag to {@link #encodeValue(BinaryValue, long)}. + */ + @Nullable private final ToLongFunction valueTagProvider; + + private ValueEncoder( + KvValueLayout kvValueLayout, @Nullable ToLongFunction valueTagProvider) { this.kvValueLayout = kvValueLayout; this.valueTagProvider = valueTagProvider; } - /** Returns an encoder for a layout without an internal value tag. */ + /** + * Returns an encoder for the given layout. Tagged values must supply their tag to {@link + * #encodeValue(BinaryValue, long)}. + */ public static ValueEncoder forLayout(KvValueLayout kvValueLayout) { checkNotNull(kvValueLayout, "kvValueLayout must not be null."); - if (kvValueLayout != KvValueLayout.PLAIN) { - throw new IllegalArgumentException( - "A value tag provider is required for this KV value layout."); - } - return PLAIN_ENCODER; + return kvValueLayout == KvValueLayout.PLAIN + ? PLAIN_ENCODER + : new ValueEncoder(kvValueLayout, null); } /** Returns an encoder for a layout with an internal value tag. */ @@ -66,11 +76,27 @@ public boolean hasValueTag() { /** Encodes a binary value using the layout bound to this encoder. */ public byte[] encodeValue(BinaryValue value) { + checkState( + !kvValueLayout.hasValueTag() || valueTagProvider != null, + "An explicit value tag is required for this KV value encoder."); + return encodeValueInternal( + value, valueTagProvider == null ? 0L : valueTagProvider.applyAsLong(value.row)); + } + + /** Encodes a binary value with the supplied opaque value tag. */ + public byte[] encodeValue(BinaryValue value, long valueTag) { + checkState( + kvValueLayout.hasValueTag(), + "An explicit value tag is not supported for this KV value layout."); + return encodeValueInternal(value, valueTag); + } + + private byte[] encodeValueInternal(BinaryValue value, long valueTag) { int rowPayloadOffset = kvValueLayout.rowPayloadOffset(); byte[] values = new byte[rowPayloadOffset + value.row.getSizeInBytes()]; kvValueLayout.writeSchemaId(values, value.schemaId); - if (valueTagProvider != null) { - kvValueLayout.writeValueTag(values, valueTagProvider.applyAsLong(value.row)); + if (kvValueLayout.hasValueTag()) { + kvValueLayout.writeValueTag(values, valueTag); } value.row.copyTo(values, rowPayloadOffset); return values; diff --git a/fluss-common/src/test/java/org/apache/fluss/row/encode/KvValueLayoutTest.java b/fluss-common/src/test/java/org/apache/fluss/row/encode/KvValueLayoutTest.java index a884fd05d16..98ba190f36f 100644 --- a/fluss-common/src/test/java/org/apache/fluss/row/encode/KvValueLayoutTest.java +++ b/fluss-common/src/test/java/org/apache/fluss/row/encode/KvValueLayoutTest.java @@ -68,6 +68,22 @@ void testLongTagKeepsRpcValueBodyAsSuffix() { assertThat(KvValueLayout.TAGGED.toValueBodySlice(null)).isNull(); } + @Test + void testEncodeValueWithCallerProvidedTag() { + BinaryRow row = compactedRow(DATA1_ROW_TYPE, new Object[] {1, "a"}); + byte[] value = + ValueEncoder.forLayout(KvValueLayout.TAGGED) + .encodeValue(new BinaryValue(DEFAULT_SCHEMA_ID, row), 42L); + + assertThat(KvValueLayout.TAGGED.readValueTag(MemorySegment.wrap(value))).isEqualTo(42L); + assertThatThrownBy( + () -> + ValueEncoder.forLayout(KvValueLayout.TAGGED) + .encodeValue(new BinaryValue(DEFAULT_SCHEMA_ID, row))) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("explicit value tag"); + } + @Test void testTwoArgumentValueDecoderUsesPlainLayout() { BinaryRow row = compactedRow(DATA1_ROW_TYPE, new Object[] {1, "a"}); diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/enumerator/TieringSourceEnumerator.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/enumerator/TieringSourceEnumerator.java index f1bf98527af..eb27397c9c6 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/enumerator/TieringSourceEnumerator.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/enumerator/TieringSourceEnumerator.java @@ -158,7 +158,7 @@ public void start() { this.coordinatorGateway = GatewayClientProxy.createGatewayProxy( metadataUpdater::getCoordinatorServer, rpcClient, CoordinatorGateway.class); - this.splitGenerator = new TieringSplitGenerator(flussAdmin); + this.splitGenerator = new TieringSplitGenerator(flussAdmin, metadataUpdater); LOG.info("Starting register Tiering Service to Fluss Coordinator..."); try { diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/split/TieringSplitGenerator.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/split/TieringSplitGenerator.java index a4b2638f309..5dd0bad2374 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/split/TieringSplitGenerator.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/split/TieringSplitGenerator.java @@ -22,8 +22,10 @@ import org.apache.fluss.client.initializer.OffsetsInitializer.BucketOffsetsRetriever; import org.apache.fluss.client.metadata.KvSnapshots; import org.apache.fluss.client.metadata.LakeSnapshot; +import org.apache.fluss.client.metadata.MetadataUpdater; import org.apache.fluss.exception.LakeTableSnapshotNotExistException; import org.apache.fluss.metadata.PartitionInfo; +import org.apache.fluss.metadata.PhysicalTablePath; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.metadata.TablePath; @@ -36,6 +38,7 @@ import javax.annotation.Nullable; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; @@ -43,6 +46,7 @@ import java.util.stream.IntStream; import static org.apache.fluss.client.table.scanner.log.LogScanner.EARLIEST_OFFSET; +import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; import static org.apache.fluss.utils.Preconditions.checkState; /** A generator for lake splits. */ @@ -51,9 +55,11 @@ public class TieringSplitGenerator { private static final Logger LOG = LoggerFactory.getLogger(TieringSplitGenerator.class); private final Admin flussAdmin; + private final MetadataUpdater metadataUpdater; - public TieringSplitGenerator(Admin flussAdmin) { + public TieringSplitGenerator(Admin flussAdmin, MetadataUpdater metadataUpdater) { this.flussAdmin = flussAdmin; + this.metadataUpdater = metadataUpdater; } public List generateTableSplits(TableInfo tableInfo) throws Exception { @@ -92,6 +98,21 @@ public List generateTableSplits(TableInfo tableInfo) throws Except Collectors.toMap( PartitionInfo::getPartitionId, PartitionInfo::getPartitionName)); + if (tableInfo.getTableConfig().isHistoricalPartitionEnabled()) { + // The internal historical partition is intentionally omitted from + // listPartitionInfos(), but tiering must consume it to synchronize historical + // writes to the lake table. Resolve it explicitly and include it in the splits. + PhysicalTablePath historicalPath = + PhysicalTablePath.of(tablePath, HISTORICAL_PARTITION_VALUE); + // Partition metadata is decoded using the tableId-to-path mapping already present + // in the Cluster, so initialize the table metadata before requesting the internal + // partition directly. + metadataUpdater.checkAndUpdateTableMetadata(Collections.singleton(tablePath)); + metadataUpdater.checkAndUpdatePartitionMetadata(historicalPath); + partitionNameById.put( + metadataUpdater.getPartitionIdOrElseThrow(historicalPath), + HISTORICAL_PARTITION_VALUE); + } return generatePartitionTableSplit( tableInfo, partitionNameById, bucketOffsetsRetriever, lakeSnapshotInfo); @@ -112,6 +133,7 @@ private List generatePartitionTableSplit( for (Map.Entry partitionNameByIdEntry : partitionNameById.entrySet()) { long partitionId = partitionNameByIdEntry.getKey(); String partitionName = partitionNameByIdEntry.getValue(); + boolean historicalPartition = HISTORICAL_PARTITION_VALUE.equals(partitionName); Map latestBucketsOffset = bucketOffsetsRetriever.latestOffsets( partitionName, @@ -120,21 +142,31 @@ private List generatePartitionTableSplit( .collect(Collectors.toList())); KvSnapshots latestKvSnapshots = null; if (tableInfo.hasPrimaryKey()) { - // get the table partition latest kv snapshot info - try { + if (historicalPartition) { + // Historical KV replicas use the lake snapshot as their durable base and tier + // only the retained WAL, so they have no local KV snapshots to tier. latestKvSnapshots = - flussAdmin - .getLatestKvSnapshots(tableInfo.getTablePath(), partitionName) - .get(); - } catch (Exception e) { - throw new FlinkRuntimeException( - String.format( - "Failed to get table snapshot for table %s and partition %s", - tableInfo.getTablePath(), partitionName), - ExceptionUtils.stripCompletionException(e)); + new KvSnapshots( + tableInfo.getTableId(), + partitionId, + Collections.emptyMap(), + Collections.emptyMap()); + } else { + try { + latestKvSnapshots = + flussAdmin + .getLatestKvSnapshots( + tableInfo.getTablePath(), partitionName) + .get(); + } catch (Exception e) { + throw new FlinkRuntimeException( + String.format( + "Failed to get table snapshot for table %s and partition %s", + tableInfo.getTablePath(), partitionName), + ExceptionUtils.stripCompletionException(e)); + } } } - splits.addAll( generateTableSplit( tableInfo, diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java index 43497c23781..67a4c016af0 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java @@ -61,10 +61,13 @@ import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Supplier; import static org.apache.fluss.config.ConfigOptions.KV_FORMAT_VERSION_2; import static org.apache.fluss.lake.paimon.PaimonLakeCatalog.LEGACY_SYSTEM_COLUMNS; @@ -87,11 +90,16 @@ * lookup I/O failure refreshes that partition-bucket with the files from the latest snapshot and * retries once. * + *

An explicit refresh request is applied during the next lookup initialization. It rescans every + * registered partition-bucket and updates its file set in place. Paimon keeps lookup files for data + * files that remain active and lazily downloads lookup files only for newly added data files. + * *

Calls to {@link LocalTableQuery#lookup} are serialized because Paimon 2.0 shares mutable * lookup-store comparator state across local lookup files. * *

Close is expected only after the owner has drained active lookups. It is synchronized with - * lazy initialization, but deliberately does not add a lifecycle lock to every lookup. + * lazy initialization and file-set updates, but deliberately does not add a lifecycle lock to every + * lookup. */ public class PaimonLakeTableLookuper implements LakeTableLookuper { @@ -104,7 +112,8 @@ public class PaimonLakeTableLookuper implements LakeTableLookuper { private final ThreadLocal lookupFileDownloaded; private final Object paimonLookupLock; - private final Object initializationLock; + // Guards lazy initialization, close, and registered file-set updates. + private final Object lookupStateLock; private final Map> registeredFiles; private @Nullable Catalog catalog; @@ -117,8 +126,10 @@ public class PaimonLakeTableLookuper implements LakeTableLookuper { private @Nullable CompactedKeyDecoder compactedKeyDecoder; private volatile @Nullable LocalTableQuery localTableQuery; - // Guarded by initializationLock. + // Guarded by lookupStateLock. private volatile boolean closed; + private volatile boolean refreshRequired; + private volatile boolean refreshInProgress; /** Creates a lookuper with the specified local lookup cache limit. */ public PaimonLakeTableLookuper( @@ -138,7 +149,7 @@ public PaimonLakeTableLookuper( this.diskWriteGuard = checkNotNull(diskWriteGuard, "diskWriteGuard must not be null."); this.lookupFileDownloaded = new ThreadLocal<>(); this.paimonLookupLock = new Object(); - this.initializationLock = new Object(); + this.lookupStateLock = new Object(); this.registeredFiles = new ConcurrentHashMap<>(); } @@ -147,7 +158,7 @@ public PaimonLakeTableLookuper( checkNotNull(key, "key must not be null."); checkNotNull(context, "context must not be null."); checkNotClosed(); - ensureInitialized(context.valueRowType()); + initialize(context.valueRowType()); try (TrackingMetrics ignored = new TrackingMetrics(lookupFileDownloaded, context)) { return lookupInternal(key, context); @@ -161,9 +172,15 @@ public PaimonLakeTableLookuper( } } + @Override + public void refresh() { + checkNotClosed(); + refreshRequired = true; + } + @Override public void close() { - synchronized (initializationLock) { + synchronized (lookupStateLock) { if (closed) { return; } @@ -187,17 +204,31 @@ private void checkNotClosed() { } } - private void ensureInitialized(RowType valueRowType) throws Exception { - if (localTableQuery == null) { - synchronized (initializationLock) { + private void initialize(RowType valueRowType) throws Exception { + if (localTableQuery == null || refreshRequired || refreshInProgress) { + synchronized (lookupStateLock) { if (localTableQuery == null) { - initialize(valueRowType); + initializeLookupState(valueRowType); + } + if (refreshRequired) { + // Clear the flag before refreshing so a concurrent request is retained for the + // next lookup. Restore it if this refresh fails. + refreshInProgress = true; + refreshRequired = false; + try { + refreshFilesFromLatestSnapshot(); + } catch (RuntimeException e) { + refreshRequired = true; + throw e; + } finally { + refreshInProgress = false; + } } } } } - private void initialize(RowType valueRowType) throws Exception { + private void initializeLookupState(RowType valueRowType) throws Exception { Catalog newCatalog = null; IOManager newIOManager = null; LocalTableQuery newLocalTableQuery = null; @@ -255,6 +286,17 @@ private void initialize(RowType valueRowType) throws Exception { } } + private void refreshFilesFromLatestSnapshot() { + Map> filesBeforeRefresh = + new LinkedHashMap<>(registeredFiles); + Map> latestFiles = + scanDataFiles(filesBeforeRefresh.keySet()); + filesBeforeRefresh.forEach( + (partitionBucket, files) -> + refreshFiles( + partitionBucket, files, () -> latestFiles.get(partitionBucket))); + } + private FileStoreTable withLookupCacheOptions(FileStoreTable table) { String key = CoreOptions.LOOKUP_CACHE_MAX_DISK_SIZE.key(); String maxDiskSize = new MemorySize(lookupCacheMaxDiskBytes).toString(); @@ -338,11 +380,12 @@ private org.apache.paimon.data.BinaryRow getKey(byte[] key, LookupContext contex int bucket, org.apache.paimon.data.InternalRow key) throws IOException { - List filesBeforeLookup = initializeFiles(partition, bucket); + PaimonPartitionBucket partitionBucket = new PaimonPartitionBucket(partition, bucket); + List filesBeforeLookup = getOrInitializeFiles(partitionBucket); try { return lookupLocalTable(partition, bucket, key); } catch (IOException firstError) { - refreshFilesIfUnchanged(partition, bucket, filesBeforeLookup); + refreshFiles(partitionBucket, filesBeforeLookup, () -> scanDataFiles(partitionBucket)); try { return lookupLocalTable(partition, bucket, key); } catch (IOException retryError) { @@ -366,56 +409,106 @@ private org.apache.paimon.data.BinaryRow getKey(byte[] key, LookupContext contex } } - private List initializeFiles( - org.apache.paimon.data.BinaryRow partition, int bucket) { - PaimonPartitionBucket partitionBucket = new PaimonPartitionBucket(partition, bucket); - return registeredFiles.computeIfAbsent( - partitionBucket, - ignored -> scanAndUpdateFiles(partition, bucket, Collections.emptyList())); + private List getOrInitializeFiles(PaimonPartitionBucket partitionBucket) { + List files = registeredFiles.get(partitionBucket); + if (files != null) { + return files; + } + // Coordinate only first registration with bulk refresh; existing entries use the fast path. + synchronized (lookupStateLock) { + return registeredFiles.computeIfAbsent( + partitionBucket, + ignored -> { + List latestFiles = scanDataFiles(partitionBucket); + return applyFileRefresh( + partitionBucket, Collections.emptyList(), latestFiles); + }); + } } - private void refreshFilesIfUnchanged( - org.apache.paimon.data.BinaryRow partition, - int bucket, - List filesBeforeLookup) { - PaimonPartitionBucket partitionBucket = new PaimonPartitionBucket(partition, bucket); - registeredFiles.compute( - partitionBucket, - (ignored, currentFiles) -> { - List files = - checkNotNull( - currentFiles, "Partition-bucket files must be initialized."); - return files == filesBeforeLookup - ? scanAndUpdateFiles(partition, bucket, filesBeforeLookup) - : files; - }); + private void refreshFiles( + PaimonPartitionBucket partitionBucket, + List filesBeforeRefresh, + Supplier> latestFilesSupplier) { + synchronized (lookupStateLock) { + registeredFiles.compute( + partitionBucket, + (ignored, currentFiles) -> { + List files = + checkNotNull( + currentFiles, + "Partition-bucket files must be initialized."); + // File lists are immutable and replaced on refresh. Identity equality means + // no other refresh has updated this partition-bucket since the caller + // captured it. + return files == filesBeforeRefresh + ? applyFileRefresh( + partitionBucket, + filesBeforeRefresh, + latestFilesSupplier.get()) + : files; + }); + } } - private List scanAndUpdateFiles( - org.apache.paimon.data.BinaryRow partition, - int bucket, - List filesBeforeRefresh) { - List latestFiles = scanDataFiles(partition, bucket); + private List applyFileRefresh( + PaimonPartitionBucket partitionBucket, + List filesBeforeRefresh, + List latestFiles) { + org.apache.paimon.data.BinaryRow partition = partitionBucket.getPartition(); + int bucket = partitionBucket.getBucket(); localTableQuery.refreshFiles(partition, bucket, filesBeforeRefresh, latestFiles); return latestFiles; } - private List scanDataFiles( - org.apache.paimon.data.BinaryRow partition, int bucket) { - LinkedHashMap dataFilesByName = new LinkedHashMap<>(); + private List scanDataFiles(PaimonPartitionBucket partitionBucket) { + return scanDataFiles(Collections.singleton(partitionBucket)).get(partitionBucket); + } + + private Map> scanDataFiles( + Set partitionBuckets) { + if (partitionBuckets.isEmpty()) { + return Collections.emptyMap(); + } + + Set partitions = new HashSet<>(); + Set buckets = new HashSet<>(); + Map> dataFilesByPartitionBucket = + new LinkedHashMap<>(); + for (PaimonPartitionBucket partitionBucket : partitionBuckets) { + partitions.add(partitionBucket.getPartition()); + buckets.add(partitionBucket.getBucket()); + dataFilesByPartitionBucket.put(partitionBucket, new LinkedHashMap<>()); + } + InnerTableScan tableScan = fileStoreTable .newScan() - .withPartitionFilter(Collections.singletonList(partition)) - .withBucket(bucket); + .withPartitionFilter(new ArrayList<>(partitions)) + .withBucketFilter(buckets::contains); for (Split split : tableScan.plan().splits()) { if (split instanceof DataSplit) { - for (DataFileMeta file : ((DataSplit) split).dataFiles()) { - dataFilesByName.put(file.fileName(), file); + DataSplit dataSplit = (DataSplit) split; + LinkedHashMap dataFilesByName = + dataFilesByPartitionBucket.get( + new PaimonPartitionBucket( + dataSplit.partition(), dataSplit.bucket())); + if (dataFilesByName != null) { + for (DataFileMeta file : dataSplit.dataFiles()) { + dataFilesByName.put(file.fileName(), file); + } } } } - return Collections.unmodifiableList(new ArrayList<>(dataFilesByName.values())); + + Map> latestFiles = new LinkedHashMap<>(); + dataFilesByPartitionBucket.forEach( + (partitionBucket, dataFilesByName) -> + latestFiles.put( + partitionBucket, + Collections.unmodifiableList( + new ArrayList<>(dataFilesByName.values())))); + return latestFiles; } private byte[] encodeValue( diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeCommitter.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeCommitter.java index c23cc373cda..7b16280eaf8 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeCommitter.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeCommitter.java @@ -35,6 +35,7 @@ import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.TableSnapshot; import org.apache.paimon.table.sink.CommitCallback; +import org.apache.paimon.table.sink.CommitMessage; import org.apache.paimon.table.sink.TableCommitImpl; import org.apache.paimon.utils.SnapshotManager; import org.slf4j.Logger; @@ -91,7 +92,9 @@ public PaimonCommittable toCommittable(List paimonWriteResult throws IOException { ManifestCommittable committable = new ManifestCommittable(COMMIT_IDENTIFIER); for (PaimonWriteResult paimonWriteResult : paimonWriteResults) { - committable.addFileCommittable(paimonWriteResult.commitMessage()); + for (CommitMessage commitMessage : paimonWriteResult.commitMessages()) { + committable.addFileCommittable(commitMessage); + } } return new PaimonCommittable(committable); } diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeWriter.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeWriter.java index 9b82403b6ce..a4bb0d44a41 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeWriter.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeWriter.java @@ -32,7 +32,6 @@ import org.apache.paimon.CoreOptions; import org.apache.paimon.catalog.Catalog; import org.apache.paimon.table.FileStoreTable; -import org.apache.paimon.table.sink.CommitMessage; import java.io.IOException; import java.util.Collections; @@ -40,6 +39,7 @@ import java.util.Map; import static org.apache.fluss.lake.paimon.utils.PaimonConversions.toPaimon; +import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; /** Implementation of {@link LakeWriter} for Paimon. */ public class PaimonLakeWriter implements LakeWriter, SupportsRecordBatchWrite { @@ -58,6 +58,8 @@ public PaimonLakeWriter( List partitionKeys = fileStoreTable.partitionKeys(); RowType flussRowType = writerInitContext.tableInfo().getRowType(); + boolean historicalPartition = + HISTORICAL_PARTITION_VALUE.equals(writerInitContext.partition()); // FIP-27: detect whether the target Paimon table is a clean table (only user columns) or a // legacy table (carrying the three Fluss system columns). Writers emit system columns only @@ -72,7 +74,8 @@ public PaimonLakeWriter( writerInitContext.partition(), partitionKeys, flussRowType, - paimonIncludingSystemColumns) + paimonIncludingSystemColumns, + historicalPartition) : new MergeTreeWriter( fileStoreTable, writerInitContext.tableBucket(), @@ -80,7 +83,8 @@ public PaimonLakeWriter( partitionKeys, flussRowType, writerInitContext.ioTmpDirs(), - paimonIncludingSystemColumns); + paimonIncludingSystemColumns, + historicalPartition); } @Override @@ -113,13 +117,11 @@ public void write(RecordBatch recordBatch) throws IOException { @Override public PaimonWriteResult complete() throws IOException { - CommitMessage commitMessage; try { - commitMessage = recordWriter.complete(); + return new PaimonWriteResult(recordWriter.complete()); } catch (Exception e) { throw new IOException("Failed to complete Paimon write.", e); } - return new PaimonWriteResult(commitMessage); } @Override diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonWriteResult.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonWriteResult.java index 70575c00e16..b2d40fbb25f 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonWriteResult.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonWriteResult.java @@ -20,19 +20,29 @@ import org.apache.paimon.table.sink.CommitMessage; import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; /** The write result of Paimon lake writer to pass to committer to commit. */ -public class PaimonWriteResult implements Serializable { +public final class PaimonWriteResult implements Serializable { private static final long serialVersionUID = 1L; - private final CommitMessage commitMessage; + private final List commitMessages; - public PaimonWriteResult(CommitMessage commitMessage) { - this.commitMessage = commitMessage; + /** Creates a write result containing all commit messages produced by one lake writer. */ + public PaimonWriteResult(List commitMessages) { + checkNotNull(commitMessages, "commitMessages must not be null"); + checkArgument(!commitMessages.isEmpty(), "commitMessages must not be empty"); + this.commitMessages = Collections.unmodifiableList(new ArrayList<>(commitMessages)); } - public CommitMessage commitMessage() { - return commitMessage; + /** Returns all commit messages produced by the lake writer. */ + public List commitMessages() { + return commitMessages; } } diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonWriteResultSerializer.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonWriteResultSerializer.java index 7efb3d37548..3ffecc9855d 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonWriteResultSerializer.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonWriteResultSerializer.java @@ -19,10 +19,14 @@ import org.apache.fluss.lake.serializer.SimpleVersionedSerializer; +import org.apache.paimon.io.DataInputViewStreamWrapper; +import org.apache.paimon.io.DataOutputSerializer; import org.apache.paimon.table.sink.CommitMessage; import org.apache.paimon.table.sink.CommitMessageSerializer; +import java.io.ByteArrayInputStream; import java.io.IOException; +import java.util.List; /** The {@link SimpleVersionedSerializer} for {@link PaimonWriteResult}. */ public class PaimonWriteResultSerializer implements SimpleVersionedSerializer { @@ -38,8 +42,9 @@ public int getVersion() { @Override public byte[] serialize(PaimonWriteResult paimonWriteResult) throws IOException { - CommitMessage commitMessage = paimonWriteResult.commitMessage(); - return messageSer.serialize(commitMessage); + DataOutputSerializer output = new DataOutputSerializer(64); + messageSer.serializeList(paimonWriteResult.commitMessages(), output); + return output.getCopyOfBuffer(); } @Override @@ -52,7 +57,11 @@ public PaimonWriteResult deserialize(int version, byte[] serialized) throws IOEx + version + "."); } - CommitMessage commitMessage = messageSer.deserialize(messageSer.getVersion(), serialized); - return new PaimonWriteResult(commitMessage); + try (DataInputViewStreamWrapper input = + new DataInputViewStreamWrapper(new ByteArrayInputStream(serialized))) { + List commitMessages = + messageSer.deserializeList(messageSer.getVersion(), input); + return new PaimonWriteResult(commitMessages); + } } } diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/RecordWriter.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/RecordWriter.java index 2260d553bc9..25d43d8400f 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/RecordWriter.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/RecordWriter.java @@ -31,6 +31,7 @@ import java.util.List; import static org.apache.fluss.lake.paimon.utils.PaimonConversions.toPaimonPartition; +import static org.apache.fluss.utils.Preconditions.checkNotNull; import static org.apache.fluss.utils.Preconditions.checkState; /** A base interface to write {@link LogRecord} to Paimon. */ @@ -40,7 +41,9 @@ public abstract class RecordWriter implements AutoCloseable { protected final RowType tableRowType; protected final int bucket; protected final List partitionKeys; - protected final BinaryRow partition; + protected final boolean historicalPartition; + // Null for historical writers, which derive the original partition from each record. + protected final @Nullable BinaryRow fixedPartition; protected final FlussRecordAsPaimonRow flussRecordAsPaimonRow; public RecordWriter( @@ -50,17 +53,21 @@ public RecordWriter( @Nullable String partition, List partitionKeys, org.apache.fluss.types.RowType flussRowType, - boolean paimonIncludingSystemColumns) { + boolean paimonIncludingSystemColumns, + boolean historicalPartition) { this.tableWrite = tableWrite; this.tableRowType = tableRowType; this.bucket = tableBucket.getBucket(); this.partitionKeys = partitionKeys; - if (partition == null || partitionKeys.isEmpty()) { + this.historicalPartition = historicalPartition; + if (historicalPartition) { + this.fixedPartition = null; + } else if (partition == null || partitionKeys.isEmpty()) { // non-partitioned table - this.partition = BinaryRow.EMPTY_ROW; + this.fixedPartition = BinaryRow.EMPTY_ROW; } else { // eagerly resolve BinaryRow partition from partition name string - this.partition = resolvePartition(partition, partitionKeys, flussRowType); + this.fixedPartition = resolvePartition(partition, partitionKeys, flussRowType); } this.flussRecordAsPaimonRow = new FlussRecordAsPaimonRow( @@ -69,19 +76,31 @@ public RecordWriter( public abstract void write(LogRecord record) throws Exception; - CommitMessage complete() throws Exception { + List complete() throws Exception { List commitMessages = tableWrite.prepareCommit(); - checkState( - commitMessages.size() == 1, - "The size of CommitMessage must be 1, but got %s.", - commitMessages); - return commitMessages.get(0); + // A normal writer targets one fixed partition, while a historical writer may write to + // multiple original partitions and therefore produce multiple commit messages. + if (!historicalPartition) { + checkState( + commitMessages.size() == 1, + "The size of CommitMessage must be 1, but got %s.", + commitMessages); + } + return commitMessages; } public void close() throws Exception { tableWrite.close(); } + /** Sets the current Fluss record and returns the Paimon partition it should be written to. */ + protected BinaryRow prepareRecordAndGetPartition(LogRecord record) { + flussRecordAsPaimonRow.setFlussRecord(record); + return historicalPartition + ? tableWrite.getPartition(flussRecordAsPaimonRow) + : checkNotNull(fixedPartition); + } + /** * Resolves a Paimon {@link BinaryRow} partition from the partition name string by parsing each * partition value to its typed Fluss representation, constructing a synthetic row, and diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyArrowBatchHelper.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyArrowBatchHelper.java index ac3b75cad3b..83518179ddf 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyArrowBatchHelper.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyArrowBatchHelper.java @@ -44,6 +44,8 @@ import java.util.ArrayList; import java.util.List; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + /** * Helper class that encapsulates Arrow-dependent batch writing logic for append-only tables. * @@ -106,7 +108,11 @@ class AppendOnlyArrowBatchHelper implements AutoCloseable { * system columns (__bucket, __offset, __timestamp) and uses Paimon's {@link ArrowBundleRecords} * for efficient batch writing. */ - void writeArrowBatch(ArrowBatchData arrowBatchData, BinaryRow partition) throws Exception { + void writeArrowBatch( + ArrowBatchData arrowBatchData, + @Nullable BinaryRow fixedPartition, + boolean historicalPartition) + throws Exception { int writtenBucket = bucket; if (fileStoreTable.store().bucketMode() == BucketMode.BUCKET_UNAWARE) { writtenBucket = 0; @@ -119,7 +125,7 @@ void writeArrowBatch(ArrowBatchData arrowBatchData, BinaryRow partition) throws // the Paimon table schema. Write it directly without enriching system columns. ArrowBundleRecords cleanRecords = new ArrowBundleRecords(originalRoot, tableRowType, CASE_SENSITIVE); - tableWrite.writeBundle(partition, writtenBucket, cleanRecords); + writeArrowBundle(cleanRecords, fixedPartition, historicalPartition, writtenBucket); return; } @@ -133,7 +139,25 @@ void writeArrowBatch(ArrowBatchData arrowBatchData, BinaryRow partition) throws ArrowBundleRecords arrowBundleRecords = new ArrowBundleRecords(enrichedRoot, tableRowType, CASE_SENSITIVE); - tableWrite.writeBundle(partition, writtenBucket, arrowBundleRecords); + writeArrowBundle(arrowBundleRecords, fixedPartition, historicalPartition, writtenBucket); + } + + private void writeArrowBundle( + ArrowBundleRecords arrowBundleRecords, + @Nullable BinaryRow fixedPartition, + boolean historicalPartition, + int writtenBucket) + throws Exception { + if (historicalPartition) { + // writeBundle accepts one fixed partition, but a historical batch may contain rows + // from multiple original partitions. + for (InternalRow row : arrowBundleRecords) { + BinaryRow partition = tableWrite.getPartition(row); + tableWrite.getWrite().write(partition, writtenBucket, row); + } + } else { + tableWrite.writeBundle(checkNotNull(fixedPartition), writtenBucket, arrowBundleRecords); + } } /** diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyWriter.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyWriter.java index 23f61a33171..a6d4f4292fd 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyWriter.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyWriter.java @@ -23,6 +23,7 @@ import org.apache.fluss.record.LogRecord; import org.apache.fluss.types.RowType; +import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.table.BucketMode; import org.apache.paimon.table.FileStoreTable; @@ -54,7 +55,8 @@ public AppendOnlyWriter( @Nullable String partition, List partitionKeys, RowType flussRowType, - boolean paimonIncludingSystemColumns) { + boolean paimonIncludingSystemColumns, + boolean historicalPartition) { //noinspection unchecked super( (TableWriteImpl) @@ -65,14 +67,15 @@ public AppendOnlyWriter( partition, partitionKeys, flussRowType, - paimonIncludingSystemColumns); + paimonIncludingSystemColumns, + historicalPartition); this.fileStoreTable = fileStoreTable; this.paimonIncludingSystemColumns = paimonIncludingSystemColumns; } @Override public void write(LogRecord record) throws Exception { - flussRecordAsPaimonRow.setFlussRecord(record); + BinaryRow targetPartition = prepareRecordAndGetPartition(record); // hacky, call internal method tableWrite.getWrite() to support // to write to given partition, otherwise, it'll always extract a partition from Paimon row @@ -82,7 +85,7 @@ public void write(LogRecord record) throws Exception { if (fileStoreTable.store().bucketMode() == BucketMode.BUCKET_UNAWARE) { writtenBucket = 0; } - tableWrite.getWrite().write(partition, writtenBucket, flussRecordAsPaimonRow); + tableWrite.getWrite().write(targetPartition, writtenBucket, flussRecordAsPaimonRow); } /** @@ -104,7 +107,7 @@ public void writeArrowBatch(ArrowBatchData arrowBatchData) throws Exception { } else { helper = (AppendOnlyArrowBatchHelper) arrowBatchHelper; } - helper.writeArrowBatch(arrowBatchData, partition); + helper.writeArrowBatch(arrowBatchData, fixedPartition, historicalPartition); } @Override diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/mergetree/MergeTreeWriter.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/mergetree/MergeTreeWriter.java index 37aeef7afe6..4dbc8100cb5 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/mergetree/MergeTreeWriter.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/mergetree/MergeTreeWriter.java @@ -23,6 +23,7 @@ import org.apache.fluss.types.RowType; import org.apache.paimon.KeyValue; +import org.apache.paimon.data.BinaryRow; import org.apache.paimon.disk.IOManager; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.sink.RowKeyExtractor; @@ -44,23 +45,6 @@ public class MergeTreeWriter extends RecordWriter { private final IOManager ioManager; - public MergeTreeWriter( - FileStoreTable fileStoreTable, - TableBucket tableBucket, - @Nullable String partition, - List partitionKeys, - RowType flussRowType, - boolean paimonIncludingSystemColumns) { - this( - fileStoreTable, - tableBucket, - partition, - partitionKeys, - flussRowType, - (String[]) null, - paimonIncludingSystemColumns); - } - public MergeTreeWriter( FileStoreTable fileStoreTable, TableBucket tableBucket, @@ -68,7 +52,8 @@ public MergeTreeWriter( List partitionKeys, RowType flussRowType, @Nullable String[] ioTmpDirs, - boolean paimonIncludingSystemColumns) { + boolean paimonIncludingSystemColumns, + boolean historicalPartition) { this( fileStoreTable, createIOManager(ioTmpDirs), @@ -76,7 +61,8 @@ public MergeTreeWriter( partition, partitionKeys, flussRowType, - paimonIncludingSystemColumns); + paimonIncludingSystemColumns, + historicalPartition); } MergeTreeWriter( @@ -86,7 +72,8 @@ public MergeTreeWriter( @Nullable String partition, List partitionKeys, RowType flussRowType, - boolean paimonIncludingSystemColumns) { + boolean paimonIncludingSystemColumns, + boolean historicalPartition) { super( createTableWrite(fileStoreTable, ioManager), fileStoreTable.rowType(), @@ -94,7 +81,8 @@ public MergeTreeWriter( partition, partitionKeys, flussRowType, - paimonIncludingSystemColumns); + paimonIncludingSystemColumns, + historicalPartition); this.rowKeyExtractor = fileStoreTable.createRowKeyExtractor(); this.ioManager = ioManager; } @@ -128,7 +116,7 @@ public void close() throws Exception { @Override public void write(LogRecord record) throws Exception { - flussRecordAsPaimonRow.setFlussRecord(record); + BinaryRow targetPartition = prepareRecordAndGetPartition(record); rowKeyExtractor.setRecord(flussRecordAsPaimonRow); keyValue.replace( @@ -139,6 +127,6 @@ public void write(LogRecord record) throws Exception { // hacky, call internal method tableWrite.getWrite() to support // to write to given partition, otherwise, it'll always extract a partition from Paimon row // which may be costly - tableWrite.getWrite().write(partition, bucket, keyValue); + tableWrite.getWrite().write(targetPartition, bucket, keyValue); } } diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/HistoricalPartitionLookupITCase.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/HistoricalPartitionITCase.java similarity index 60% rename from fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/HistoricalPartitionLookupITCase.java rename to fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/HistoricalPartitionITCase.java index 091c72bc21b..20b05d55f28 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/HistoricalPartitionLookupITCase.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/HistoricalPartitionITCase.java @@ -38,12 +38,15 @@ import org.apache.fluss.types.DataTypes; import org.apache.flink.core.execution.JobClient; +import org.apache.paimon.utils.CloseableIterator; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; import java.time.Duration; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -56,8 +59,8 @@ import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; import static org.assertj.core.api.Assertions.assertThat; -/** End-to-end IT case for looking up expired Fluss partitions from Paimon. */ -class HistoricalPartitionLookupITCase extends FlinkPaimonTieringTestBase { +/** End-to-end IT case for historical partition writes, tiering, recovery, and lookup. */ +class HistoricalPartitionITCase extends FlinkPaimonTieringTestBase { private static final String EXPIRED_PARTITION_NAME = "20240101"; private static final String SECOND_EXPIRED_PARTITION_NAME = "20240102"; @@ -76,6 +79,91 @@ protected static void beforeAll() { FlinkPaimonTieringTestBase.beforeAll(FLUSS_CLUSTER_EXTENSION.getClientConfig()); } + @Test + void testWriteAndTierHistoricalKvToPaimon() throws Exception { + TablePath tablePath = TablePath.of(DEFAULT_DB, "historical_write_tiering"); + Schema schema = partitionedPkSchema(true); + long tableId = + createTable( + tablePath, + partitionedDescriptor(schema, true, EXPIRED_PARTITION_RETENTION)); + + try { + long historicalPartitionId = waitUntilHistoricalPartitionReady(tablePath, tableId); + + InternalRow tieredRow = dataRow(true, 1, "unused", "Alice"); + assertThat(admin.listPartitionInfos(tablePath).get()) + .noneMatch(p -> EXPIRED_PARTITION_NAME.equals(p.getPartitionName())); + writeRows(tablePath, Collections.singletonList(tieredRow), false); + // Historical writes must not recreate the expired original partition. + assertThat(admin.listPartitionInfos(tablePath).get()) + .noneMatch(p -> EXPIRED_PARTITION_NAME.equals(p.getPartitionName())); + + TableBucket historicalBucket = new TableBucket(tableId, historicalPartitionId, 0); + assertThat(getLeaderReplica(historicalBucket).getLocalLogEndOffset()).isEqualTo(1); + tierAndVerifyPaimonRows( + tablePath, historicalBucket, 1L, "1|" + EXPIRED_PARTITION_NAME + "|Alice"); + + // Leave an untiered update after the Paimon snapshot. Restart recovery must apply this + // changelog over the tiered row. + InternalRow updatedRow = dataRow(true, 1, "unused", "Alice-updated"); + writeRows(tablePath, Collections.singletonList(updatedRow), false); + // FULL changelog emits UPDATE_BEFORE and UPDATE_AFTER for the update. + assertThat(getLeaderReplica(historicalBucket).getLocalLogEndOffset()).isEqualTo(3); + assertThat(getLeaderReplica(historicalBucket).getLakeLogEndOffset()).isEqualTo(1); + + restartLeaderAndVerifyLookup(tablePath, historicalBucket, schema, updatedRow); + + // Verify that the recovered local state can resolve the previous value for another + // update, and that a newly started tiering job can synchronize the resulting changelog. + InternalRow postRecoveryRow = dataRow(true, 1, "unused", "Alice-after-recovery"); + writeRows(tablePath, Collections.singletonList(postRecoveryRow), false); + assertThat(getLeaderReplica(historicalBucket).getLocalLogEndOffset()).isEqualTo(5); + assertThat(getLeaderReplica(historicalBucket).getLakeLogEndOffset()).isEqualTo(1); + tierAndVerifyPaimonRows( + tablePath, + historicalBucket, + 5L, + "1|" + EXPIRED_PARTITION_NAME + "|Alice-after-recovery"); + } finally { + dropTable(tablePath); + } + } + + @Test + void testWriteAndTierHistoricalLogToPaimon() throws Exception { + TablePath tablePath = TablePath.of(DEFAULT_DB, "historical_log_write_tiering"); + long tableId = + createTable( + tablePath, + partitionedDescriptor( + partitionedLogSchema(), true, EXPIRED_PARTITION_RETENTION)); + + try { + long historicalPartitionId = waitUntilHistoricalPartitionReady(tablePath, tableId); + List rows = + Arrays.asList( + row(1, EXPIRED_PARTITION_NAME, "Alice"), + row(2, EXPIRED_PARTITION_NAME, "Bob")); + + writeRows(tablePath, rows, true); + // Historical writes must not recreate the expired original partition. + assertThat(admin.listPartitionInfos(tablePath).get()) + .noneMatch(p -> EXPIRED_PARTITION_NAME.equals(p.getPartitionName())); + + TableBucket historicalBucket = new TableBucket(tableId, historicalPartitionId, 0); + assertThat(getLeaderReplica(historicalBucket).getLocalLogEndOffset()).isEqualTo(2); + tierAndVerifyPaimonRows( + tablePath, + historicalBucket, + 2L, + "1|" + EXPIRED_PARTITION_NAME + "|Alice", + "2|" + EXPIRED_PARTITION_NAME + "|Bob"); + } finally { + dropTable(tablePath); + } + } + @ParameterizedTest(name = "defaultBucketKey={0}") @ValueSource(booleans = {true, false}) void testLookupExpiredPartitionFromPaimon(boolean defaultBucketKey) throws Exception { @@ -86,7 +174,7 @@ void testLookupExpiredPartitionFromPaimon(boolean defaultBucketKey) throws Excep ? "historical_lookup_default_bucket" : "historical_lookup_bucket_subset"); Schema oldSchema = partitionedPkSchema(defaultBucketKey); - long tableId = createTable(tablePath, partitionedPkDescriptor(oldSchema)); + long tableId = createTable(tablePath, partitionedDescriptor(oldSchema, false)); // Enable historical lookup through ALTER TABLE to cover dynamic creation of the // coordinator-owned historical system partition. @@ -232,6 +320,76 @@ protected FlussClusterExtension getFlussClusterExtension() { return FLUSS_CLUSTER_EXTENSION; } + private static long waitUntilHistoricalPartitionReady(TablePath tablePath, long tableId) + throws Exception { + Optional historicalPartition = + FLUSS_CLUSTER_EXTENSION + .getZooKeeperClient() + .getPartition(tablePath, HISTORICAL_PARTITION_VALUE); + assertThat(historicalPartition).isPresent(); + long partitionId = historicalPartition.get().getPartitionId(); + FLUSS_CLUSTER_EXTENSION.waitUntilTablePartitionReady(tableId, partitionId); + return partitionId; + } + + private List readPaimonRows(TablePath tablePath) throws Exception { + List actualRows = new ArrayList<>(); + try (CloseableIterator rows = + getPaimonRowCloseableIterator(tablePath)) { + while (rows.hasNext()) { + org.apache.paimon.data.InternalRow row = rows.next(); + actualRows.add(row.getInt(0) + "|" + row.getString(1) + "|" + row.getString(2)); + } + } + return actualRows; + } + + private void tierAndVerifyPaimonRows( + TablePath tablePath, + TableBucket historicalBucket, + long expectedLogEndOffset, + String... expectedRows) + throws Exception { + JobClient jobClient = buildTieringJob(execEnv); + try { + assertReplicaStatus(historicalBucket, expectedLogEndOffset); + checkFlussOffsetsInSnapshot( + tablePath, Collections.singletonMap(historicalBucket, expectedLogEndOffset)); + assertThat(readPaimonRows(tablePath)).containsExactlyInAnyOrder(expectedRows); + } finally { + jobClient.cancel().get(); + } + } + + private void restartLeaderAndVerifyLookup( + TablePath tablePath, + TableBucket historicalBucket, + Schema schema, + InternalRow expectedRow) + throws Exception { + int tabletServerId = FLUSS_CLUSTER_EXTENSION.waitAndGetLeader(historicalBucket); + FLUSS_CLUSTER_EXTENSION.stopTabletServer(tabletServerId); + try { + FLUSS_CLUSTER_EXTENSION.startTabletServer(tabletServerId); + FLUSS_CLUSTER_EXTENSION.waitAndGetLeaderReplica(historicalBucket); + + try (Connection connection = ConnectionFactory.createConnection(clientConf); + Table table = connection.getTable(tablePath)) { + InternalRow actualRow = + table.newLookup() + .createLookuper() + .lookup(lookupKey(true, 1, "unused")) + .get() + .getSingletonRow(); + assertThatRow(actualRow).withSchema(schema.getRowType()).isEqualTo(expectedRow); + } + } finally { + if (FLUSS_CLUSTER_EXTENSION.getTabletServerById(tabletServerId) == null) { + FLUSS_CLUSTER_EXTENSION.startTabletServer(tabletServerId); + } + } + } + private static Schema partitionedPkSchema(boolean defaultBucketKey) { if (defaultBucketKey) { return Schema.newBuilder() @@ -250,6 +408,14 @@ private static Schema partitionedPkSchema(boolean defaultBucketKey) { .build(); } + private static Schema partitionedLogSchema() { + return Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("dt", DataTypes.STRING()) + .column("name", DataTypes.STRING()) + .build(); + } + private static Schema evolvedPartitionedPkSchema(boolean defaultBucketKey) { if (defaultBucketKey) { return Schema.newBuilder() @@ -270,23 +436,36 @@ private static Schema evolvedPartitionedPkSchema(boolean defaultBucketKey) { .build(); } - private static TableDescriptor partitionedPkDescriptor(Schema schema) { - return TableDescriptor.builder() - .schema(schema) - // This is the default bucket key for (id, dt), and a strict subset of the physical - // primary key for (id, sub_id, dt). - .distributedBy(1, "id") - .partitionedBy("dt") - .property(ConfigOptions.TABLE_AUTO_PARTITION_ENABLED, true) - .property(ConfigOptions.TABLE_AUTO_PARTITION_KEY, "dt") - .property(ConfigOptions.TABLE_AUTO_PARTITION_TIME_UNIT, AutoPartitionTimeUnit.DAY) - .property( - ConfigOptions.TABLE_AUTO_PARTITION_NUM_RETENTION, - INITIAL_PARTITION_RETENTION) - .property(ConfigOptions.TABLE_AUTO_PARTITION_TIMEZONE, "UTC") - .property(ConfigOptions.TABLE_DATALAKE_ENABLED, true) - .property(ConfigOptions.TABLE_DATALAKE_FRESHNESS, Duration.ofMillis(500)) - .build(); + private static TableDescriptor partitionedDescriptor( + Schema schema, boolean historicalPartitionEnabled) { + return partitionedDescriptor( + schema, historicalPartitionEnabled, INITIAL_PARTITION_RETENTION); + } + + private static TableDescriptor partitionedDescriptor( + Schema schema, boolean historicalPartitionEnabled, int partitionRetention) { + TableDescriptor.Builder builder = + TableDescriptor.builder() + .schema(schema) + // For primary-key tables, id is the default bucket key for (id, dt) and a + // strict subset of the physical primary key for (id, sub_id, dt). + .distributedBy(1, "id") + .partitionedBy("dt") + .property(ConfigOptions.TABLE_AUTO_PARTITION_ENABLED, true) + .property(ConfigOptions.TABLE_AUTO_PARTITION_KEY, "dt") + .property( + ConfigOptions.TABLE_AUTO_PARTITION_TIME_UNIT, + AutoPartitionTimeUnit.DAY) + .property( + ConfigOptions.TABLE_AUTO_PARTITION_NUM_RETENTION, + partitionRetention) + .property(ConfigOptions.TABLE_AUTO_PARTITION_TIMEZONE, "UTC") + .property(ConfigOptions.TABLE_DATALAKE_ENABLED, true) + .property(ConfigOptions.TABLE_DATALAKE_FRESHNESS, Duration.ofMillis(500)); + if (historicalPartitionEnabled) { + builder.property(ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_ENABLED, true); + } + return builder.build(); } private static InternalRow dataRow( diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java index f493f62317c..6492e53c88e 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java @@ -231,6 +231,101 @@ void testConcurrentFirstLookupsForDifferentPartitions() throws Exception { } } + @Test + void testRefreshesFilesWhenLakeSnapshotChanges() throws Exception { + TablePath tablePath = TablePath.of(DB, "refresh_registered_files"); + Schema schema = pkSchema(); + FileStoreTable table = createPaimonTable(tablePath, partitionedPkDescriptor(schema)); + writeAndCommitData( + table, + Collections.singletonMap( + 0, + Arrays.asList( + paimonRow(1, "20240101", "Alice"), + paimonRow(2, "20240102", "Bob")))); + + try (LakeTableLookuper lookuper = + new PaimonLakeTableLookuper( + paimonConfig, + tablePath, + tempWarehouseDir.getAbsolutePath(), + tableConfig(KvFormat.COMPACTED), + LOOKUP_CACHE_MAX_DISK_BYTES, + NO_OP_DISK_WRITE_GUARD)) { + LakeTableLookuper.LookupContext firstPartition = + lookupContext(schema, "20240101", 0, SCHEMA_ID); + LakeTableLookuper.LookupContext secondPartition = + lookupContext(schema, "20240102", 0, SCHEMA_ID); + // Register two partition-buckets and populate their local lookup files. + assertThat(lookuper.lookup(paimonKey(schema, 1, "20240101"), firstPartition)) + .isNotNull(); + assertThat(lookuper.lookup(paimonKey(schema, 2, "20240102"), secondPartition)) + .isNotNull(); + + writeAndCommitData( + table, + Collections.singletonMap( + 0, Collections.singletonList(paimonRow(3, "20240101", "Carol")))); + // The newly committed file is not registered before the explicit refresh. + assertThat(lookuper.lookup(paimonKey(schema, 3, "20240101"), firstPartition)).isNull(); + + List newFileDownloads = new ArrayList<>(); + List retainedFileDownloads = new ArrayList<>(); + List unchangedPartitionDownloads = new ArrayList<>(); + lookuper.refresh(); + + BinaryValue newFileValue = + decodeValue( + lookuper.lookup( + paimonKey(schema, 3, "20240101"), + lookupContext( + schema, + "20240101", + 0, + SCHEMA_ID, + (lookupTimeNanos, lookupFileDownloaded) -> + newFileDownloads.add(lookupFileDownloaded))), + SCHEMA_ID, + schema); + BinaryValue retainedFileValue = + decodeValue( + lookuper.lookup( + paimonKey(schema, 1, "20240101"), + lookupContext( + schema, + "20240101", + 0, + SCHEMA_ID, + (lookupTimeNanos, lookupFileDownloaded) -> + retainedFileDownloads.add( + lookupFileDownloaded))), + SCHEMA_ID, + schema); + BinaryValue unchangedValue = + decodeValue( + lookuper.lookup( + paimonKey(schema, 2, "20240102"), + lookupContext( + schema, + "20240102", + 0, + SCHEMA_ID, + (lookupTimeNanos, lookupFileDownloaded) -> + unchangedPartitionDownloads.add( + lookupFileDownloaded))), + SCHEMA_ID, + schema); + + assertRow(newFileValue.row, 3, "20240101", "Carol"); + assertRow(retainedFileValue.row, 1, "20240101", "Alice"); + assertRow(unchangedValue.row, 2, "20240102", "Bob"); + // Only the new data file needs a local lookup-file download after the bulk refresh. + assertThat(newFileDownloads).containsExactly(true); + assertThat(retainedFileDownloads).containsExactly(false); + assertThat(unchangedPartitionDownloads).containsExactly(false); + } + } + @Test void testDiskWriteLockBlocksOnlyLookupFileDownloads() throws Exception { TablePath tablePath = TablePath.of(DB, "disk_write_lock"); diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringTest.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringTest.java index a433af06c82..19c59600f92 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringTest.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringTest.java @@ -19,23 +19,31 @@ import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; +import org.apache.fluss.lake.batch.ArrowRecordBatch; import org.apache.fluss.lake.committer.CommittedLakeSnapshot; import org.apache.fluss.lake.committer.CommitterInitContext; import org.apache.fluss.lake.committer.LakeCommitter; import org.apache.fluss.lake.serializer.SimpleVersionedSerializer; import org.apache.fluss.lake.writer.LakeWriter; +import org.apache.fluss.lake.writer.SupportsRecordBatchWrite; import org.apache.fluss.lake.writer.WriterInitContext; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.record.ArrowBatchData; import org.apache.fluss.record.ChangeType; import org.apache.fluss.record.GenericRecord; import org.apache.fluss.record.LogRecord; import org.apache.fluss.row.BinaryString; import org.apache.fluss.row.GenericRow; +import org.apache.fluss.utils.UnshadedArrowReadUtils; import org.apache.fluss.utils.types.Tuple2; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.paimon.CoreOptions; import org.apache.paimon.catalog.Catalog; import org.apache.paimon.catalog.CatalogContext; @@ -77,11 +85,13 @@ import static org.apache.fluss.metadata.TableDescriptor.BUCKET_COLUMN_NAME; import static org.apache.fluss.metadata.TableDescriptor.OFFSET_COLUMN_NAME; import static org.apache.fluss.metadata.TableDescriptor.TIMESTAMP_COLUMN_NAME; +import static org.apache.fluss.record.ChangeType.APPEND_ONLY; import static org.apache.fluss.record.ChangeType.DELETE; import static org.apache.fluss.record.ChangeType.INSERT; import static org.apache.fluss.record.ChangeType.UPDATE_AFTER; import static org.apache.fluss.record.ChangeType.UPDATE_BEFORE; import static org.apache.fluss.record.TestData.DEFAULT_REMOTE_DATA_DIR; +import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; import static org.assertj.core.api.Assertions.assertThat; /** The UT for tiering to Paimon via {@link PaimonLakeTieringFactory}. */ @@ -212,6 +222,72 @@ void testTieringWriteTable(boolean isPrimaryKeyTable, boolean isPartitioned) thr } } + @Test + void testHistoricalPrimaryKeyTiering() throws Exception { + TablePath tablePath = TablePath.of("paimon", "test_historical_primary_key"); + TableInfo tableInfo = createHistoricalTable(tablePath, true); + long timestamp = 1_000L; + List records = + Arrays.asList( + historicalRecord(0L, timestamp, 1, "20240101", INSERT), + historicalRecord(1L, timestamp, 1, "20240102", INSERT)); + + PaimonWriteResult writeResult; + try (LakeWriter lakeWriter = + createLakeWriter(tablePath, 0, HISTORICAL_PARTITION_VALUE, 1L, tableInfo)) { + for (LogRecord record : records) { + lakeWriter.write(record); + } + writeResult = lakeWriter.complete(); + } + + SimpleVersionedSerializer serializer = + paimonLakeTieringFactory.getWriteResultSerializer(); + writeResult = + serializer.deserialize(serializer.getVersion(), serializer.serialize(writeResult)); + assertHistoricalPartitions(writeResult); + + try (LakeCommitter committer = + createLakeCommitter(tablePath, tableInfo, new Configuration())) { + committer.commit( + committer.toCommittable(Collections.singletonList(writeResult)), + Collections.emptyMap()); + } + assertThat(paimonCatalog.listPartitions(toPaimon(tablePath))) + .extracting(partition -> partition.spec().get("c3")) + .containsExactlyInAnyOrder("20240101", "20240102"); + } + + @Test + void testHistoricalArrowBatchTiering() throws Exception { + TablePath tablePath = TablePath.of("paimon", "test_historical_arrow"); + TableInfo tableInfo = createHistoricalTable(tablePath, false); + long baseOffset = 10L; + long timestamp = 1_000L; + List records = + Arrays.asList( + historicalRecord(baseOffset, timestamp, 1, "20240101", APPEND_ONLY), + historicalRecord(baseOffset + 1, timestamp, 2, "20240102", APPEND_ONLY)); + + PaimonWriteResult writeResult; + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE); + LakeWriter lakeWriter = + createLakeWriter(tablePath, 0, HISTORICAL_PARTITION_VALUE, 1L, tableInfo)) { + VectorSchemaRoot root = + VectorSchemaRoot.create( + UnshadedArrowReadUtils.toArrowSchema(tableInfo.getRowType()), + allocator); + try (ArrowRecordBatch arrowRecordBatch = + new ArrowRecordBatch(new ArrowBatchData(root, baseOffset, timestamp, 1))) { + writeArrowRows(root, records); + ((SupportsRecordBatchWrite) lakeWriter).write(arrowRecordBatch); + } + writeResult = lakeWriter.complete(); + } + + assertHistoricalPartitions(writeResult); + } + @Test void testEmptyCommitCreatesSnapshot() throws Exception { TablePath tablePath = TablePath.of("paimon", "test_empty_commit"); @@ -593,6 +669,12 @@ private void verifyLogTableRecordsThreePartition( actualRecords.close(); } + private void assertHistoricalPartitions(PaimonWriteResult writeResult) { + assertThat(writeResult.commitMessages()) + .extracting(message -> message.partition().getString(0).toString()) + .containsExactlyInAnyOrder("20240101", "20240102"); + } + private void verifyTableRecords( CloseableIterator actualRecords, List expectRecords, @@ -737,6 +819,28 @@ private GenericRecord toRecord(long offset, GenericRow row, ChangeType changeTyp return new GenericRecord(offset, System.currentTimeMillis(), changeType, row); } + private LogRecord historicalRecord( + long offset, long timestamp, int key, String partition, ChangeType changeType) { + GenericRow row = + GenericRow.of( + key, BinaryString.fromString("value"), BinaryString.fromString(partition)); + return new GenericRecord(offset, timestamp, changeType, row); + } + + private void writeArrowRows(VectorSchemaRoot root, List records) { + root.allocateNew(); + IntVector keyVector = (IntVector) root.getVector("c1"); + VarCharVector valueVector = (VarCharVector) root.getVector("c2"); + VarCharVector partitionVector = (VarCharVector) root.getVector("c3"); + for (int i = 0; i < records.size(); i++) { + org.apache.fluss.row.InternalRow row = records.get(i).getRow(); + keyVector.setSafe(i, row.getInt(0)); + valueVector.setSafe(i, row.getString(1).toBytes()); + partitionVector.setSafe(i, row.getString(2).toBytes()); + } + root.setRowCount(records.size()); + } + private CloseableIterator getPaimonRows( TablePath tablePath, @Nullable String partition, boolean isPrimaryKeyTable, int bucket) throws Exception { @@ -911,6 +1015,33 @@ private void createTable( doCreatePaimonTable(tablePath, builder); } + private TableInfo createHistoricalTable(TablePath tablePath, boolean isPrimaryKeyTable) + throws Exception { + createTable( + tablePath, + isPrimaryKeyTable, + true, + isPrimaryKeyTable ? 1 : null, + Collections.emptyMap()); + + org.apache.fluss.metadata.Schema.Builder schemaBuilder = + org.apache.fluss.metadata.Schema.newBuilder() + .column("c1", org.apache.fluss.types.DataTypes.INT()) + .column("c2", org.apache.fluss.types.DataTypes.STRING()) + .column("c3", org.apache.fluss.types.DataTypes.STRING()); + if (isPrimaryKeyTable) { + schemaBuilder.primaryKey("c1", "c3"); + } + TableDescriptor descriptor = + TableDescriptor.builder() + .schema(schemaBuilder.build()) + .partitionedBy("c3") + .distributedBy(1) + .property(ConfigOptions.TABLE_DATALAKE_ENABLED, true) + .build(); + return TableInfo.of(tablePath, 0, 1, descriptor, DEFAULT_REMOTE_DATA_DIR, 1L, 1L); + } + private void createMultiPartitionTable(TablePath tablePath) throws Exception { Schema.Builder builder = Schema.newBuilder() diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyArrowBatchCaseSensitivityTest.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyArrowBatchCaseSensitivityTest.java index e3d961d6605..31b664e1e55 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyArrowBatchCaseSensitivityTest.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyArrowBatchCaseSensitivityTest.java @@ -28,6 +28,7 @@ import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.paimon.FileStore; +import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.io.BundleRecords; import org.apache.paimon.table.BucketMode; @@ -48,7 +49,6 @@ import static org.apache.fluss.metadata.TableDescriptor.TIMESTAMP_COLUMN_NAME; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -150,10 +150,10 @@ private Object[] writeAndRead( fileStoreTable, tableWrite, paimonRowType, 0, legacyTable); ArrowBatchData batch = new ArrowBatchData(root.slice(0, root.getRowCount()), 0L, 1L, 1)) { - helper.writeArrowBatch(batch, null); + helper.writeArrowBatch(batch, BinaryRow.EMPTY_ROW, false); ArgumentCaptor captor = ArgumentCaptor.forClass(BundleRecords.class); - verify(tableWrite).writeBundle(isNull(), eq(0), captor.capture()); + verify(tableWrite).writeBundle(eq(BinaryRow.EMPTY_ROW), eq(0), captor.capture()); Iterator rows = captor.getValue().iterator(); assertThat(rows.hasNext()).isTrue(); InternalRow row = rows.next(); diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/ProduceLogResultForBucket.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/ProduceLogResultForBucket.java index 37efad6c646..434e4c68f43 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/ProduceLogResultForBucket.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/ProduceLogResultForBucket.java @@ -23,35 +23,69 @@ import org.apache.fluss.rpc.protocol.ApiError; import org.apache.fluss.rpc.protocol.Errors; +import javax.annotation.Nullable; + +import java.util.Objects; + /** Result of {@link ProduceLogRequest} for each table bucket. */ @Internal public class ProduceLogResultForBucket extends WriteResultForBucket { private final long baseOffset; + // Identifies the original partition for a historical write; null for a normal write. + private final @Nullable String originalPartitionName; public ProduceLogResultForBucket(TableBucket tableBucket, long baseOffset, long endOffset) { - this(tableBucket, baseOffset, endOffset, ApiError.NONE); + this(tableBucket, baseOffset, endOffset, ApiError.NONE, null); } public ProduceLogResultForBucket(TableBucket tableBucket, ApiError error) { - this(tableBucket, -1L, -1L, error); + this(tableBucket, -1L, -1L, error, null); + } + + public static ProduceLogResultForBucket historicalSuccess( + TableBucket tableBucket, + long baseOffset, + long endOffset, + String originalPartitionName) { + return new ProduceLogResultForBucket( + tableBucket, baseOffset, endOffset, ApiError.NONE, originalPartitionName); + } + + public static ProduceLogResultForBucket historicalFailure( + TableBucket tableBucket, ApiError error, String originalPartitionName) { + return new ProduceLogResultForBucket(tableBucket, -1L, -1L, error, originalPartitionName); } private ProduceLogResultForBucket( - TableBucket tableBucket, long baseOffset, long endOffset, ApiError error) { + TableBucket tableBucket, + long baseOffset, + long endOffset, + ApiError error, + @Nullable String originalPartitionName) { super(tableBucket, endOffset, error); this.baseOffset = baseOffset; + this.originalPartitionName = originalPartitionName; } public long getBaseOffset() { return baseOffset; } + /** Returns the original partition name for a historical write, or null for a normal write. */ + public @Nullable String getOriginalPartitionName() { + return originalPartitionName; + } + @Override public T copy(Errors newError) { //noinspection unchecked return (T) new ProduceLogResultForBucket( - tableBucket, baseOffset, getWriteLogEndOffset(), newError.toApiError()); + tableBucket, + baseOffset, + getWriteLogEndOffset(), + newError.toApiError(), + originalPartitionName); } @Override @@ -66,6 +100,12 @@ public boolean equals(Object o) { return false; } ProduceLogResultForBucket that = (ProduceLogResultForBucket) o; - return baseOffset == that.baseOffset; + return baseOffset == that.baseOffset + && Objects.equals(originalPartitionName, that.originalPartitionName); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), baseOffset, originalPartitionName); } } diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/client/ServerConnection.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/client/ServerConnection.java index 46ec0806110..1bc678452eb 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/client/ServerConnection.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/client/ServerConnection.java @@ -25,11 +25,14 @@ import org.apache.fluss.exception.InvalidServerTypeException; import org.apache.fluss.exception.NetworkException; import org.apache.fluss.exception.RetriableAuthenticationException; +import org.apache.fluss.exception.UnsupportedVersionException; import org.apache.fluss.rpc.messages.ApiMessage; import org.apache.fluss.rpc.messages.ApiVersionsRequest; import org.apache.fluss.rpc.messages.ApiVersionsResponse; import org.apache.fluss.rpc.messages.AuthenticateRequest; import org.apache.fluss.rpc.messages.AuthenticateResponse; +import org.apache.fluss.rpc.messages.ProduceLogRequest; +import org.apache.fluss.rpc.messages.PutKvRequest; import org.apache.fluss.rpc.metrics.ClientMetricGroup; import org.apache.fluss.rpc.metrics.ConnectionMetrics; import org.apache.fluss.rpc.protocol.ApiKeys; @@ -62,12 +65,16 @@ import java.util.concurrent.TimeUnit; import java.util.function.BiConsumer; +import static org.apache.fluss.rpc.util.CommonRpcMessageUtils.hasHistoricalProduce; +import static org.apache.fluss.rpc.util.CommonRpcMessageUtils.hasHistoricalPut; import static org.apache.fluss.utils.IOUtils.closeQuietly; /** Connection to a Netty server used by the {@link NettyClient}. */ @ThreadSafe final class ServerConnection { private static final Logger LOG = LoggerFactory.getLogger(ServerConnection.class); + private static final short HISTORICAL_PRODUCE_LOG_MIN_VERSION = 1; + private static final short HISTORICAL_PUT_KV_MIN_VERSION = 3; private final ServerNode node; @@ -309,6 +316,7 @@ private CompletableFuture doSend( if (serverApiVersions != null) { try { version = serverApiVersions.highestAvailableVersion(apiKey); + validateVersionCompatibility(apiKey, version, rawRequest); } catch (Exception e) { responseFuture.completeExceptionally(e); return responseFuture; @@ -361,6 +369,37 @@ private CompletableFuture doSend( } } + private void validateVersionCompatibility( + ApiKeys apiKey, short version, ApiMessage rawRequest) { + if (apiKey == ApiKeys.PRODUCE_LOG && version < HISTORICAL_PRODUCE_LOG_MIN_VERSION) { + ProduceLogRequest produceLogRequest = (ProduceLogRequest) rawRequest; + if (hasHistoricalProduce(produceLogRequest)) { + throw new UnsupportedVersionException( + "Historical partition writes require PRODUCE_LOG version " + + HISTORICAL_PRODUCE_LOG_MIN_VERSION + + " or newer, but server " + + node + + " negotiated version " + + version + + '.'); + } + } + + if (apiKey == ApiKeys.PUT_KV && version < HISTORICAL_PUT_KV_MIN_VERSION) { + PutKvRequest putKvRequest = (PutKvRequest) rawRequest; + if (hasHistoricalPut(putKvRequest)) { + throw new UnsupportedVersionException( + "Historical partition writes require PUT_KV version " + + HISTORICAL_PUT_KV_MIN_VERSION + + " or newer, but server " + + node + + " negotiated version " + + version + + '.'); + } + } + } + private void handleApiVersionsResponse(ApiMessage response, Throwable cause) { if (cause != null) { close(cause); diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/ApiKeys.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/ApiKeys.java index baf4256650e..ffdd1978de9 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/ApiKeys.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/ApiKeys.java @@ -42,7 +42,8 @@ public enum ApiKeys { GET_TABLE_SCHEMA(1011, 0, 0, PUBLIC), GET_METADATA(1012, 0, 0, PUBLIC), UPDATE_METADATA(1013, 0, 0, PRIVATE), - PRODUCE_LOG(1014, 0, 0, PUBLIC), + // Version 1: Supports original_partition_name in requests and responses for historical writes. + PRODUCE_LOG(1014, 0, 1, PUBLIC), FETCH_LOG(1015, 0, 0, PUBLIC), // Version 0: Uses lake's encoder for primary key encoding (legacy behavior). diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/util/CommonRpcMessageUtils.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/util/CommonRpcMessageUtils.java index 219b51087a4..befd3fc7f3e 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/util/CommonRpcMessageUtils.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/util/CommonRpcMessageUtils.java @@ -34,6 +34,7 @@ import org.apache.fluss.rpc.messages.PbPartitionSpec; import org.apache.fluss.rpc.messages.PbRemoteLogFetchInfo; import org.apache.fluss.rpc.messages.PbRemoteLogSegment; +import org.apache.fluss.rpc.messages.ProduceLogRequest; import org.apache.fluss.rpc.messages.PutKvRequest; import org.apache.fluss.rpc.protocol.ApiError; import org.apache.fluss.security.acl.AccessControlEntry; @@ -83,6 +84,17 @@ public static boolean hasHistoricalPut(PutKvRequest putKvRequest) { && putKvRequest.getBucketsReqAt(0).hasOriginalPartitionName(); } + /** + * Returns whether the produce-log request is for historical partition writes. + * + *

Normal and historical write buckets cannot be mixed in the same request, so the first + * bucket determines the request type. + */ + public static boolean hasHistoricalProduce(ProduceLogRequest produceLogRequest) { + return produceLogRequest.getBucketsReqsCount() > 0 + && produceLogRequest.getBucketsReqAt(0).hasOriginalPartitionName(); + } + public static List toPbAclInfos(Collection aclBindings) { return aclBindings.stream() .map(CommonRpcMessageUtils::toPbAclInfo) diff --git a/fluss-rpc/src/main/proto/FlussApi.proto b/fluss-rpc/src/main/proto/FlussApi.proto index 9d5e05fbdae..1ae6b1f126b 100644 --- a/fluss-rpc/src/main/proto/FlussApi.proto +++ b/fluss-rpc/src/main/proto/FlussApi.proto @@ -886,6 +886,8 @@ message PbProduceLogReqForBucket { optional int64 partition_id = 1; required int32 bucket_id = 2; required bytes records = 3; + // The original partition name for a historical write; unset for a normal write. + optional string original_partition_name = 4; } message PbProduceLogRespForBucket { @@ -894,6 +896,8 @@ message PbProduceLogRespForBucket { optional int32 error_code = 3; optional string error_message = 4; optional int64 base_offset = 5; + // The original partition name echoed from a historical write request; unset for a normal write. + optional string original_partition_name = 6; } message PbFetchLogReqForTable { diff --git a/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/client/ServerConnectionTest.java b/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/client/ServerConnectionTest.java index 4368872ec73..3b4e0596f3e 100644 --- a/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/client/ServerConnectionTest.java +++ b/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/client/ServerConnectionTest.java @@ -23,6 +23,7 @@ import org.apache.fluss.config.Configuration; import org.apache.fluss.exception.DisconnectException; import org.apache.fluss.exception.InvalidServerTypeException; +import org.apache.fluss.exception.UnsupportedVersionException; import org.apache.fluss.metrics.Gauge; import org.apache.fluss.metrics.Metric; import org.apache.fluss.metrics.MetricType; @@ -33,11 +34,18 @@ import org.apache.fluss.rpc.TestingGatewayService; import org.apache.fluss.rpc.TestingTabletGatewayService; import org.apache.fluss.rpc.messages.ApiMessage; +import org.apache.fluss.rpc.messages.ApiVersionsRequest; +import org.apache.fluss.rpc.messages.ApiVersionsResponse; import org.apache.fluss.rpc.messages.GetTableSchemaRequest; import org.apache.fluss.rpc.messages.ListDatabasesRequest; import org.apache.fluss.rpc.messages.LookupRequest; +import org.apache.fluss.rpc.messages.PbApiVersion; import org.apache.fluss.rpc.messages.PbLookupReqForBucket; import org.apache.fluss.rpc.messages.PbTablePath; +import org.apache.fluss.rpc.messages.ProduceLogRequest; +import org.apache.fluss.rpc.messages.ProduceLogResponse; +import org.apache.fluss.rpc.messages.PutKvRequest; +import org.apache.fluss.rpc.messages.PutKvResponse; import org.apache.fluss.rpc.metrics.ClientMetricGroup; import org.apache.fluss.rpc.metrics.TestingClientMetricGroup; import org.apache.fluss.rpc.netty.client.ServerConnection.ConnectionState; @@ -239,7 +247,75 @@ public ChannelFuture connect(String host, int port) { .isInstanceOf(DisconnectException.class); } + @Test + void testRejectHistoricalWritesForOldServer() throws Exception { + nettyServer.close(); + buildNettyServer(new OldWriteGatewayService()); + + ServerConnection connection = + new ServerConnection( + bootstrap, + serverNode, + TestingClientMetricGroup.newInstance(), + clientAuthenticator, + (con, ignore) -> {}); + try { + assertThat(connection.send(ApiKeys.PUT_KV, putKvRequest(null)).get()) + .isInstanceOf(PutKvResponse.class); + + assertThatThrownBy( + () -> + connection + .send(ApiKeys.PUT_KV, putKvRequest("dt=20260823")) + .get()) + .rootCause() + .isInstanceOf(UnsupportedVersionException.class) + .hasMessageContaining("require PUT_KV version 3 or newer") + .hasMessageContaining("negotiated version 2"); + + assertThat(connection.send(ApiKeys.PRODUCE_LOG, produceLogRequest(null)).get()) + .isInstanceOf(ProduceLogResponse.class); + + assertThatThrownBy( + () -> + connection + .send( + ApiKeys.PRODUCE_LOG, + produceLogRequest("dt=20260823")) + .get()) + .rootCause() + .isInstanceOf(UnsupportedVersionException.class) + .hasMessageContaining("require PRODUCE_LOG version 1 or newer") + .hasMessageContaining("negotiated version 0"); + } finally { + connection.close().get(); + } + } + + private static PutKvRequest putKvRequest(String originalPartitionName) { + PutKvRequest request = new PutKvRequest().setTableId(1L).setAcks(1).setTimeoutMs(10_000); + request.addBucketsReq().setBucketId(0).setRecords(new byte[0]); + if (originalPartitionName != null) { + request.getBucketsReqAt(0).setOriginalPartitionName(originalPartitionName); + } + return request; + } + + private static ProduceLogRequest produceLogRequest(String originalPartitionName) { + ProduceLogRequest request = + new ProduceLogRequest().setTableId(1L).setAcks(1).setTimeoutMs(10_000); + request.addBucketsReq().setBucketId(0).setRecords(new byte[0]); + if (originalPartitionName != null) { + request.getBucketsReqAt(0).setOriginalPartitionName(originalPartitionName); + } + return request; + } + private void buildNettyServer() throws Exception { + buildNettyServer(new TestingTabletGatewayService()); + } + + private void buildNettyServer(TestingGatewayService gatewayService) throws Exception { try (NetUtils.Port availablePort = getAvailablePort(); NetUtils.Port availablePort2 = getAvailablePort()) { serverNode = @@ -248,7 +324,7 @@ private void buildNettyServer() throws Exception { serverNode2 = new ServerNode( 2, "localhost", availablePort2.getPort(), ServerType.TABLET_SERVER); - service = new TestingTabletGatewayService(); + service = gatewayService; MetricGroup metricGroup = NOPMetricsGroup.newInstance(); nettyServer = new NettyServer( @@ -263,6 +339,34 @@ private void buildNettyServer() throws Exception { } } + private static class OldWriteGatewayService extends TestingTabletGatewayService { + @Override + public CompletableFuture apiVersions(ApiVersionsRequest request) { + return super.apiVersions(request) + .thenApply( + response -> { + for (PbApiVersion apiVersion : response.getApiVersionsList()) { + if (apiVersion.getApiKey() == ApiKeys.PUT_KV.id) { + apiVersion.setMaxVersion(2); + } else if (apiVersion.getApiKey() == ApiKeys.PRODUCE_LOG.id) { + apiVersion.setMaxVersion(0); + } + } + return response; + }); + } + + @Override + public CompletableFuture putKv(PutKvRequest request) { + return CompletableFuture.completedFuture(new PutKvResponse()); + } + + @Override + public CompletableFuture produceLog(ProduceLogRequest request) { + return CompletableFuture.completedFuture(new ProduceLogResponse()); + } + } + private static class MockMetricRegistry extends NOPMetricRegistry { Map registeredMetrics = new HashMap<>(); diff --git a/fluss-rust/crates/fluss/proto/FlussApi.proto b/fluss-rust/crates/fluss/proto/FlussApi.proto index 9d5e05fbdae..1ae6b1f126b 100644 --- a/fluss-rust/crates/fluss/proto/FlussApi.proto +++ b/fluss-rust/crates/fluss/proto/FlussApi.proto @@ -886,6 +886,8 @@ message PbProduceLogReqForBucket { optional int64 partition_id = 1; required int32 bucket_id = 2; required bytes records = 3; + // The original partition name for a historical write; unset for a normal write. + optional string original_partition_name = 4; } message PbProduceLogRespForBucket { @@ -894,6 +896,8 @@ message PbProduceLogRespForBucket { optional int32 error_code = 3; optional string error_message = 4; optional int64 base_offset = 5; + // The original partition name echoed from a historical write request; unset for a normal write. + optional string original_partition_name = 6; } message PbFetchLogReqForTable { diff --git a/fluss-rust/crates/fluss/src/proto/fluss.rs b/fluss-rust/crates/fluss/src/proto/fluss.rs index 95e7045c57e..e29dbaea874 100644 --- a/fluss-rust/crates/fluss/src/proto/fluss.rs +++ b/fluss-rust/crates/fluss/src/proto/fluss.rs @@ -1165,6 +1165,9 @@ pub struct PbProduceLogReqForBucket { pub bucket_id: i32, #[prost(bytes = "bytes", required, tag = "3")] pub records: ::prost::bytes::Bytes, + /// The original partition name for a historical write; unset for a normal write. + #[prost(string, optional, tag = "4")] + pub original_partition_name: ::core::option::Option<::prost::alloc::string::String>, } #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct PbProduceLogRespForBucket { @@ -1178,6 +1181,9 @@ pub struct PbProduceLogRespForBucket { pub error_message: ::core::option::Option<::prost::alloc::string::String>, #[prost(int64, optional, tag = "5")] pub base_offset: ::core::option::Option, + /// The original partition name echoed from a historical write request; unset for a normal write. + #[prost(string, optional, tag = "6")] + pub original_partition_name: ::core::option::Option<::prost::alloc::string::String>, } #[derive(Clone, PartialEq, ::prost::Message)] pub struct PbFetchLogReqForTable { diff --git a/fluss-rust/crates/fluss/src/rpc/api_key.rs b/fluss-rust/crates/fluss/src/rpc/api_key.rs index e9720a4e2b6..1341d1e6180 100644 --- a/fluss-rust/crates/fluss/src/rpc/api_key.rs +++ b/fluss-rust/crates/fluss/src/rpc/api_key.rs @@ -93,7 +93,6 @@ impl ApiKey { | ApiKey::TableExists | ApiKey::GetTableSchema | ApiKey::MetaData - | ApiKey::ProduceLog | ApiKey::FetchLog | ApiKey::ListOffsets | ApiKey::GetLatestKvSnapshots @@ -129,6 +128,8 @@ impl ApiKey { | ApiKey::GetClusterHealth | ApiKey::ListRemoteLogManifests | ApiKey::ListKvSnapshots => Some(ApiVersionRange::new(ApiVersion(0), ApiVersion(0))), + // ProduceLog v1 adds historical partition context to requests and responses. + ApiKey::ProduceLog => Some(ApiVersionRange::new(ApiVersion(0), ApiVersion(1))), // PutKv v2 adds the storage backpressure error code; v3 adds historical partition // context to requests and responses. ApiKey::PutKv => Some(ApiVersionRange::new(ApiVersion(0), ApiVersion(3))), diff --git a/fluss-rust/crates/fluss/src/rpc/message/produce_log.rs b/fluss-rust/crates/fluss/src/rpc/message/produce_log.rs index de9dce118d2..041e0adde01 100644 --- a/fluss-rust/crates/fluss/src/rpc/message/produce_log.rs +++ b/fluss-rust/crates/fluss/src/rpc/message/produce_log.rs @@ -49,6 +49,7 @@ impl ProduceLogRequest { partition_id: ready_batch.table_bucket.partition_id(), bucket_id: ready_batch.table_bucket.bucket_id(), records: ready_batch.write_batch.build()?, + original_partition_name: None, }) } diff --git a/fluss-rust/crates/fluss/src/rpc/server_connection.rs b/fluss-rust/crates/fluss/src/rpc/server_connection.rs index a8b36cbecc2..66825ab265a 100644 --- a/fluss-rust/crates/fluss/src/rpc/server_connection.rs +++ b/fluss-rust/crates/fluss/src/rpc/server_connection.rs @@ -1196,7 +1196,7 @@ mod tests { min_version: 0, max_version: 3, }, - // ProduceLog: server v0..v2, client v0 only → negotiated v0 + // ProduceLog: server v0..v2, client v0..v1 → negotiated v1 PbApiVersion { api_key: 1014, min_version: 0, @@ -1238,7 +1238,7 @@ mod tests { negotiated .highest_available_version(ApiKey::ProduceLog) .unwrap(), - ApiVersion(0) + ApiVersion(1) ); // Disjoint range → error diff --git a/fluss-server/src/main/java/org/apache/fluss/server/entity/ProduceLogDataForBucket.java b/fluss-server/src/main/java/org/apache/fluss/server/entity/ProduceLogDataForBucket.java new file mode 100644 index 00000000000..96232246152 --- /dev/null +++ b/fluss-server/src/main/java/org/apache/fluss/server/entity/ProduceLogDataForBucket.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.entity; + +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.record.MemoryLogRecords; + +import javax.annotation.Nullable; + +/** Log records for one normal or historical partition bucket. */ +public class ProduceLogDataForBucket { + private final TableBucket tableBucket; + private final MemoryLogRecords records; + // Identifies the original partition for a historical write; null for a normal write. + private final @Nullable String originalPartitionName; + + public ProduceLogDataForBucket( + TableBucket tableBucket, + MemoryLogRecords records, + @Nullable String originalPartitionName) { + this.tableBucket = tableBucket; + this.records = records; + this.originalPartitionName = originalPartitionName; + } + + public TableBucket tableBucket() { + return tableBucket; + } + + public MemoryLogRecords records() { + return records; + } + + public @Nullable String originalPartitionName() { + return originalPartitionName; + } +} diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvRecoverHelper.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvRecoverHelper.java index a3afeec4bbf..0deed60a418 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvRecoverHelper.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvRecoverHelper.java @@ -39,6 +39,7 @@ import org.apache.fluss.row.indexed.IndexedRow; import org.apache.fluss.server.kv.autoinc.AutoIncIDRange; import org.apache.fluss.server.kv.historical.HistoricalKvKeyEncoder; +import org.apache.fluss.server.kv.historical.HistoricalKvTombstone; import org.apache.fluss.server.log.FetchIsolation; import org.apache.fluss.server.log.LogTablet; import org.apache.fluss.server.zk.ZooKeeperClient; @@ -56,7 +57,6 @@ import java.util.List; import static org.apache.fluss.server.TabletManagerBase.getTableInfo; -import static org.apache.fluss.server.kv.KvStateAccessor.HISTORICAL_TOMBSTONE; import static org.apache.fluss.utils.Preconditions.checkArgument; import static org.apache.fluss.utils.Preconditions.checkNotNull; @@ -149,7 +149,9 @@ public void recover() throws Exception { (resumeRecord) -> { if (resumeRecord.value == null) { if (historicalPartition) { - kvBatchWriter.put(resumeRecord.key, HISTORICAL_TOMBSTONE); + kvBatchWriter.put( + resumeRecord.key, + HistoricalKvTombstone.encode(resumeRecord.logOffset)); } else { kvBatchWriter.delete(resumeRecord.key); } @@ -296,9 +298,13 @@ private long applyLogRecordBatch( // the log row format may not compatible with kv row format, // e.g, arrow vs. compacted, thus needs a conversion here. BinaryRow row = toKvRow(logRow); + BinaryValue binaryValue = + new BinaryValue(currentSchemaId.shortValue(), row); value = - valueEncoder.encodeValue( - new BinaryValue(currentSchemaId.shortValue(), row)); + historicalPartition + ? valueEncoder.encodeValue( + binaryValue, logRecord.logOffset()) + : valueEncoder.encodeValue(binaryValue); } resumeRecordConsumer.accept( new KeyValueAndLogOffset( diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvStateAccessor.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvStateAccessor.java index 1e0cd39962e..daba99a6c84 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvStateAccessor.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvStateAccessor.java @@ -19,6 +19,7 @@ import org.apache.fluss.annotation.Internal; import org.apache.fluss.server.kv.historical.HistoricalKvKeyEncoder; +import org.apache.fluss.server.kv.historical.HistoricalKvTombstone; import org.apache.fluss.server.kv.prewrite.KvPreWriteBuffer; import org.apache.fluss.server.kv.prewrite.KvPreWriteBuffer.Key; import org.apache.fluss.server.kv.prewrite.KvPreWriteBuffer.TruncateReason; @@ -35,9 +36,6 @@ @Internal public final class KvStateAccessor { - /** Encoded RocksDB value marking a deleted key in historical KV state. */ - static final byte[] HISTORICAL_TOMBSTONE = new byte[0]; - private final KvPreWriteBuffer preWriteBuffer; private final RocksDBKv rocksDBKv; private final boolean historicalPartition; @@ -85,9 +83,9 @@ public KvStateLookupResult lookup(Key key) throws IOException { if (value == null) { return KvStateLookupResult.notFound(); } - // Historical KV tablets persist deletes as empty values so that a local miss does not - // expose a stale value from lake storage after the buffered delete has been flushed. - return value.length == 0 + // Historical KV tablets persist deletes as offset-tagged tombstones so that a local miss + // does not expose a stale value from lake storage after the buffered delete is flushed. + return historicalPartition && HistoricalKvTombstone.isTombstone(value) ? KvStateLookupResult.deleted() : KvStateLookupResult.present(value); } @@ -118,4 +116,8 @@ public void delete(Key key, long logOffset) { public void truncateTo(long logOffset, TruncateReason reason) { preWriteBuffer.truncateTo(logOffset, reason); } + + boolean isHistoricalPartition() { + return historicalPartition; + } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java index 6118bb40fb3..760b8cd19c9 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java @@ -41,7 +41,9 @@ import org.apache.fluss.rpc.protocol.MergeMode; import org.apache.fluss.server.kv.autoinc.AutoIncIDRange; import org.apache.fluss.server.kv.autoinc.AutoIncrementManager; +import org.apache.fluss.server.kv.historical.HistoricalKvTombstone; import org.apache.fluss.server.kv.historical.HistoricalValueLookup; +import org.apache.fluss.server.kv.historical.LocalPreviousValueLookupResult; import org.apache.fluss.server.kv.prewrite.KvPreWriteBuffer; import org.apache.fluss.server.kv.prewrite.KvPreWriteBuffer.PreparedFlush; import org.apache.fluss.server.kv.rocksdb.RocksDBKv; @@ -90,11 +92,12 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; -import static org.apache.fluss.server.kv.KvStateAccessor.HISTORICAL_TOMBSTONE; import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; +import static org.apache.fluss.utils.Preconditions.checkArgument; import static org.apache.fluss.utils.Preconditions.checkNotNull; import static org.apache.fluss.utils.Preconditions.checkState; import static org.apache.fluss.utils.concurrent.LockUtils.inReadLock; @@ -147,6 +150,7 @@ public final class KvTablet { private final ReadWriteLock kvLock = new ReentrantReadWriteLock(); private final KvValueLayout kvValueLayout; private final ValueEncoder valueEncoder; + private final AtomicLong historicalCleanupOffset; @Nullable private final RowTtlTimestampProvider rowTtlTimestampProvider; private final boolean rowTtlEnabled; private final AutoIncrementManager autoIncrementManager; @@ -197,6 +201,7 @@ private KvTablet( KvValueLayout kvValueLayout, ValueEncoder valueEncoder, ValueDecoder valueDecoder, + AtomicLong historicalCleanupOffset, @Nullable RocksDBStatistics rocksDBStatistics, KvFlushScheduler kvFlushScheduler, boolean closeFlushScheduler, @@ -221,6 +226,7 @@ private KvTablet( new KvStateAccessor(kvPreWriteBuffer, rocksDBKv, historicalPartition); this.kvValueLayout = kvValueLayout; this.valueEncoder = valueEncoder; + this.historicalCleanupOffset = historicalCleanupOffset; this.rowTtlTimestampProvider = rowTtlTimestampProvider; this.rowTtlEnabled = rowTtlEnabled; this.kvWriteProcessor = @@ -367,11 +373,16 @@ private static KvTablet create( TableConfig tableConfig) throws IOException { checkNotNull(tableConfig, "tableConfig must not be null."); + boolean historicalPartition = + HISTORICAL_PARTITION_VALUE.equals(tablePath.getPartitionName()); Optional rowTtl = tableConfig.getKvTTL(); - KvValueLayout kvValueLayout = KvValueLayout.fromTableConfig(tableConfig); + KvValueLayout kvValueLayout = + historicalPartition + ? KvValueLayout.TAGGED + : KvValueLayout.fromTableConfig(tableConfig); @Nullable RowTtlTimestampProvider rowTtlTimestampProvider = - kvValueLayout.hasValueTag() + !historicalPartition && kvValueLayout.hasValueTag() ? RowTtlTimestampProvider.create( tableConfig, schemaGetter, ZoneId.systemDefault()) : null; @@ -380,13 +391,17 @@ private static KvTablet create( ? ValueEncoder.forLayout(kvValueLayout) : ValueEncoder.forLayout(kvValueLayout, rowTtlTimestampProvider); ValueDecoder valueDecoder = new ValueDecoder(schemaGetter, kvFormat, kvValueLayout); + AtomicLong historicalCleanupOffset = new AtomicLong(0L); @Nullable AbstractCompactionFilterFactory> compactionFilterFactory = - rowTtl.isPresent() + historicalPartition ? RowTtlCompactionFilterFactory.create( - kvValueLayout, rowTtl.get(), clock) - : null; + kvValueLayout, 0L, () -> historicalCleanupOffset.get() - 1L) + : rowTtl.isPresent() + ? RowTtlCompactionFilterFactory.create( + kvValueLayout, rowTtl.get(), clock) + : null; RocksDBKv kv = buildRocksDBKv(serverConf, kvTabletDir, sharedRateLimiter, compactionFilterFactory); @@ -420,6 +435,7 @@ private static KvTablet create( kvValueLayout, valueEncoder, valueDecoder, + historicalCleanupOffset, rocksDBStatistics, kvFlushScheduler, closeFlushScheduler, @@ -655,16 +671,15 @@ public LogAppendInfo putAsLeader( * Puts records for one original partition into this historical KV tablet. * *

The original partition name namespaces the physical primary keys because one historical - * bucket can contain records from multiple original partitions. The supplied fallback may only - * read lake results already resolved for this request; it must not perform lake I/O while the - * tablet lock is held. + * bucket can contain records from multiple original partitions. The supplied lookup must + * contain every previous value required by this batch and must not perform I/O. */ public LogAppendInfo putHistoricalAsLeader( KvRecordBatch kvRecords, @Nullable int[] targetColumns, MergeMode mergeMode, String originalPartitionName, - HistoricalValueLookup memoizedLakeLookup) + HistoricalValueLookup historicalValueLookup) throws Exception { checkState(historicalPartition, "%s is not a historical KV tablet", tableBucket); return putAsLeader( @@ -672,16 +687,16 @@ public LogAppendInfo putHistoricalAsLeader( targetColumns, mergeMode, checkNotNull(originalPartitionName, "originalPartitionName must not be null"), - checkNotNull(memoizedLakeLookup, "memoizedLakeLookup must not be null")); + checkNotNull(historicalValueLookup, "Historical value lookup must not be null")); } /** - * Finds keys whose historical write requires an old value that is absent from local state. + * Probes the local previous values required by a historical write. * *

This method only reads KV entries and uses the tablet read lock. Lake I/O must be * performed by the caller after this method releases the tablet lock. */ - public List findKeysRequiringLakeLookup( + public LocalPreviousValueLookupResult probeLocalPreviousValues( KvRecordBatch kvRecords, @Nullable int[] targetColumns, MergeMode mergeMode, @@ -692,7 +707,7 @@ public List findKeysRequiringLakeLookup( kvLock, () -> { rocksDBKv.checkIfRocksDBClosed(); - return kvWriteProcessor.findKeysRequiringLakeLookup( + return kvWriteProcessor.probeLocalPreviousValues( kvRecords, targetColumns, mergeMode, @@ -708,7 +723,7 @@ private LogAppendInfo putAsLeader( @Nullable int[] targetColumns, MergeMode mergeMode, @Nullable String originalPartitionName, - @Nullable HistoricalValueLookup memoizedLakeLookup) + @Nullable HistoricalValueLookup historicalValueLookup) throws Exception { return inWriteLock( kvLock, @@ -737,7 +752,7 @@ private LogAppendInfo putAsLeader( mergeMode, kvStateAccessor, originalPartitionName, - memoizedLakeLookup); + historicalValueLookup); }); } @@ -761,6 +776,21 @@ public long getFlushedLogOffset() { return flushedLogOffset; } + /** Advances the exclusive historical cleanup offset without allowing it to move backwards. */ + public boolean advanceHistoricalCleanupOffset(long cleanupOffset) { + checkState(historicalPartition, "%s is not a historical KV tablet", tableBucket); + checkArgument(cleanupOffset >= 0L, "Historical cleanup offset must be non-negative."); + long previousCleanupOffset = + historicalCleanupOffset.getAndAccumulate(cleanupOffset, Math::max); + return cleanupOffset > previousCleanupOffset; + } + + /** Returns the current exclusive cleanup offset for a historical overlay. */ + public long getHistoricalCleanupOffset() { + checkState(historicalPartition, "%s is not a historical KV tablet", tableBucket); + return historicalCleanupOffset.get(); + } + @VisibleForTesting FlushState getFlushState() { return inReadLock(kvLock, () -> flushState); @@ -906,7 +936,9 @@ private void writePreparedFlush(PreparedFlush preparedFlush) throws Exception { if (historicalPartition) { // A physical delete would turn a local miss into a lake lookup and // could expose the stale value that this mutation deleted. - kvBatchWriter.put(entry.getKey().get(), HISTORICAL_TOMBSTONE); + kvBatchWriter.put( + entry.getKey().get(), + HistoricalKvTombstone.encode(entry.getLogSequenceNumber())); } else { kvBatchWriter.delete(entry.getKey().get()); } @@ -1119,7 +1151,7 @@ public KvStateLookupResult lookupHistoricalLocal(String originalPartitionName, b if (value == null) { return KvStateLookupResult.notFound(); } - return value.length == 0 + return HistoricalKvTombstone.isTombstone(value) ? KvStateLookupResult.deleted() : KvStateLookupResult.present(value); }); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvWriteProcessor.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvWriteProcessor.java index bc0fb53fff0..9dc46ac53a0 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvWriteProcessor.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvWriteProcessor.java @@ -45,6 +45,7 @@ import org.apache.fluss.server.kv.autoinc.AutoIncrementManager; import org.apache.fluss.server.kv.autoinc.AutoIncrementUpdater; import org.apache.fluss.server.kv.historical.HistoricalValueLookup; +import org.apache.fluss.server.kv.historical.LocalPreviousValueLookupResult; import org.apache.fluss.server.kv.prewrite.KvPreWriteBuffer; import org.apache.fluss.server.kv.prewrite.KvPreWriteBuffer.TruncateReason; import org.apache.fluss.server.kv.rowmerger.DefaultRowMerger; @@ -66,11 +67,12 @@ import javax.annotation.Nullable; import javax.annotation.concurrent.NotThreadSafe; -import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; -import java.util.List; import java.util.Set; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + /** * Processes a KV record batch into local state mutations and the corresponding WAL records. * @@ -81,10 +83,9 @@ * *

The supplied {@link KvStateAccessor} defines how keys and state are accessed. Normal writes * use the original primary key and local state, while historical writes use partition-scoped keys. - * On a historical local miss, the processor can consult a lake result already memoized for the - * current request. Resolving that result from lake storage remains the caller's responsibility and - * must happen outside the tablet lock. The merge and WAL generation path is shared by both write - * kinds. + * Historical writes use previous values resolved before the tablet write lock is acquired. + * Resolving local misses from lake storage remains the caller's responsibility and must happen + * outside the tablet lock. The merge and WAL generation path is shared by both write kinds. */ @Internal @NotThreadSafe @@ -158,7 +159,7 @@ public LogAppendInfo putAsLeader( MergeMode mergeMode, KvStateAccessor stateAccessor, @Nullable String originalPartitionName, - @Nullable HistoricalValueLookup memoizedLakeLookup) + @Nullable HistoricalValueLookup historicalValueLookup) throws Exception { WriteContext writeContext = createWriteContext(kvRecords, targetColumns, mergeMode); RowType latestRowType = writeContext.latestSchema.getRowType(); @@ -185,7 +186,7 @@ public LogAppendInfo putAsLeader( logEndOffsetOfPrevBatch, stateAccessor, originalPartitionName, - memoizedLakeLookup); + historicalValueLookup); // There will be a situation that these batches of kvRecordBatch have not // generated any CDC logs, for example, when client attempts to delete @@ -221,14 +222,13 @@ public LogAppendInfo putAsLeader( } /** - * Finds the original primary keys whose previous values must be loaded from lake storage before - * applying a historical write batch. + * Probes the local previous values required by a historical write batch. * - *

A key is returned only when its previous value is required and its partition-scoped key is - * absent from local state. Records that can establish their result without a previous value are - * skipped, and each key is returned at most once. + *

Every key whose previous value is required is probed at most once. Local values and + * deletes are decoded into the returned collection, while true local misses are exposed for + * lake lookup. Records that establish their result without a previous value are skipped. */ - List findKeysRequiringLakeLookup( + LocalPreviousValueLookupResult probeLocalPreviousValues( KvRecordBatch kvRecords, @Nullable int[] targetColumns, MergeMode mergeMode, @@ -237,9 +237,9 @@ List findKeysRequiringLakeLookup( throws Exception { WriteContext writeContext = createWriteContext(kvRecords, targetColumns, mergeMode); - List keysRequiringLakeLookup = new ArrayList<>(); - // Track keys whose lake-lookup requirement has already been evaluated. - Set keysEvaluatedForLakeLookup = new HashSet<>(); + LocalPreviousValueLookupResult localLookupResult = + new LocalPreviousValueLookupResult(valueDecoder, lakeValueDecoder); + Set probedKeys = new HashSet<>(); KvRecordBatch.ReadContext readContext = KvRecordReadContext.createReadContext(kvFormat, schemaGetter); for (KvRecord kvRecord : kvRecords.records(readContext)) { @@ -252,23 +252,20 @@ List findKeysRequiringLakeLookup( } else if (canSkipOldValueLookup( writeContext.rowMerger, writeContext.autoIncrementUpdater)) { // A full-row WAL upsert establishes the state without reading its previous value. - keysEvaluatedForLakeLookup.add(wrappedKey); + probedKeys.add(wrappedKey); continue; } // Probe local state only for the first record that needs a previous value. A true local // miss schedules one lake lookup shared by all records for this key in the batch. - if (keysEvaluatedForLakeLookup.add(wrappedKey) - && stateAccessor - .lookup( - stateAccessor.encodeKey( - primaryKey, originalPartitionName)) - .status() - == KvStateLookupResult.Status.NOT_FOUND) { - keysRequiringLakeLookup.add(primaryKey); + if (probedKeys.add(wrappedKey)) { + localLookupResult.add( + primaryKey, + stateAccessor.lookup( + stateAccessor.encodeKey(primaryKey, originalPartitionName))); } } - return keysRequiringLakeLookup; + return localLookupResult; } private WriteContext createWriteContext( @@ -313,9 +310,13 @@ private void processKvRecords( long startLogOffset, KvStateAccessor stateAccessor, @Nullable String originalPartitionName, - @Nullable HistoricalValueLookup memoizedLakeLookup) + @Nullable HistoricalValueLookup historicalValueLookup) throws Exception { long logOffset = startLogOffset; + // Once this batch mutates a historical key, later same-key records must read the staged + // value from local state instead of reusing the value memoized before apply. + Set stagedHistoricalKeys = + historicalValueLookup == null ? Collections.emptySet() : new HashSet<>(); // TODO: reuse the read context KvRecordBatch.ReadContext readContext = @@ -326,6 +327,11 @@ private void processKvRecords( KvPreWriteBuffer.Key key = stateAccessor.encodeKey(keyBytes, originalPartitionName); BinaryRow row = kvRecord.getRow(); BinaryValue currentValue = row == null ? null : new BinaryValue(schemaIdOfNewData, row); + long previousLogOffset = logOffset; + ByteArrayWrapper historicalKey = + historicalValueLookup == null ? null : new ByteArrayWrapper(keyBytes); + boolean useMemoizedPreviousValue = + historicalKey != null && !stagedHistoricalKeys.contains(historicalKey); if (currentValue == null) { logOffset = @@ -337,7 +343,8 @@ private void processKvRecords( logOffset, stateAccessor, keyBytes, - memoizedLakeLookup); + historicalValueLookup, + useMemoizedPreviousValue); } else { logOffset = processUpsert( @@ -350,7 +357,11 @@ private void processKvRecords( logOffset, stateAccessor, keyBytes, - memoizedLakeLookup); + historicalValueLookup, + useMemoizedPreviousValue); + } + if (historicalKey != null && logOffset > previousLogOffset) { + stagedHistoricalKeys.add(historicalKey); } } } @@ -363,13 +374,20 @@ private long processDeletion( long logOffset, KvStateAccessor stateAccessor, byte[] primaryKey, - @Nullable HistoricalValueLookup memoizedLakeLookup) + @Nullable HistoricalValueLookup historicalValueLookup, + boolean useMemoizedPreviousValue) throws Exception { if (shouldIgnoreDeletion(currentMerger)) { return logOffset; } - BinaryValue oldValue = getPreviousValue(key, primaryKey, stateAccessor, memoizedLakeLookup); + BinaryValue oldValue = + getPreviousValue( + key, + primaryKey, + stateAccessor, + historicalValueLookup, + useMemoizedPreviousValue); if (oldValue == null) { LOG.debug( "The specific key can't be found in kv tablet although the kv record is for deletion, " @@ -399,14 +417,21 @@ private long processUpsert( long logOffset, KvStateAccessor stateAccessor, byte[] primaryKey, - @Nullable HistoricalValueLookup memoizedLakeLookup) + @Nullable HistoricalValueLookup historicalValueLookup, + boolean useMemoizedPreviousValue) throws Exception { if (canSkipOldValueLookup(currentMerger, autoIncrementUpdater)) { return applyUpdate( key, null, currentValue, walBuilder, latestSchemaRow, logOffset, stateAccessor); } - BinaryValue oldValue = getPreviousValue(key, primaryKey, stateAccessor, memoizedLakeLookup); + BinaryValue oldValue = + getPreviousValue( + key, + primaryKey, + stateAccessor, + historicalValueLookup, + useMemoizedPreviousValue); if (oldValue == null) { BinaryValue valueToInsert = currentMerger.merge(null, currentValue); return applyInsert( @@ -454,7 +479,7 @@ private long applyInsert( throws Exception { BinaryValue newValue = autoIncrementUpdater.updateAutoIncrementColumns(currentValue); walBuilder.append(ChangeType.INSERT, latestSchemaRow.replaceRow(newValue.row)); - stateAccessor.insert(key, valueEncoder.encodeValue(newValue), logOffset); + stateAccessor.insert(key, encodeStateValue(newValue, logOffset, stateAccessor), logOffset); return logOffset + 1; } @@ -469,42 +494,52 @@ private long applyUpdate( throws Exception { if (changelogImage == ChangelogImage.WAL) { walBuilder.append(ChangeType.UPDATE_AFTER, latestSchemaRow.replaceRow(newValue.row)); - stateAccessor.update(key, valueEncoder.encodeValue(newValue), logOffset); + stateAccessor.update( + key, encodeStateValue(newValue, logOffset, stateAccessor), logOffset); return logOffset + 1; } else { walBuilder.append(ChangeType.UPDATE_BEFORE, latestSchemaRow.replaceRow(oldValue.row)); walBuilder.append(ChangeType.UPDATE_AFTER, latestSchemaRow.replaceRow(newValue.row)); - stateAccessor.update(key, valueEncoder.encodeValue(newValue), logOffset + 1); + long updateAfterOffset = logOffset + 1; + stateAccessor.update( + key, + encodeStateValue(newValue, updateAfterOffset, stateAccessor), + updateAfterOffset); return logOffset + 2; } } + private byte[] encodeStateValue( + BinaryValue value, long logOffset, KvStateAccessor stateAccessor) { + return stateAccessor.isHistoricalPartition() + ? valueEncoder.encodeValue(value, logOffset) + : valueEncoder.encodeValue(value); + } + /** - * Returns the previous value from local state, falling back to the memoized lake result only on - * a genuine local miss. + * Returns the previous value from request-scoped historical state or the current local state. * * @param localStateKey the key used by the local prewrite buffer and RocksDB; it wraps {@code * primaryKey} for a normal partition and adds the original partition namespace for a * historical partition * @param primaryKey the encoded bytes of the logical primary key, without the historical - * partition namespace; used by the lake lookup + * partition namespace * @return the previous value, or null if the key is absent or locally marked as deleted */ private BinaryValue getPreviousValue( KvPreWriteBuffer.Key localStateKey, byte[] primaryKey, KvStateAccessor stateAccessor, - @Nullable HistoricalValueLookup memoizedLakeLookup) + @Nullable HistoricalValueLookup historicalValueLookup, + boolean useMemoizedPreviousValue) throws Exception { - KvStateLookupResult localResult = stateAccessor.lookup(localStateKey); - if (localResult.status() != KvStateLookupResult.Status.NOT_FOUND - || memoizedLakeLookup == null) { - return localResult.value() == null - ? null - : valueDecoder.decodeValue(localResult.value()); + if (useMemoizedPreviousValue) { + return checkNotNull(historicalValueLookup, "Historical value lookup must not be null") + .lookup(primaryKey); } - byte[] lakeValue = memoizedLakeLookup.lookup(primaryKey); - return lakeValue == null ? null : lakeValueDecoder.decodeValue(lakeValue); + + KvStateLookupResult localResult = stateAccessor.lookup(localStateKey); + return localResult.value() == null ? null : valueDecoder.decodeValue(localResult.value()); } private boolean canSkipOldValueLookup( diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/RowTtlCompactionFilterFactory.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/RowTtlCompactionFilterFactory.java index 8cb01d0c1f2..e77e5b9b468 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/kv/RowTtlCompactionFilterFactory.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/RowTtlCompactionFilterFactory.java @@ -17,7 +17,6 @@ package org.apache.fluss.server.kv; -import org.apache.fluss.annotation.VisibleForTesting; import org.apache.fluss.row.encode.KvValueLayout; import org.apache.fluss.server.utils.RowTtlUtils; import org.apache.fluss.utils.clock.Clock; @@ -26,6 +25,7 @@ import org.rocksdb.RocksDB; import java.time.Duration; +import java.util.function.LongSupplier; import static org.apache.fluss.utils.Preconditions.checkArgument; import static org.apache.fluss.utils.Preconditions.checkNotNull; @@ -40,29 +40,47 @@ private RowTtlCompactionFilterFactory() {} /** Creates a configured native compaction filter factory for row TTL cleanup. */ public static FlinkCompactionFilter.FlinkCompactionFilterFactory create( KvValueLayout kvValueLayout, Duration ttl, Clock clock) { - return create(kvValueLayout, ttl, QUERY_TIME_AFTER_NUM_ENTRIES, clock); + long ttlMillis = RowTtlUtils.validateAndConvertTtlDurationToMillis(ttl); + checkNotNull(clock, "clock must not be null."); + return create(kvValueLayout, ttlMillis, clock::milliseconds); } - @VisibleForTesting + /** Removes values using the default interval for refreshing the supplied current value. */ static FlinkCompactionFilter.FlinkCompactionFilterFactory create( - KvValueLayout kvValueLayout, Duration ttl, long queryTimeAfterNumEntries, Clock clock) { - long ttlMillis = RowTtlUtils.validateAndConvertTtlDurationToMillis(ttl); + KvValueLayout kvValueLayout, + long expirationDistance, + LongSupplier currentValueSupplier) { + return create( + kvValueLayout, + expirationDistance, + QUERY_TIME_AFTER_NUM_ENTRIES, + currentValueSupplier); + } + + /** Removes a value when {@code valueTag + expirationDistance <= currentValue}. */ + static FlinkCompactionFilter.FlinkCompactionFilterFactory create( + KvValueLayout kvValueLayout, + long expirationDistance, + long queryCurrentValueAfterNumEntries, + LongSupplier currentValueSupplier) { checkNotNull(kvValueLayout, "kvValueLayout must not be null."); - checkNotNull(clock, "clock must not be null."); - checkArgument(kvValueLayout.hasValueTag(), "Row TTL requires a tagged KV value layout."); + checkNotNull(currentValueSupplier, "currentValueSupplier must not be null."); + checkArgument(kvValueLayout.hasValueTag(), "Compaction filter requires a tagged layout."); + checkArgument(expirationDistance >= 0L, "Expiration distance must be non-negative."); checkArgument( - queryTimeAfterNumEntries > 0, - "queryTimeAfterNumEntries must be greater than zero."); + queryCurrentValueAfterNumEntries > 0L, + "queryCurrentValueAfterNumEntries must be greater than zero."); RocksDB.loadLibrary(); FlinkCompactionFilter.FlinkCompactionFilterFactory factory = - new FlinkCompactionFilter.FlinkCompactionFilterFactory(clock::milliseconds); + new FlinkCompactionFilter.FlinkCompactionFilterFactory( + currentValueSupplier::getAsLong); factory.configure( FlinkCompactionFilter.Config.createNotList( FlinkCompactionFilter.StateType.Value, kvValueLayout.valueTagOffset(), - ttlMillis, - queryTimeAfterNumEntries)); + expirationDistance, + queryCurrentValueAfterNumEntries)); return factory; } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/historical/HistoricalKvTombstone.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/historical/HistoricalKvTombstone.java new file mode 100644 index 00000000000..6191978d67b --- /dev/null +++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/historical/HistoricalKvTombstone.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.kv.historical; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.row.encode.KvValueLayout; + +import static org.apache.fluss.utils.Preconditions.checkArgument; + +/** Utilities for tombstones stored in a historical KV overlay. */ +@Internal +public final class HistoricalKvTombstone { + + private HistoricalKvTombstone() {} + + /** Encodes a tombstone tagged with the WAL offset that produced the delete. */ + public static byte[] encode(long logOffset) { + checkArgument(logOffset >= 0L, "Historical KV log offset must be non-negative."); + byte[] tombstone = new byte[KvValueLayout.TAGGED.valueTagLength()]; + KvValueLayout.TAGGED.writeValueTag(tombstone, logOffset); + return tombstone; + } + + /** Returns whether the raw historical value is an offset-tagged tombstone. */ + public static boolean isTombstone(byte[] rawValue) { + return rawValue.length == KvValueLayout.TAGGED.valueTagLength(); + } +} diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/historical/HistoricalValueLookup.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/historical/HistoricalValueLookup.java index bdc723df96e..e632fd2a213 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/kv/historical/HistoricalValueLookup.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/historical/HistoricalValueLookup.java @@ -2,7 +2,7 @@ * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 + * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * @@ -18,20 +18,21 @@ package org.apache.fluss.server.kv.historical; import org.apache.fluss.annotation.Internal; +import org.apache.fluss.record.BinaryValue; import javax.annotation.Nullable; -/** Resolves a lake value already memoized for the current historical write request. */ +/** Looks up a previous value already memoized for the current historical write request. */ @Internal @FunctionalInterface public interface HistoricalValueLookup { /** - * Returns the encoded value for the primary key, or null when it does not exist. + * Returns the decoded previous value, or null when the key was absent or deleted. * - *

This method is invoked while the KV write lock is held and must not perform lake or file + *

This method is invoked while the KV write lock is held and must not perform local or lake * I/O. */ @Nullable - byte[] lookup(byte[] primaryKey); + BinaryValue lookup(byte[] primaryKey); } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/historical/LocalPreviousValueLookupResult.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/historical/LocalPreviousValueLookupResult.java new file mode 100644 index 00000000000..96b6eabc652 --- /dev/null +++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/historical/LocalPreviousValueLookupResult.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.kv.historical; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.record.BinaryValue; +import org.apache.fluss.row.encode.ValueDecoder; +import org.apache.fluss.server.kv.KvStateLookupResult; +import org.apache.fluss.utils.ByteArrayWrapper; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.NotThreadSafe; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; +import static org.apache.fluss.utils.Preconditions.checkState; + +/** Previous-value lookup results captured from local state for one historical write request. */ +@Internal +@NotThreadSafe +public final class LocalPreviousValueLookupResult { + + /** Decodes tagged values returned by local historical KV state. */ + private final ValueDecoder localValueDecoder; + + /** Decodes plain values returned by lake storage. */ + private final ValueDecoder lakeValueDecoder; + + /** Previous values found locally, including tombstones represented by an empty optional. */ + private final Map> localValuesByKey = new HashMap<>(); + + /** True local misses in lake request order. */ + private final Set keysMissingLocally = new LinkedHashSet<>(); + + /** Creates a local lookup result with decoders for the local and lake value layouts. */ + public LocalPreviousValueLookupResult( + ValueDecoder localValueDecoder, ValueDecoder lakeValueDecoder) { + this.localValueDecoder = localValueDecoder; + this.lakeValueDecoder = lakeValueDecoder; + } + + /** Records the local result for one key whose previous value is required. */ + public void add(byte[] primaryKey, KvStateLookupResult localResult) { + ByteArrayWrapper wrappedKey = new ByteArrayWrapper(primaryKey); + checkState( + !localValuesByKey.containsKey(wrappedKey) + && !keysMissingLocally.contains(wrappedKey), + "Historical write key has already been probed"); + + if (localResult.status() == KvStateLookupResult.Status.NOT_FOUND) { + keysMissingLocally.add(wrappedKey); + } else { + localValuesByKey.put(wrappedKey, decode(localResult.value(), localValueDecoder)); + } + } + + /** Returns whether at least one previous value was not found in local state. */ + public boolean hasLocalMisses() { + return !keysMissingLocally.isEmpty(); + } + + /** Returns a snapshot of true local misses in lake request order. */ + public List keysMissingLocally() { + List primaryKeys = new ArrayList<>(keysMissingLocally.size()); + for (ByteArrayWrapper keyMissingLocally : keysMissingLocally) { + primaryKeys.add(keyMissingLocally.getData()); + } + return Collections.unmodifiableList(primaryKeys); + } + + /** + * Combines lake results with the local lookup results and creates an in-memory lookup for + * apply. + * + *

Lake values must have the same order as {@link #keysMissingLocally()}. + */ + public HistoricalValueLookup createValueLookup(List lakeValues) { + checkNotNull(lakeValues, "Historical lake values must not be null"); + checkArgument( + lakeValues.size() == keysMissingLocally.size(), + "Expected %s historical lake values, but received %s", + keysMissingLocally.size(), + lakeValues.size()); + Map> previousValuesByKey = + new HashMap<>(localValuesByKey); + Iterator missingKeyIterator = keysMissingLocally.iterator(); + for (byte[] lakeValue : lakeValues) { + previousValuesByKey.put(missingKeyIterator.next(), decode(lakeValue, lakeValueDecoder)); + } + return primaryKey -> + checkNotNull( + previousValuesByKey.get(new ByteArrayWrapper(primaryKey)), + "No previous value for a historical write key") + .orElse(null); + } + + private static Optional decode( + @Nullable byte[] encodedValue, ValueDecoder valueDecoder) { + return encodedValue == null + ? Optional.empty() + : Optional.of(valueDecoder.decodeValue(encodedValue)); + } +} diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/prewrite/KvPreWriteBuffer.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/prewrite/KvPreWriteBuffer.java index 98320697685..f248b8271b9 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/kv/prewrite/KvPreWriteBuffer.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/prewrite/KvPreWriteBuffer.java @@ -391,7 +391,8 @@ public Value getValue() { return value; } - long getLogSequenceNumber() { + /** Returns the WAL offset that produced this mutation. */ + public long getLogSequenceNumber() { return logSequenceNumber; } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java index ee28567756f..516cf5b0747 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java @@ -66,6 +66,7 @@ import org.apache.fluss.server.kv.RemoteLogFetcher; import org.apache.fluss.server.kv.autoinc.AutoIncIDRange; import org.apache.fluss.server.kv.historical.HistoricalValueLookup; +import org.apache.fluss.server.kv.historical.LocalPreviousValueLookupResult; import org.apache.fluss.server.kv.rocksdb.RocksDBKvBuilder; import org.apache.fluss.server.kv.scan.OpenScanResult; import org.apache.fluss.server.kv.scan.ScannerContext; @@ -114,6 +115,7 @@ import org.apache.fluss.utils.FlussPaths; import org.apache.fluss.utils.IOUtils; import org.apache.fluss.utils.clock.Clock; +import org.apache.fluss.utils.function.FunctionWithException; import org.apache.fluss.utils.types.Tuple2; import org.slf4j.Logger; @@ -327,8 +329,8 @@ public long logicalStorageLogSize() { public long logicalStorageKvSize() { if (isLeader() && isKvTable()) { if (isHistoricalPartition()) { - // Historical KV tablets do not create snapshots, so account for the local overlay - // using live SST files instead. + // Historical KV tablets do not create snapshots, so use live SST files to account + // for their local state instead. KvTablet currentKvTablet = kvTablet; return currentKvTablet == null ? 0L : currentKvTablet.liveSstFilesSize(); } @@ -740,24 +742,35 @@ private void createKv() { // init kv tablet and get the snapshot it uses to init if have any Optional snapshotUsed = Optional.empty(); + Exception lastError = null; for (int i = 1; i <= INIT_KV_TABLET_MAX_RETRY_TIMES; i++) { try { snapshotUsed = initKvTablet(); + lastError = null; break; } catch (Exception e) { + lastError = e; LOG.warn( - "Fail to init kv tablet for bucket {}, retrying for {} times", + "Failed to init kv tablet for bucket {} on attempt {}/{}.", tableBucket, i, + INIT_KV_TABLET_MAX_RETRY_TIMES, e); } } - // A historical KV tablet is a disposable overlay over the lake snapshot. It is recovered - // by replaying WAL from the lake log end offset and does not create its own KV snapshots. - if (isHistoricalPartition()) { - // TODO: Clean up historical KV state after the corresponding WAL is fully tiered to - // lake storage. - } else { + if (lastError != null) { + try { + dropKv(); + } catch (Exception cleanupError) { + lastError.addSuppressed(cleanupError); + } + throw new KvStorageException( + String.format( + "Failed to create KV tablet for bucket %s after %s attempts.", + tableBucket, INIT_KV_TABLET_MAX_RETRY_TIMES), + lastError); + } + if (!isHistoricalPartition()) { startPeriodicKvSnapshot(snapshotUsed.orElse(null)); } } @@ -833,8 +846,8 @@ private Optional initKvTablet() { // get the offset from which, we should restore from. default is 0 long restoreStartOffset = isHistoricalPartition() ? historicalRecoveryStartOffset() : 0; - // The lake snapshot is the durable base for a historical overlay. Historical replicas - // therefore never restore a normal KV snapshot, even if one exists from older code. + // Lake is the durable base for local historical KV state. Historical replicas therefore + // never restore a normal KV snapshot, even if one exists from older code. Optional optCompletedSnapshot = isHistoricalPartition() ? Optional.empty() : getLatestSnapshot(tableBucket); try { @@ -897,6 +910,10 @@ private Optional initKvTablet() { } logTablet.updateMinRetainOffset(restoreStartOffset); + if (isHistoricalPartition()) { + checkNotNull(kvTablet, "kv tablet should not be null.") + .advanceHistoricalCleanupOffset(restoreStartOffset); + } recoverKvTablet(restoreStartOffset, rowCount, autoIncIDRange); } catch (Exception e) { throw new KvStorageException( @@ -1044,7 +1061,14 @@ private void recoverKvTablet( private long historicalRecoveryStartOffset() { long lakeLogEndOffset = logTablet.getLakeLogEndOffset(); + long localLogEndOffset = logTablet.localLogEndOffset(); long logStartOffset = logTablet.logStartOffset(); + checkState( + lakeLogEndOffset < 0 || lakeLogEndOffset <= localLogEndOffset, + "Cannot recover historical KV state: lake log end offset %s is beyond the " + + "local log end offset %s.", + lakeLogEndOffset, + localLogEndOffset); long recoveryStartOffset = lakeLogEndOffset >= 0 ? lakeLogEndOffset : 0L; checkState( recoveryStartOffset >= logStartOffset, @@ -1161,9 +1185,11 @@ public LogAppendInfo appendRecordsToLeader(MemoryLogRecords memoryLogRecords, in "Leader not local for bucket %s on tabletServer %d", tableBucket, localTabletServerId)); } - if (isHistoricalPartition()) { + // Primary-key writes must go through PUT_KV so the log and local KV state are + // updated together. PRODUCE_LOG is only valid for append-only tables. + if (isKvTable()) { throw new InvalidPartitionException( - "Normal write request must not target a historical partition."); + "Produce-log request must not target a primary-key table."); } validateInSyncReplicaSize(requiredAcks); @@ -1250,40 +1276,39 @@ public LogAppendInfo putRecordsToLeader( } /** - * Finds historical write keys that require lake fallback without mutating local KV state. - * - *

The caller must keep historical writes for this table bucket ordered until the subsequent - * {@link #putHistoricalRecordsToLeader} call completes. + * Looks up previous values without holding replica or KV locks during lake I/O, then writes the + * records to the local historical KV state. */ - public List findKeysRequiringLakeLookup( - KvRecordBatch kvRecords, - @Nullable int[] targetColumns, - MergeMode mergeMode, - String originalPartitionName, - int expectedLeaderEpoch, - int requiredAcks) - throws Exception { - return inReadLock( - leaderIsrUpdateLock, - () -> { - validateHistoricalWrite(expectedLeaderEpoch, requiredAcks); - KvTablet kv = this.kvTablet; - checkNotNull(kv, "KvTablet for the historical replica shouldn't be null."); - return kv.findKeysRequiringLakeLookup( - kvRecords, targetColumns, mergeMode, originalPartitionName); - }); - } - - /** Writes records to the local historical KV overlay of the leader replica. */ public LogAppendInfo putHistoricalRecordsToLeader( KvRecordBatch kvRecords, @Nullable int[] targetColumns, MergeMode mergeMode, String originalPartitionName, - HistoricalValueLookup memoizedLakeLookup, + FunctionWithException, List, Exception> lakeLookup, int expectedLeaderEpoch, int requiredAcks) throws Exception { + checkNotNull(lakeLookup, "Historical lake lookup must not be null"); + LocalPreviousValueLookupResult localLookupResult = + inReadLock( + leaderIsrUpdateLock, + () -> { + validateHistoricalWrite(expectedLeaderEpoch, requiredAcks); + KvTablet kv = this.kvTablet; + checkNotNull( + kv, "KvTablet for the historical replica shouldn't be null."); + return kv.probeLocalPreviousValues( + kvRecords, targetColumns, mergeMode, originalPartitionName); + }); + + // Both the replica read lock and the KV read lock have been released before lake I/O. + List lakeValues = + localLookupResult.hasLocalMisses() + ? lakeLookup.apply(localLookupResult.keysMissingLocally()) + : Collections.emptyList(); + HistoricalValueLookup historicalValueLookup = + localLookupResult.createValueLookup(lakeValues); + return inReadLock( leaderIsrUpdateLock, () -> { @@ -1296,12 +1321,49 @@ public LogAppendInfo putHistoricalRecordsToLeader( targetColumns, mergeMode, originalPartitionName, - memoizedLakeLookup); + historicalValueLookup); maybeIncrementLeaderHW(logTablet, clock.milliseconds()); return appendInfo; }); } + /** + * Updates the historical cleanup offset if it is valid. + * + *

{@code beforeUpdate} runs after validation and before the new cleanup offset becomes + * visible to the RocksDB compaction filter. + */ + public void tryUpdateHistoricalCleanupOffset(long newCleanupOffset, Runnable beforeUpdate) { + checkNotNull(beforeUpdate, "beforeUpdate must not be null."); + inWriteLock( + leaderIsrUpdateLock, + () -> { + if (!isLeader() || !historicalPartition) { + return; + } + KvTablet currentKvTablet = kvTablet; + if (currentKvTablet == null) { + return; + } + long currentCleanupOffset = currentKvTablet.getHistoricalCleanupOffset(); + long localLogEndOffset = logTablet.localLogEndOffset(); + if (newCleanupOffset < currentCleanupOffset + || newCleanupOffset > localLogEndOffset) { + LOG.warn( + "Ignore invalid historical cleanup offset {} for {} with " + + "current cleanup offset {} and local log end " + + "offset {}.", + newCleanupOffset, + tableBucket, + currentCleanupOffset, + localLogEndOffset); + return; + } + beforeUpdate.run(); + currentKvTablet.advanceHistoricalCleanupOffset(newCleanupOffset); + }); + } + private void validateHistoricalWrite(int expectedLeaderEpoch, int requiredAcks) { if (!isLeader()) { throw new NotLeaderOrFollowerException( @@ -1323,7 +1385,7 @@ private void validateHistoricalWrite(int expectedLeaderEpoch, int requiredAcks) validateInSyncReplicaSize(requiredAcks); } - /** Looks up keys from the local historical KV overlay of the leader replica. */ + /** Looks up keys from the local historical KV state of the leader replica. */ public List lookupHistoricalLocal( String originalPartitionName, List keys) throws Exception { return inReadLock( diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java index b3357019230..b69bee9eed0 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java @@ -82,6 +82,7 @@ import org.apache.fluss.server.entity.NotifyLeaderAndIsrData; import org.apache.fluss.server.entity.NotifyLeaderAndIsrResultForBucket; import org.apache.fluss.server.entity.NotifyRemoteLogOffsetsData; +import org.apache.fluss.server.entity.ProduceLogDataForBucket; import org.apache.fluss.server.entity.PutKvDataForBucket; import org.apache.fluss.server.entity.StopReplicaData; import org.apache.fluss.server.entity.StopReplicaResultForBucket; @@ -126,6 +127,7 @@ import org.apache.fluss.utils.FileUtils; import org.apache.fluss.utils.FlussPaths; import org.apache.fluss.utils.clock.Clock; +import org.apache.fluss.utils.concurrent.FutureUtils; import org.apache.fluss.utils.concurrent.Scheduler; import org.slf4j.Logger; @@ -688,6 +690,47 @@ public void appendRecordsToLog( timeoutMs, requiredAcks, entriesPerBucket.size(), appendResult, responseCallback); } + /** Appends historical log batches while preserving each original partition in the response. */ + public void appendHistoricalRecordsToLog( + int timeoutMs, + int requiredAcks, + Collection entriesPerBucket, + @Nullable UserContext userContext, + Consumer> responseCallback) { + List> resultFutures = + new ArrayList<>(entriesPerBucket.size()); + for (ProduceLogDataForBucket bucketData : entriesPerBucket) { + CompletableFuture resultFuture = new CompletableFuture<>(); + resultFutures.add(resultFuture); + String originalPartitionName = + checkNotNull( + bucketData.originalPartitionName(), + "originalPartitionName must not be null"); + appendRecordsToLog( + timeoutMs, + requiredAcks, + Collections.singletonMap(bucketData.tableBucket(), bucketData.records()), + userContext, + bucketResults -> { + ProduceLogResultForBucket result = bucketResults.get(0); + ProduceLogResultForBucket historicalResult = + result.failed() + ? ProduceLogResultForBucket.historicalFailure( + result.getTableBucket(), + result.getError(), + originalPartitionName) + : ProduceLogResultForBucket.historicalSuccess( + result.getTableBucket(), + result.getBaseOffset(), + result.getWriteLogEndOffset(), + originalPartitionName); + resultFuture.complete(historicalResult); + }); + } + FutureUtils.combineAll(resultFutures) + .thenAccept(results -> responseCallback.accept(new ArrayList<>(results))); + } + /** * Fetch records from a replica. Currently, we will return the fetched records immediately. * @@ -1350,7 +1393,8 @@ public void notifyLakeTableOffset( lakeBucketOffsets.entrySet()) { TableBucket tb = lakeBucketOffsetEntry.getKey(); LakeBucketOffset lakeBucketOffset = lakeBucketOffsetEntry.getValue(); - LogTablet logTablet = getReplicaOrException(tb).getLogTablet(); + Replica replica = getReplicaOrException(tb); + LogTablet logTablet = replica.getLogTablet(); logTablet.updateLakeTableSnapshotId(lakeBucketOffset.getSnapshotId()); lakeBucketOffset @@ -1359,7 +1403,16 @@ public void notifyLakeTableOffset( lakeBucketOffset .getLogEndOffset() - .ifPresent(logTablet::updateLakeLogEndOffset); + .ifPresent( + lakeLogEndOffset -> { + logTablet.updateLakeLogEndOffset(lakeLogEndOffset); + if (replica.isHistoricalPartition()) { + historicalPartitionManager.onLakeProgress( + replica, + lakeBucketOffset.getSnapshotId(), + lakeLogEndOffset); + } + }); lakeBucketOffset .getMaxTimestamp() @@ -1425,22 +1478,22 @@ private void updateWithLakeTableSnapshot(Replica replica) throws Exception { LakeTableSnapshot lakeTableSnapshot = optLakeTableSnapshot.get(); long snapshotId = optLakeTableSnapshot.get().getSnapshotId(); replica.getLogTablet().updateLakeTableSnapshotId(snapshotId); + lakeTableSnapshot + .getLogEndOffset(tb) + .ifPresent(replica.getLogTablet()::updateLakeLogEndOffset); if (replica.isHistoricalPartition()) { - // The historical overlay will be rebuilt from this snapshot's lake offset. + // Local historical KV state will be rebuilt from this snapshot's lake offset. // Refresh a cached lookuper before it becomes the fallback for data omitted - // from the rebuilt overlay. + // from the rebuilt local state. historicalPartitionManager.requireLakeSnapshot( replica.getTableBucket().getTableId(), snapshotId); } - lakeTableSnapshot - .getLogEndOffset(tb) - .ifPresent(replica.getLogTablet()::updateLakeLogEndOffset); } } catch (Exception e) { if (replica.isHistoricalPartition()) { // Historical recovery uses the lake offset as its durable base and replays the // retained WAL from that offset. Reject leader activation if the latest lake - // progress cannot be loaded, instead of rebuilding the overlay from stale state. + // progress cannot be loaded, instead of rebuilding local KV from stale state. throw e; } // Lake commit cleanup can race with this best-effort refresh and remove the diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManager.java index af86581e44d..39d2f02fcc8 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManager.java @@ -80,7 +80,7 @@ * than table path to prevent a deleted and recreated table from reusing the old table's lookuper. A * cached lookuper is replaced when its schema ID or lake configuration version no longer matches * the current request. Active lookups can finish on the old lookuper, which is closed after its - * last lookup releases it. + * last lookup releases it. A new required lake snapshot refreshes the cached lookuper in place. * *

Up to ten table lookupers are cached. Each lookuper receives one tenth of the server-level * disk budget, and Caffeine evicts lookupers when the table limit is exceeded. @@ -272,7 +272,7 @@ void invalidateTableLookuper(long tableId) { lakeTableLookupers.invalidate(tableId); } - /** Records the required opaque lake snapshot ID, which is compared only by equality. */ + /** Records the required opaque lake snapshot ID for the next in-place file refresh. */ void requireLakeSnapshot(long tableId, long snapshotId) { requiredLakeSnapshotIds.put(tableId, snapshotId); } @@ -481,12 +481,16 @@ private CachedLakeTableLookuper acquireLookuper(LookupContext context, TableInfo long currentLakeConfigVersion = lakeConfigVersion; Configuration currentConf = conf; long cacheSizeBytes = lookupCacheMaxDiskBytesPerTable; - Long requiredLakeSnapshotId = requiredLakeSnapshotIds.get(context.tableId); return lakeTableLookupers .asMap() .compute( context.tableId, (ignored, currentLookuper) -> { + // Read inside the per-table atomic update to avoid refreshing back to a + // snapshot captured while waiting for another lookup to finish updating + // this lookuper. + Long requiredLakeSnapshotId = + requiredLakeSnapshotIds.get(context.tableId); CachedLakeTableLookuper selectedLookuper = currentLookuper; // Create the lookuper lazily, and recreate it after schema, // lake configuration, or server cache size changes so it @@ -496,10 +500,7 @@ private CachedLakeTableLookuper acquireLookuper(LookupContext context, TableInfo || selectedLookuper.schemaId != context.schemaId || selectedLookuper.lakeConfigVersion != currentLakeConfigVersion - || selectedLookuper.cacheSizeBytes != cacheSizeBytes - || !Objects.equals( - selectedLookuper.lakeSnapshotId, - requiredLakeSnapshotId)) { + || selectedLookuper.cacheSizeBytes != cacheSizeBytes) { File tableLookupDir = FlussPaths.historicalLookupTableDir( historicalLookupCacheRootDir, @@ -526,7 +527,7 @@ private CachedLakeTableLookuper acquireLookuper(LookupContext context, TableInfo // Pin the lookuper before leaving the atomic cache update. // Eviction or invalidation can then defer closing it until // this lookup releases it. - selectedLookuper.acquire(); + selectedLookuper.acquire(requiredLakeSnapshotId); return selectedLookuper; }); } @@ -555,8 +556,8 @@ private static final class CachedLakeTableLookuper { private final int schemaId; private final long lakeConfigVersion; private final long cacheSizeBytes; - /** The required opaque lake snapshot ID when this lookuper was created, or null if none. */ - private final @Nullable Long lakeSnapshotId; + /** The opaque lake snapshot ID covered by the last file refresh, or null if none. */ + private @Nullable Long lakeSnapshotId; private final File tableLookupDir; private final LakeTableLookuper lookuper; @@ -583,10 +584,14 @@ private CachedLakeTableLookuper( this.lookuper = lookuper; } - private synchronized void acquire() { + private synchronized void acquire(@Nullable Long requiredLakeSnapshotId) { if (invalidated) { throw new IllegalStateException("Lake table lookuper has been invalidated."); } + if (!Objects.equals(lakeSnapshotId, requiredLakeSnapshotId)) { + lookuper.refresh(); + lakeSnapshotId = requiredLakeSnapshotId; + } activeLookups++; } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java index 13ec002e0df..d62e49ba1c0 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java @@ -37,12 +37,10 @@ import org.apache.fluss.server.entity.PutKvDataForBucket; import org.apache.fluss.server.kv.KvStateLookupResult; import org.apache.fluss.server.kv.KvStateLookupResult.Status; -import org.apache.fluss.server.kv.historical.HistoricalValueLookup; import org.apache.fluss.server.log.LogAppendInfo; import org.apache.fluss.server.replica.Replica; import org.apache.fluss.server.storage.LocalDiskManager; import org.apache.fluss.utils.ByteArraySlice; -import org.apache.fluss.utils.ByteArrayWrapper; import org.apache.fluss.utils.concurrent.Scheduler; import javax.annotation.Nullable; @@ -50,10 +48,8 @@ import java.io.File; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.concurrent.CompletableFuture; import static org.apache.fluss.utils.Preconditions.checkNotNull; @@ -98,7 +94,7 @@ public void startup(Scheduler scheduler) { lakeLookupManager.startup(scheduler); } - /** Looks up historical keys from the local overlay and then lake storage. */ + /** Looks up historical keys from local KV state and then lake storage. */ public CompletableFuture lookup( Replica replica, LookupDataForBucket lookupData, @@ -119,7 +115,8 @@ public CompletableFuture lookup( + tableBucket + " (original partition " + lookupData.originalPartitionName() - + ").")))); + + ") because the historical request " + + "queue is full.")))); } catch (RuntimeException e) { return CompletableFuture.completedFuture( new LookupResultForBucket( @@ -129,7 +126,7 @@ public CompletableFuture lookup( } } - /** Writes records to the local overlay of a historical partition. */ + /** Writes records to the local KV state of a historical partition. */ public CompletableFuture putKv( Replica replica, PutKvDataForBucket putData, @@ -163,17 +160,7 @@ public CompletableFuture putKv( originalPartitionName); } }, - () -> - PutKvResultForBucket.historicalFailure( - putData.tableBucket(), - ApiError.fromThrowable( - new HistoricalPartitionThrottledException( - "Historical write is throttled for " - + putData.tableBucket() - + " (original partition " - + originalPartitionName - + ").")), - originalPartitionName)); + () -> requestLimitThrottledResult(putData, originalPartitionName)); } catch (RuntimeException e) { return CompletableFuture.completedFuture( PutKvResultForBucket.historicalFailure( @@ -193,11 +180,21 @@ public void invalidateTableLookuper(long tableId) { lakeLookupManager.invalidateTableLookuper(tableId); } - /** Requires future fallback lookups to reload after the given lake snapshot notification. */ + /** Requires future fallback lookups to refresh after the given lake snapshot notification. */ public void requireLakeSnapshot(long tableId, long lakeSnapshotId) { lakeLookupManager.requireLakeSnapshot(tableId, lakeSnapshotId); } + /** Publishes lake coverage used by natural RocksDB compaction for a historical leader. */ + public void onLakeProgress(Replica replica, long lakeSnapshotId, long lakeLogEndOffset) { + replica.tryUpdateHistoricalCleanupOffset( + lakeLogEndOffset, + () -> + // Future fallback lookups must require the covering snapshot before local + // entries become eligible for physical removal. + requireLakeSnapshot(replica.getTableBucket().getTableId(), lakeSnapshotId)); + } + /** Returns the number of accepted historical operations that have not completed. */ public int numInflightRequests() { return taskExecutor.numInflightRequests(); @@ -234,58 +231,41 @@ LogAppendInfo processPut( ResolvedPartitionSpec.fromPartitionName( tableInfo.getPartitionKeys(), originalPartitionName); // The public put path holds the TableBucket ordering slot until processPut returns, so - // local state cannot be changed by a later historical write between resolve and apply. + // local state cannot be changed by a later historical write between lookup and apply. int expectedLeaderEpoch = replica.getLeaderEpoch(); - List keysRequiringLakeLookup = - replica.findKeysRequiringLakeLookup( - putData.records(), - targetColumns, - mergeMode, - originalPartitionName, - expectedLeaderEpoch, - requiredAcks); - - Map lakeResults = new HashMap<>(); - if (!keysRequiringLakeLookup.isEmpty()) { - List lakeValues = - lakeLookupManager.lookup( - new LookupDataForBucket( - putData.tableBucket(), - keysRequiringLakeLookup, - originalPartitionName), - tableInfo, - replica.getLatestSchemaInfo(), - originalPartitionSpec, - replica.tableMetrics()::recordHistoricalLakeLookup); - for (int i = 0; i < keysRequiringLakeLookup.size(); i++) { - byte[] lakeValue = lakeValues.get(i); - lakeResults.put( - new ByteArrayWrapper(keysRequiringLakeLookup.get(i)), - lakeValue == null - ? KvStateLookupResult.notFound() - : KvStateLookupResult.present(lakeValue)); - } - } - - HistoricalValueLookup memoizedLakeLookup = - primaryKey -> { - KvStateLookupResult result = - checkNotNull( - lakeResults.get(new ByteArrayWrapper(primaryKey)), - "No resolved lake value for a historical write key"); - return result.value(); - }; - return replica.putHistoricalRecordsToLeader( putData.records(), targetColumns, mergeMode, originalPartitionName, - memoizedLakeLookup, + lakeLookupKeys -> + lakeLookupManager.lookup( + new LookupDataForBucket( + putData.tableBucket(), + lakeLookupKeys, + originalPartitionName), + tableInfo, + replica.getLatestSchemaInfo(), + originalPartitionSpec, + replica.tableMetrics()::recordHistoricalLakeLookup), expectedLeaderEpoch, requiredAcks); } + private static PutKvResultForBucket requestLimitThrottledResult( + PutKvDataForBucket putData, String originalPartitionName) { + return PutKvResultForBucket.historicalFailure( + putData.tableBucket(), + ApiError.fromThrowable( + new HistoricalPartitionThrottledException( + "Historical write is throttled for " + + putData.tableBucket() + + " (original partition " + + originalPartitionName + + ") because the historical request queue is full.")), + originalPartitionName); + } + @Override public void close() { taskExecutor.close(); @@ -335,15 +315,13 @@ private LookupResultForBucket lookupInternal( Iterator lakeValueIterator = lakeValues.iterator(); List values = new ArrayList<>(localResults.size()); - KvValueLayout localValueLayout = - KvValueLayout.fromTableConfig(tableInfo.getTableConfig()); for (KvStateLookupResult localResult : localResults) { // Consume one lake value for each NOT_FOUND result; local values and tombstones // keep their original positions without advancing the lake iterator. if (localResult.status() == Status.NOT_FOUND) { values.add(KvValueLayout.PLAIN.toValueBodySlice(lakeValueIterator.next())); } else { - values.add(localValueLayout.toValueBodySlice(localResult.value())); + values.add(KvValueLayout.TAGGED.toValueBodySlice(localResult.value())); } } return new LookupResultForBucket(tableBucket, values, originalPartitionName); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java b/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java index bd3ef49b35a..c3468b7a941 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java @@ -36,6 +36,7 @@ import org.apache.fluss.rpc.entity.FetchLogResultForBucket; import org.apache.fluss.rpc.entity.LookupResultForBucket; import org.apache.fluss.rpc.entity.PrefixLookupResultForBucket; +import org.apache.fluss.rpc.entity.ProduceLogResultForBucket; import org.apache.fluss.rpc.entity.ResultForBucket; import org.apache.fluss.rpc.gateway.CoordinatorGateway; import org.apache.fluss.rpc.gateway.TabletServerGateway; @@ -92,6 +93,7 @@ import org.apache.fluss.server.entity.NotifyLakeTableOffsetData; import org.apache.fluss.server.entity.NotifyLeaderAndIsrData; import org.apache.fluss.server.entity.NotifyRemoteLogOffsetsData; +import org.apache.fluss.server.entity.ProduceLogDataForBucket; import org.apache.fluss.server.entity.PutKvDataForBucket; import org.apache.fluss.server.entity.StopReplicaData; import org.apache.fluss.server.entity.UserContext; @@ -125,6 +127,7 @@ import java.util.stream.Collectors; import static org.apache.fluss.rpc.util.CommonRpcMessageUtils.hasHistoricalLookup; +import static org.apache.fluss.rpc.util.CommonRpcMessageUtils.hasHistoricalProduce; import static org.apache.fluss.rpc.util.CommonRpcMessageUtils.hasHistoricalPut; import static org.apache.fluss.security.acl.OperationType.DESCRIBE; import static org.apache.fluss.security.acl.OperationType.READ; @@ -137,7 +140,6 @@ import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getNotifyLeaderAndIsrRequestData; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getNotifyRemoteLogOffsetsData; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getNotifySnapshotOffsetData; -import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getProduceLogData; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getStopReplicaData; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getTableFilterInfoMap; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getTableStatsRequestData; @@ -157,6 +159,7 @@ import static org.apache.fluss.server.utils.ServerRpcMessageUtils.toHistoricalLookupData; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.toLookupData; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.toPrefixLookupData; +import static org.apache.fluss.server.utils.ServerRpcMessageUtils.toProduceLogDataForBuckets; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.toPutKvDataForBuckets; /** An RPC Gateway service for tablet server. */ @@ -216,13 +219,29 @@ public void shutdown() {} public CompletableFuture produceLog(ProduceLogRequest request) { authorizeTable(WRITE, request.getTableId()); CompletableFuture response = new CompletableFuture<>(); - Map produceLogData = getProduceLogData(request); - replicaManager.appendRecordsToLog( - request.getTimeoutMs(), - request.getAcks(), - produceLogData, - new UserContext(currentSession().getPrincipal()), - bucketResponseMap -> response.complete(makeProduceLogResponse(bucketResponseMap))); + List produceLogData = toProduceLogDataForBuckets(request); + UserContext userContext = new UserContext(currentSession().getPrincipal()); + Consumer> responseCallback = + results -> response.complete(makeProduceLogResponse(results)); + if (hasHistoricalProduce(request)) { + replicaManager.appendHistoricalRecordsToLog( + request.getTimeoutMs(), + request.getAcks(), + produceLogData, + userContext, + responseCallback); + } else { + Map recordsByBucket = new HashMap<>(); + for (ProduceLogDataForBucket bucketData : produceLogData) { + recordsByBucket.put(bucketData.tableBucket(), bucketData.records()); + } + replicaManager.appendRecordsToLog( + request.getTimeoutMs(), + request.getAcks(), + recordsByBucket, + userContext, + responseCallback); + } return response; } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java index 02297ead2dd..58a35b392b0 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java @@ -187,6 +187,7 @@ import org.apache.fluss.server.entity.NotifyLeaderAndIsrData; import org.apache.fluss.server.entity.NotifyLeaderAndIsrResultForBucket; import org.apache.fluss.server.entity.NotifyRemoteLogOffsetsData; +import org.apache.fluss.server.entity.ProduceLogDataForBucket; import org.apache.fluss.server.entity.PutKvDataForBucket; import org.apache.fluss.server.entity.StopReplicaData; import org.apache.fluss.server.entity.StopReplicaResultForBucket; @@ -231,6 +232,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.apache.fluss.rpc.util.CommonRpcMessageUtils.hasHistoricalProduce; import static org.apache.fluss.rpc.util.CommonRpcMessageUtils.toByteBuffer; import static org.apache.fluss.rpc.util.CommonRpcMessageUtils.toPbAclInfo; import static org.apache.fluss.utils.Preconditions.checkNotNull; @@ -892,12 +894,20 @@ public static StopReplicaResponse makeStopReplicaResponse( return stopReplicaResponse; } - public static Map getProduceLogData( + /** Converts produce-log requests while preserving their historical partition context. */ + public static List toProduceLogDataForBuckets( ProduceLogRequest produceRequest) { long tableId = produceRequest.getTableId(); - Map produceEntryData = new HashMap<>(); + List produceLogData = + new ArrayList<>(produceRequest.getBucketsReqsCount()); + Map> originalPartitionsByBucket = new HashMap<>(); + boolean historicalWriteRequest = hasHistoricalProduce(produceRequest); for (PbProduceLogReqForBucket produceLogReqForBucket : produceRequest.getBucketsReqsList()) { + if (produceLogReqForBucket.hasOriginalPartitionName() != historicalWriteRequest) { + throw new IllegalArgumentException( + "Normal and historical writes cannot be mixed in the same request."); + } ByteBuffer recordBuffer = toByteBuffer(produceLogReqForBucket.getRecordsSlice()); MemoryLogRecords logRecords = MemoryLogRecords.pointToByteBuffer(recordBuffer); TableBucket tb = @@ -907,9 +917,23 @@ public static Map getProduceLogData( ? produceLogReqForBucket.getPartitionId() : null, produceLogReqForBucket.getBucketId()); - produceEntryData.put(tb, logRecords); + String originalPartitionName = + produceLogReqForBucket.hasOriginalPartitionName() + ? produceLogReqForBucket.getOriginalPartitionName() + : null; + Set originalPartitions = + originalPartitionsByBucket.computeIfAbsent(tb, ignored -> new HashSet<>()); + if (!originalPartitions.add(originalPartitionName)) { + throw new IllegalArgumentException( + "A ProduceLog request contains duplicate table bucket " + + tb + + " and original partition " + + originalPartitionName + + '.'); + } + produceLogData.add(new ProduceLogDataForBucket(tb, logRecords, originalPartitionName)); } - return produceEntryData; + return produceLogData; } public static ProduceLogResponse makeProduceLogResponse( @@ -923,6 +947,9 @@ public static ProduceLogResponse makeProduceLogResponse( if (tableBucket.getPartitionId() != null) { producedBucket.setPartitionId(tableBucket.getPartitionId()); } + if (bucketResult.getOriginalPartitionName() != null) { + producedBucket.setOriginalPartitionName(bucketResult.getOriginalPartitionName()); + } if (bucketResult.failed()) { producedBucket.setError( diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java index 8ba8204a54c..0bd9ab6f522 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java @@ -214,10 +214,6 @@ private static void checkHistoricalPartition( DataLakeFormat.PAIMON, dataLakeFormat.get())); } - if (!tableDescriptor.hasPrimaryKey()) { - unmetRequirements.add("the table must define a primary key"); - } - int partitionKeyCount = tableDescriptor.getPartitionKeys().size(); if (partitionKeyCount != 1) { unmetRequirements.add( diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/HistoricalKvCompactionFilterTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/HistoricalKvCompactionFilterTest.java new file mode 100644 index 00000000000..9a17e2bf2ae --- /dev/null +++ b/fluss-server/src/test/java/org/apache/fluss/server/kv/HistoricalKvCompactionFilterTest.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.kv; + +import org.apache.fluss.record.BinaryValue; +import org.apache.fluss.rocksdb.RocksDBHandle; +import org.apache.fluss.row.BinaryRow; +import org.apache.fluss.row.encode.KvValueLayout; +import org.apache.fluss.row.encode.ValueEncoder; +import org.apache.fluss.server.kv.historical.HistoricalKvTombstone; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.rocksdb.ColumnFamilyOptions; +import org.rocksdb.DBOptions; +import org.rocksdb.FlinkCompactionFilter; +import org.rocksdb.FlushOptions; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.concurrent.atomic.AtomicLong; + +import static org.apache.fluss.record.TestData.DATA1_ROW_TYPE; +import static org.apache.fluss.record.TestData.DEFAULT_SCHEMA_ID; +import static org.apache.fluss.testutils.DataTestUtils.compactedRow; +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests exclusive-offset cleanup for offset-tagged historical values and tombstones. */ +class HistoricalKvCompactionFilterTest { + + @TempDir private Path tempDir; + + @Test + void testRemovesOnlyOffsetsBelowCleanupOffset() throws Exception { + AtomicLong cleanupOffset = new AtomicLong(0L); + byte[] valueBeforeKey = bytes("value-before"); + byte[] tombstoneBeforeKey = bytes("tombstone-before"); + byte[] valueAtKey = bytes("value-at"); + byte[] tombstoneAtKey = bytes("tombstone-at"); + byte[] valueAfterKey = bytes("value-after"); + byte[] tombstoneAfterKey = bytes("tombstone-after"); + BinaryRow row = compactedRow(DATA1_ROW_TYPE, new Object[] {1, "a"}); + + try (FlinkCompactionFilter.FlinkCompactionFilterFactory filterFactory = + RowTtlCompactionFilterFactory.create( + KvValueLayout.TAGGED, 0L, 1L, () -> cleanupOffset.get() - 1L); + DBOptions dbOptions = new DBOptions().setCreateIfMissing(true); + ColumnFamilyOptions cfOptions = + new ColumnFamilyOptions().setCompactionFilterFactory(filterFactory); + RocksDBHandle handle = new RocksDBHandle(tempDir.toFile(), dbOptions, cfOptions); + FlushOptions flushOptions = new FlushOptions().setWaitForFlush(true)) { + handle.openDB(); + handle.getDb().put(valueBeforeKey, encodeValue(row, 4L)); + handle.getDb().put(tombstoneBeforeKey, HistoricalKvTombstone.encode(4L)); + handle.getDb().put(valueAtKey, encodeValue(row, 5L)); + handle.getDb().put(tombstoneAtKey, HistoricalKvTombstone.encode(5L)); + handle.getDb().put(valueAfterKey, encodeValue(row, 6L)); + handle.getDb().put(tombstoneAfterKey, HistoricalKvTombstone.encode(6L)); + handle.getDb().flush(flushOptions); + + handle.getDb().compactRange(); + assertThat(handle.getDb().get(valueBeforeKey)).isNotNull(); + assertThat(handle.getDb().get(tombstoneBeforeKey)).isNotNull(); + + cleanupOffset.set(5L); + handle.getDb().compactRange(); + + assertThat(handle.getDb().get(valueBeforeKey)).isNull(); + assertThat(handle.getDb().get(tombstoneBeforeKey)).isNull(); + assertThat(handle.getDb().get(valueAtKey)).isNotNull(); + assertThat(handle.getDb().get(tombstoneAtKey)).isNotNull(); + assertThat(handle.getDb().get(valueAfterKey)).isNotNull(); + assertThat(handle.getDb().get(tombstoneAfterKey)).isNotNull(); + } + } + + private static byte[] encodeValue(BinaryRow row, long logOffset) { + return ValueEncoder.forLayout(KvValueLayout.TAGGED) + .encodeValue(new BinaryValue(DEFAULT_SCHEMA_ID, row), logOffset); + } + + private static byte[] bytes(String value) { + return value.getBytes(StandardCharsets.UTF_8); + } +} diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/RowTtlCompactionFilterTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/RowTtlCompactionFilterTest.java index 762b3756c2e..644922c6d5a 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/kv/RowTtlCompactionFilterTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/kv/RowTtlCompactionFilterTest.java @@ -56,10 +56,7 @@ void testFlinkCompactionFilterReadsTimestampFromTaggedValue() throws Exception { try (FlinkCompactionFilter.FlinkCompactionFilterFactory filterFactory = RowTtlCompactionFilterFactory.create( - KvValueLayout.TAGGED, - Duration.ofHours(1L), - 1L, - new ManualClock(now)); + KvValueLayout.TAGGED, Duration.ofHours(1L), new ManualClock(now)); DBOptions dbOptions = new DBOptions().setCreateIfMissing(true); ColumnFamilyOptions cfOptions = new ColumnFamilyOptions().setCompactionFilterFactory(filterFactory); @@ -83,10 +80,7 @@ void testCreateRejectsInvalidTtlDuration() { assertThatThrownBy( () -> RowTtlCompactionFilterFactory.create( - KvValueLayout.TAGGED, - Duration.ZERO, - 1L, - new ManualClock(0L))) + KvValueLayout.TAGGED, Duration.ZERO, new ManualClock(0L))) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining(ConfigOptions.TABLE_KV_TTL.key()); } diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/historical/LocalPreviousValueLookupResultTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/historical/LocalPreviousValueLookupResultTest.java new file mode 100644 index 00000000000..6622152162d --- /dev/null +++ b/fluss-server/src/test/java/org/apache/fluss/server/kv/historical/LocalPreviousValueLookupResultTest.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.kv.historical; + +import org.apache.fluss.metadata.KvFormat; +import org.apache.fluss.record.BinaryValue; +import org.apache.fluss.record.TestingSchemaGetter; +import org.apache.fluss.row.BinaryRow; +import org.apache.fluss.row.encode.KvValueLayout; +import org.apache.fluss.row.encode.ValueDecoder; +import org.apache.fluss.row.encode.ValueEncoder; +import org.apache.fluss.server.kv.KvStateLookupResult; + +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; + +import static org.apache.fluss.record.TestData.DATA1_ROW_TYPE; +import static org.apache.fluss.record.TestData.DATA1_SCHEMA; +import static org.apache.fluss.record.TestData.DEFAULT_SCHEMA_ID; +import static org.apache.fluss.testutils.DataTestUtils.compactedRow; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for {@link LocalPreviousValueLookupResult}. */ +class LocalPreviousValueLookupResultTest { + + @Test + void testCreatesLookupFromLocalAndLakeResults() { + LocalPreviousValueLookupResult localLookupResult = createLookupResult(); + byte[] localValueKey = new byte[] {1}; + byte[] localDeleteKey = new byte[] {2}; + byte[] lakeValueKey = new byte[] {3}; + byte[] lakeMissKey = new byte[] {4}; + BinaryValue localValue = binaryValue(1, "local"); + BinaryValue lakeValue = binaryValue(3, "lake"); + + localLookupResult.add( + localValueKey, + KvStateLookupResult.present( + ValueEncoder.forLayout(KvValueLayout.TAGGED).encodeValue(localValue, 10L))); + localLookupResult.add(localDeleteKey, KvStateLookupResult.deleted()); + localLookupResult.add(lakeValueKey, KvStateLookupResult.notFound()); + localLookupResult.add(lakeMissKey, KvStateLookupResult.notFound()); + + assertThat(localLookupResult.hasLocalMisses()).isTrue(); + assertThat(localLookupResult.keysMissingLocally()) + .containsExactly(lakeValueKey, lakeMissKey); + + HistoricalValueLookup valueLookup = + localLookupResult.createValueLookup( + Arrays.asList( + ValueEncoder.forLayout(KvValueLayout.PLAIN).encodeValue(lakeValue), + null)); + + assertThat(valueLookup.lookup(localValueKey)).isEqualTo(localValue); + assertThat(valueLookup.lookup(localDeleteKey)).isNull(); + assertThat(valueLookup.lookup(lakeValueKey)).isEqualTo(lakeValue); + assertThat(valueLookup.lookup(lakeMissKey)).isNull(); + assertThatThrownBy(() -> valueLookup.lookup(new byte[] {5})) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("No previous value for a historical write key"); + } + + @Test + void testValidatesLakeResultCountBeforeCreatingLookup() { + LocalPreviousValueLookupResult localLookupResult = createLookupResult(); + localLookupResult.add(new byte[] {1}, KvStateLookupResult.notFound()); + + assertThatThrownBy(() -> localLookupResult.createValueLookup(Collections.emptyList())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Expected 1 historical lake values, but received 0"); + } + + private static LocalPreviousValueLookupResult createLookupResult() { + TestingSchemaGetter schemaGetter = new TestingSchemaGetter(DEFAULT_SCHEMA_ID, DATA1_SCHEMA); + return new LocalPreviousValueLookupResult( + new ValueDecoder(schemaGetter, KvFormat.COMPACTED, KvValueLayout.TAGGED), + new ValueDecoder(schemaGetter, KvFormat.COMPACTED, KvValueLayout.PLAIN)); + } + + private static BinaryValue binaryValue(int id, String value) { + BinaryRow row = compactedRow(DATA1_ROW_TYPE, new Object[] {id, value}); + return new BinaryValue(DEFAULT_SCHEMA_ID, row); + } +} diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/HighWatermarkPersistenceTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/HighWatermarkPersistenceTest.java index a520f9dad2e..ddd0574be5d 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/HighWatermarkPersistenceTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/HighWatermarkPersistenceTest.java @@ -18,25 +18,18 @@ package org.apache.fluss.server.replica; import org.apache.fluss.config.ConfigOptions; -import org.apache.fluss.metadata.PhysicalTablePath; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.server.coordinator.TestCoordinatorGateway; -import org.apache.fluss.server.entity.NotifyLeaderAndIsrData; import org.apache.fluss.server.log.checkpoint.OffsetCheckpointFile; -import org.apache.fluss.server.zk.data.LeaderAndIsr; import org.junit.jupiter.api.Test; import java.io.File; import java.time.Duration; -import java.util.Collections; import static org.apache.fluss.record.TestData.ANOTHER_DATA1; import static org.apache.fluss.record.TestData.DATA1; import static org.apache.fluss.record.TestData.DATA1_TABLE_ID; -import static org.apache.fluss.record.TestData.DATA2_TABLE_ID; -import static org.apache.fluss.record.TestData.DATA2_TABLE_PATH; -import static org.apache.fluss.server.coordinator.CoordinatorContext.INITIAL_COORDINATOR_EPOCH; import static org.apache.fluss.server.replica.ReplicaManager.HIGH_WATERMARK_CHECKPOINT_FILE_NAME; import static org.apache.fluss.testutils.DataTestUtils.genMemoryLogRecordsByObject; import static org.apache.fluss.testutils.common.CommonTestUtils.retry; @@ -92,23 +85,9 @@ void testHighWatermarkPersistenceMultipleReplicas() throws Exception { assertThat(highWatermark0).isEqualTo(10L); assertThat(replica0.getLogTablet().getHighWatermark()).isEqualTo(10L); - // add another replica and set highWatermark. - TableBucket tableBucket1 = new TableBucket(DATA2_TABLE_ID, 0); - replicaManager.becomeLeaderOrFollower( - INITIAL_COORDINATOR_EPOCH, - Collections.singletonList( - new NotifyLeaderAndIsrData( - PhysicalTablePath.of(DATA2_TABLE_PATH), - tableBucket1, - Collections.singletonList(TABLET_SERVER_ID), - new LeaderAndIsr( - TABLET_SERVER_ID, - LeaderAndIsr.INITIAL_LEADER_EPOCH, - Collections.singletonList(TABLET_SERVER_ID), - Collections.emptyList(), - INITIAL_COORDINATOR_EPOCH, - LeaderAndIsr.INITIAL_BUCKET_EPOCH))), - result -> {}); + // Add another append-only replica and set its high watermark. + TableBucket tableBucket1 = new TableBucket(DATA1_TABLE_ID, 1); + makeLogTableAsLeader(tableBucket1.getBucket()); replicaManager.checkpointHighWatermarks(); long highWatermark1 = highWatermarkFor(tableBucket1); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerTest.java index db4a88bc085..3364aa61b08 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerTest.java @@ -67,6 +67,7 @@ import org.apache.fluss.server.entity.FetchReqInfo; import org.apache.fluss.server.entity.NotifyLeaderAndIsrData; import org.apache.fluss.server.entity.NotifyLeaderAndIsrResultForBucket; +import org.apache.fluss.server.entity.ProduceLogDataForBucket; import org.apache.fluss.server.entity.StopReplicaData; import org.apache.fluss.server.entity.StopReplicaResultForBucket; import org.apache.fluss.server.kv.KvTablet; @@ -238,6 +239,32 @@ tb, genMemoryLogRecordsByObject(DATA1)), "Unknown table or bucket: TableBucket{tableId=10001, bucket=0}"))); } + @Test + void testProduceHistoricalLogBatchesToSameTableBucket() throws Exception { + replicaManager.getDiskUsageMonitor().update(0.10); + TableBucket tableBucket = new TableBucket(DATA1_TABLE_ID, 1); + makeLogTableAsLeader(tableBucket.getBucket()); + + CompletableFuture> future = new CompletableFuture<>(); + replicaManager.appendHistoricalRecordsToLog( + 20_000, + 1, + Arrays.asList( + new ProduceLogDataForBucket( + tableBucket, genMemoryLogRecordsByObject(DATA1), "dt=2025-01-01"), + new ProduceLogDataForBucket( + tableBucket, genMemoryLogRecordsByObject(DATA1), "dt=2025-01-02")), + null, + future::complete); + + assertThat(future.get()) + .containsExactlyInAnyOrder( + ProduceLogResultForBucket.historicalSuccess( + tableBucket, 0L, 10L, "dt=2025-01-01"), + ProduceLogResultForBucket.historicalSuccess( + tableBucket, 10L, 20L, "dt=2025-01-02")); + } + @Test void testFetchLog() throws Exception { SchemaGetter schemaGetter = diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManagerTest.java index 364ec46bedc..0be4672c01b 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManagerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManagerTest.java @@ -162,29 +162,30 @@ void testInvalidatesLookuperOnSchemaAndLifecycleChanges() throws Exception { } @Test - void testRefreshesLookuperWhenLakeSnapshotChanges() throws Exception { + void testRefreshesFilesWhenLakeSnapshotChanges() throws Exception { TestingHistoricalLakeLookupManager manager = createTestingManager(); lookup(manager, PARTITION_TABLE_INFO); - TestingLakeTableLookuper initialLookuper = manager.createdLookupers.get(0); + TestingLakeTableLookuper lookuper = manager.createdLookupers.get(0); manager.requireLakeSnapshot(PARTITION_TABLE_ID, 10L); lookup(manager, PARTITION_TABLE_INFO); - assertThat(initialLookuper.closed).isTrue(); - assertThat(manager.createdLookupers).hasSize(2); + assertThat(lookuper.closed).isFalse(); + assertThat(lookuper.refreshCount).isOne(); + assertThat(manager.createdLookupers).containsExactly(lookuper); - TestingLakeTableLookuper snapshotTenLookuper = manager.createdLookupers.get(1); // Snapshot IDs are opaque; a numerically smaller ID may identify a newer snapshot. manager.requireLakeSnapshot(PARTITION_TABLE_ID, 9L); lookup(manager, PARTITION_TABLE_INFO); - assertThat(snapshotTenLookuper.closed).isTrue(); - assertThat(manager.createdLookupers).hasSize(3); + assertThat(lookuper.closed).isFalse(); + assertThat(lookuper.refreshCount).isEqualTo(2); + assertThat(manager.createdLookupers).containsExactly(lookuper); - TestingLakeTableLookuper snapshotNineLookuper = manager.createdLookupers.get(2); manager.requireLakeSnapshot(PARTITION_TABLE_ID, 9L); lookup(manager, PARTITION_TABLE_INFO); - assertThat(snapshotNineLookuper.closed).isFalse(); - assertThat(manager.createdLookupers).hasSize(3); + assertThat(lookuper.closed).isFalse(); + assertThat(lookuper.refreshCount).isEqualTo(2); + assertThat(manager.createdLookupers).containsExactly(lookuper); } @Test @@ -421,6 +422,7 @@ private static final class TestingLakeTableLookuper implements LakeTableLookuper private final long cacheFileBytes; private boolean closed; private boolean cacheFileDownloaded; + private int refreshCount; private final List lookupContexts = new ArrayList<>(); private TestingLakeTableLookuper(File lookupDir, long cacheFileBytes) { @@ -447,6 +449,14 @@ public byte[] lookup(byte[] key, LookupContext context) throws Exception { return key; } + @Override + public void refresh() { + if (closed) { + throw new IllegalStateException("Lookuper is already closed."); + } + refreshCount++; + } + @Override public void close() throws Exception { closed = true; diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java index b18f96b3a7a..2de12ea77e9 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java @@ -25,6 +25,7 @@ import org.apache.fluss.exception.HistoricalPartitionThrottledException; import org.apache.fluss.exception.InvalidPartitionException; import org.apache.fluss.lake.lakestorage.LakeTableLookuper; +import org.apache.fluss.memory.MemorySegment; import org.apache.fluss.metadata.ChangelogImage; import org.apache.fluss.metadata.DataLakeFormat; import org.apache.fluss.metadata.PhysicalTablePath; @@ -50,16 +51,21 @@ import org.apache.fluss.rpc.entity.FetchLogResultForBucket; import org.apache.fluss.rpc.entity.LookupResultForBucket; import org.apache.fluss.rpc.entity.PutKvResultForBucket; +import org.apache.fluss.rpc.messages.NotifyLakeTableOffsetResponse; import org.apache.fluss.rpc.protocol.ApiKeys; import org.apache.fluss.rpc.protocol.Errors; import org.apache.fluss.rpc.protocol.MergeMode; import org.apache.fluss.server.entity.FetchReqInfo; +import org.apache.fluss.server.entity.LakeBucketOffset; import org.apache.fluss.server.entity.LookupDataForBucket; +import org.apache.fluss.server.entity.NotifyLakeTableOffsetData; import org.apache.fluss.server.entity.NotifyLeaderAndIsrData; import org.apache.fluss.server.entity.NotifyLeaderAndIsrResultForBucket; import org.apache.fluss.server.entity.PutKvDataForBucket; import org.apache.fluss.server.kv.KvStateLookupResult; import org.apache.fluss.server.kv.KvTablet; +import org.apache.fluss.server.kv.historical.HistoricalKvKeyEncoder; +import org.apache.fluss.server.kv.historical.HistoricalKvTombstone; import org.apache.fluss.server.log.FetchParams; import org.apache.fluss.server.log.LogAppendInfo; import org.apache.fluss.server.metadata.BucketMetadata; @@ -83,9 +89,11 @@ import com.github.benmanes.caffeine.cache.Scheduler; import com.github.benmanes.caffeine.cache.Ticker; import org.junit.jupiter.api.Test; +import org.rocksdb.FlushOptions; import javax.annotation.Nullable; +import java.io.IOException; import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; @@ -144,6 +152,7 @@ void testResolvesMultipleLakeMissesWithoutPrewriteRollback() throws Exception { CompactedKeyEncoder keyEncoder = new CompactedKeyEncoder(keyType); byte[] firstKey = keyEncoder.encodeKey(row(1, "us")); byte[] secondKey = keyEncoder.encodeKey(row(2, "eu")); + byte[] thirdKey = keyEncoder.encodeKey(row(3, "ap")); KvRecordBatch insertBatch = batch( keyType, @@ -158,7 +167,13 @@ void testResolvesMultipleLakeMissesWithoutPrewriteRollback() throws Exception { new Object[] {1, "us", ORIGINAL_PARTITION, "v1-updated"}), Tuple2.of( new Object[] {2, "eu"}, - new Object[] {2, "eu", ORIGINAL_PARTITION, "v2"})); + new Object[] {2, "eu", ORIGINAL_PARTITION, "v2"}), + // The absent delete is a no-op. The following upsert must reuse the + // resolved absence instead of treating this key as already staged. + Tuple2.of(new Object[] {3, "ap"}, null), + Tuple2.of( + new Object[] {3, "ap"}, + new Object[] {3, "ap", ORIGINAL_PARTITION, "v3"})); long truncateCount = replicaManager.getServerMetricGroup().kvTruncateAsErrorCount().getCount(); @@ -170,7 +185,7 @@ void testResolvesMultipleLakeMissesWithoutPrewriteRollback() throws Exception { MergeMode.DEFAULT, 1); - assertThat(lakeLookupManager.lookupCount).hasValue(2); + assertThat(lakeLookupManager.lookupCount).hasValue(3); assertThat(lakeLookupManager.lookupBatchCount).hasValue(1); assertThat(replicaManager.getServerMetricGroup().kvTruncateAsErrorCount().getCount()) .isEqualTo(truncateCount); @@ -187,6 +202,118 @@ void testResolvesMultipleLakeMissesWithoutPrewriteRollback() throws Exception { secondKey, tableInfo, row(2, "eu", ORIGINAL_PARTITION, "v2")); + assertHistoricalValue( + kvTablet, + ORIGINAL_PARTITION, + thirdKey, + tableInfo, + row(3, "ap", ORIGINAL_PARTITION, "v3")); + } finally { + historicalPartitionManager.close(); + } + } + + @Test + void testReusesLocalProbeResultsWhenEntriesDisappearBeforeApply() throws Exception { + TableInfo tableInfo = registerHistoricalTableAndBecomeLeader(); + Replica replica = replicaManager.getReplicaOrException(TABLE_BUCKET); + KvTablet kvTablet = replica.getKvTablet(); + assertThat(kvTablet).isNotNull(); + TestingHistoricalLakeLookupManager lakeLookupManager = + new TestingHistoricalLakeLookupManager(lookupConfiguration()); + HistoricalPartitionManager historicalPartitionManager = + new HistoricalPartitionManager( + new HistoricalPartitionTaskExecutor(lookupConfiguration()), + lakeLookupManager); + + RowType keyType = + DataTypes.ROW( + new DataField("id", DataTypes.INT()), + new DataField("region", DataTypes.STRING())); + RowType rowType = tableInfo.getRowType(); + CompactedKeyEncoder keyEncoder = new CompactedKeyEncoder(keyType); + byte[] valueKey = keyEncoder.encodeKey(row(1, "us")); + byte[] tombstoneKey = keyEncoder.encodeKey(row(2, "eu")); + byte[] missingKey = keyEncoder.encodeKey(row(3, "ap")); + + try { + // Seed exactly the two local states that compaction can remove between phases. + kvTablet.getRocksDBKv() + .put( + HistoricalKvKeyEncoder.encode(ORIGINAL_PARTITION, valueKey), + ValueEncoder.forLayout(KvValueLayout.TAGGED) + .encodeValue( + new BinaryValue( + (short) tableInfo.getSchemaId(), + compactedRow( + rowType, + new Object[] { + 1, "us", ORIGINAL_PARTITION, "v1" + })), + 0L)); + kvTablet.getRocksDBKv() + .put( + HistoricalKvKeyEncoder.encode(ORIGINAL_PARTITION, tombstoneKey), + HistoricalKvTombstone.encode(1L)); + + // The third key forces lake I/O after all three keys have been probed. Remove the + // value and tombstone during that I/O to model compaction between probe and apply. + lakeLookupManager.setLookupHook( + () -> { + deleteHistoricalRocksDbKey(kvTablet, valueKey); + deleteHistoricalRocksDbKey(kvTablet, tombstoneKey); + }); + + LogAppendInfo appendInfo = + historicalPartitionManager.processPut( + replica, + new PutKvDataForBucket( + TABLE_BUCKET, + batch( + keyType, + rowType, + Tuple2.of( + new Object[] {1, "us"}, + new Object[] { + 1, "us", ORIGINAL_PARTITION, "v2" + }), + Tuple2.of( + new Object[] {2, "eu"}, + new Object[] { + 2, "eu", ORIGINAL_PARTITION, "recreated" + }), + Tuple2.of( + new Object[] {3, "ap"}, + new Object[] { + 3, "ap", ORIGINAL_PARTITION, "inserted" + })), + ORIGINAL_PARTITION), + null, + MergeMode.DEFAULT, + 1); + + assertThat(appendInfo.numMessages()).isEqualTo(4); + assertThat(lakeLookupManager.lookupCount).hasValue(1); + assertThat(lakeLookupManager.lookupBatchCount).hasValue(1); + flushAndWait(kvTablet, Long.MAX_VALUE); + assertHistoricalValue( + kvTablet, + ORIGINAL_PARTITION, + valueKey, + tableInfo, + row(1, "us", ORIGINAL_PARTITION, "v2")); + assertHistoricalValue( + kvTablet, + ORIGINAL_PARTITION, + tombstoneKey, + tableInfo, + row(2, "eu", ORIGINAL_PARTITION, "recreated")); + assertHistoricalValue( + kvTablet, + ORIGINAL_PARTITION, + missingKey, + tableInfo, + row(3, "ap", ORIGINAL_PARTITION, "inserted")); } finally { historicalPartitionManager.close(); } @@ -430,6 +557,221 @@ void testHistoricalInsertUpdateAndDelete() throws Exception { } } + @Test + void testHistoricalMutationsUseProducingWalOffsetsAsTags() throws Exception { + TableInfo tableInfo = registerHistoricalTableAndBecomeLeader(); + Replica replica = replicaManager.getReplicaOrException(TABLE_BUCKET); + KvTablet kvTablet = replica.getKvTablet(); + assertThat(kvTablet).isNotNull(); + HistoricalPartitionManager historicalPartitionManager = + new HistoricalPartitionManager( + new HistoricalPartitionTaskExecutor(lookupConfiguration()), + new TestingHistoricalLakeLookupManager(lookupConfiguration())); + RowType keyType = + DataTypes.ROW( + new DataField("id", DataTypes.INT()), + new DataField("region", DataTypes.STRING())); + RowType rowType = tableInfo.getRowType(); + byte[] primaryKey = new CompactedKeyEncoder(keyType).encodeKey(row(1, "us")); + + try { + LogAppendInfo insertAppend = + historicalPartitionManager.processPut( + replica, + new PutKvDataForBucket( + TABLE_BUCKET, + batch( + keyType, + rowType, + Tuple2.of( + new Object[] {1, "us"}, + new Object[] { + 1, "us", ORIGINAL_PARTITION, "v1" + })), + ORIGINAL_PARTITION), + null, + MergeMode.DEFAULT, + 1); + flushAndWait(kvTablet, Long.MAX_VALUE); + assertHistoricalValueTag( + kvTablet, ORIGINAL_PARTITION, primaryKey, insertAppend.lastOffset()); + + LogAppendInfo updateAppend = + historicalPartitionManager.processPut( + replica, + new PutKvDataForBucket( + TABLE_BUCKET, + batch( + keyType, + rowType, + Tuple2.of( + new Object[] {1, "us"}, + new Object[] { + 1, "us", ORIGINAL_PARTITION, "v2" + })), + ORIGINAL_PARTITION), + null, + MergeMode.DEFAULT, + 1); + assertThat(updateAppend.numMessages()).isEqualTo(2); + flushAndWait(kvTablet, Long.MAX_VALUE); + // In FULL changelog mode, UPDATE_AFTER is the second WAL record and produces state. + assertHistoricalValueTag( + kvTablet, ORIGINAL_PARTITION, primaryKey, updateAppend.lastOffset()); + + LogAppendInfo deleteAppend = + historicalPartitionManager.processPut( + replica, + new PutKvDataForBucket( + TABLE_BUCKET, + batch( + keyType, + rowType, + Tuple2.of(new Object[] {1, "us"}, null)), + ORIGINAL_PARTITION), + null, + MergeMode.DEFAULT, + 1); + flushAndWait(kvTablet, Long.MAX_VALUE); + byte[] tombstone = historicalRawValue(kvTablet, ORIGINAL_PARTITION, primaryKey); + assertThat(tombstone).hasSize(Long.BYTES); + assertHistoricalValueTag( + kvTablet, ORIGINAL_PARTITION, primaryKey, deleteAppend.lastOffset()); + } finally { + historicalPartitionManager.close(); + } + } + + @Test + void testCompactionUsesPublishedLakeSnapshot() throws Exception { + TableInfo tableInfo = registerHistoricalTableAndBecomeLeader(); + Replica replica = replicaManager.getReplicaOrException(TABLE_BUCKET); + KvTablet kvTablet = replica.getKvTablet(); + assertThat(kvTablet).isNotNull(); + TestingHistoricalLakeLookupManager lakeLookupManager = + new TestingHistoricalLakeLookupManager(lookupConfiguration()); + HistoricalPartitionManager historicalPartitionManager = + new HistoricalPartitionManager( + new HistoricalPartitionTaskExecutor(lookupConfiguration()), + lakeLookupManager); + RowType keyType = + DataTypes.ROW( + new DataField("id", DataTypes.INT()), + new DataField("region", DataTypes.STRING())); + byte[] coveredKey = new CompactedKeyEncoder(keyType).encodeKey(row(1, "us")); + byte[] retainedKey = new CompactedKeyEncoder(keyType).encodeKey(row(2, "eu")); + + try { + historicalPartitionManager.processPut( + replica, + new PutKvDataForBucket( + TABLE_BUCKET, + batch( + keyType, + tableInfo.getRowType(), + Tuple2.of( + new Object[] {1, "us"}, + new Object[] {1, "us", ORIGINAL_PARTITION, "covered"})), + ORIGINAL_PARTITION), + null, + MergeMode.DEFAULT, + 1); + historicalPartitionManager.processPut( + replica, + new PutKvDataForBucket( + TABLE_BUCKET, + batch( + keyType, + tableInfo.getRowType(), + Tuple2.of( + new Object[] {2, "eu"}, + new Object[] { + 2, "eu", ORIGINAL_PARTITION, "retained" + })), + ORIGINAL_PARTITION), + null, + MergeMode.DEFAULT, + 1); + flushAndWait(kvTablet, Long.MAX_VALUE); + assertHistoricalValueTag(kvTablet, ORIGINAL_PARTITION, coveredKey, 0L); + assertHistoricalValueTag(kvTablet, ORIGINAL_PARTITION, retainedKey, 1L); + + lakeLookupManager.setRequireSnapshotHook( + () -> assertThat(kvTablet.getHistoricalCleanupOffset()).isZero()); + historicalPartitionManager.onLakeProgress(replica, 7L, 1L); + + assertThat(lakeLookupManager.requiredSnapshotCount).hasValue(1); + assertThat(kvTablet.getHistoricalCleanupOffset()).isOne(); + compactHistoricalKv(kvTablet); + assertThat(historicalRawValue(kvTablet, ORIGINAL_PARTITION, coveredKey)).isNull(); + assertHistoricalValueTag(kvTablet, ORIGINAL_PARTITION, retainedKey, 1L); + + // A miss caused by cleanup must enter lake lookup only after the covering snapshot + // token has been published. + lakeLookupManager.putLakeValue( + ORIGINAL_PARTITION, + ValueEncoder.encodeValue( + (short) tableInfo.getSchemaId(), + compactedRow( + tableInfo.getRowType(), + new Object[] { + 1, "us", ORIGINAL_PARTITION, "covered-from-lake" + }))); + lakeLookupManager.setLookupHook( + () -> { + assertThat(lakeLookupManager.requiredSnapshotCount).hasValue(1); + assertThat(kvTablet.getHistoricalCleanupOffset()).isOne(); + }); + LookupResultForBucket fallbackResult = + historicalPartitionManager + .lookup( + replica, + new LookupDataForBucket( + TABLE_BUCKET, + Collections.singletonList(coveredKey), + ORIGINAL_PARTITION), + (lookupTimeNanos, lookupFileDownloaded) -> {}) + .get(10, TimeUnit.SECONDS); + assertThat(fallbackResult.failed()).isFalse(); + BinaryValue fallbackValue = + new ValueDecoder( + schemaGetter(tableInfo), + tableInfo.getTableConfig().getKvFormat(), + KvValueLayout.PLAIN) + .decodeValue(fallbackResult.lookupValues().get(0).toByteArray()); + assertThat(fallbackValue.row.getString(3)) + .isEqualTo(BinaryString.fromString("covered-from-lake")); + + // A new opaque snapshot token at the same offset is still published to the lookuper. + lakeLookupManager.setRequireSnapshotHook( + () -> assertThat(kvTablet.getHistoricalCleanupOffset()).isOne()); + historicalPartitionManager.onLakeProgress(replica, 8L, 1L); + assertThat(lakeLookupManager.requiredSnapshotCount).hasValue(2); + assertThat(kvTablet.getHistoricalCleanupOffset()).isOne(); + + // A regressing cleanup offset is ignored before it can publish an older required + // snapshot. + historicalPartitionManager.onLakeProgress(replica, 9L, 0L); + assertThat(lakeLookupManager.requiredSnapshotCount).hasValue(2); + assertThat(kvTablet.getHistoricalCleanupOffset()).isOne(); + + CompletableFuture notifyFuture = + new CompletableFuture<>(); + replicaManager.notifyLakeTableOffset( + new NotifyLakeTableOffsetData( + 1, + Collections.singletonMap( + TABLE_BUCKET, new LakeBucketOffset(10L, null, 2L, null))), + notifyFuture::complete); + notifyFuture.get(10, TimeUnit.SECONDS); + assertThat(kvTablet.getHistoricalCleanupOffset()).isEqualTo(2L); + compactHistoricalKv(kvTablet); + assertThat(historicalRawValue(kvTablet, ORIGINAL_PARTITION, retainedKey)).isNull(); + } finally { + historicalPartitionManager.close(); + } + } + @Test void testUpdateAndDeleteFromLakeFallback() throws Exception { TableInfo tableInfo = registerHistoricalTableAndBecomeLeader(); @@ -628,22 +970,26 @@ void testRecoversHistoricalOverlayFromLakeCommitOffset() throws Exception { null, MergeMode.DEFAULT, 1); - historicalPartitionManager.processPut( - replica, - new PutKvDataForBucket( - TABLE_BUCKET, - batch( - keyType, - tableInfo.getRowType(), - Tuple2.of( - new Object[] {1, "us"}, - new Object[] { - 1, "us", ANOTHER_ORIGINAL_PARTITION, "another" - })), - ANOTHER_ORIGINAL_PARTITION), - null, - MergeMode.DEFAULT, - 1); + LogAppendInfo anotherPartitionAppend = + historicalPartitionManager.processPut( + replica, + new PutKvDataForBucket( + TABLE_BUCKET, + batch( + keyType, + tableInfo.getRowType(), + Tuple2.of( + new Object[] {1, "us"}, + new Object[] { + 1, + "us", + ANOTHER_ORIGINAL_PARTITION, + "another" + })), + ANOTHER_ORIGINAL_PARTITION), + null, + MergeMode.DEFAULT, + 1); historicalPartitionManager.processPut( replica, new PutKvDataForBucket( @@ -660,18 +1006,19 @@ void testRecoversHistoricalOverlayFromLakeCommitOffset() throws Exception { null, MergeMode.DEFAULT, 1); - historicalPartitionManager.processPut( - replica, - new PutKvDataForBucket( - TABLE_BUCKET, - batch( - keyType, - tableInfo.getRowType(), - Tuple2.of(new Object[] {2, "eu"}, null)), - ORIGINAL_PARTITION), - null, - MergeMode.DEFAULT, - 1); + LogAppendInfo deleteAppend = + historicalPartitionManager.processPut( + replica, + new PutKvDataForBucket( + TABLE_BUCKET, + batch( + keyType, + tableInfo.getRowType(), + Tuple2.of(new Object[] {2, "eu"}, null)), + ORIGINAL_PARTITION), + null, + MergeMode.DEFAULT, + 1); KvTablet kvTabletBeforeFollower = replica.getKvTablet(); assertThat(kvTabletBeforeFollower).isNotNull(); flushAndWait(kvTabletBeforeFollower, Long.MAX_VALUE); @@ -704,6 +1051,10 @@ void testRecoversHistoricalOverlayFromLakeCommitOffset() throws Exception { KvTablet recoveredKvTablet = replica.getKvTablet(); assertThat(recoveredKvTablet).isNotNull(); assertThat(replica.getLakeLogEndOffset()).isEqualTo(lakeCommitOffset); + assertThat(recoveredKvTablet.getHistoricalCleanupOffset()).isEqualTo(lakeCommitOffset); + // Recovery retries can reuse the same tablet and restore offset. + assertThat(recoveredKvTablet.advanceHistoricalCleanupOffset(lakeCommitOffset)) + .isFalse(); assertThat(replica.getKvSnapshotManager()).isNull(); assertThat(recoveredKvTablet.getFlushedLogOffset()) .isEqualTo(replica.getLogHighWatermark()); @@ -723,6 +1074,16 @@ void testRecoversHistoricalOverlayFromLakeCommitOffset() throws Exception { tieredPrimaryKey, tableInfo, row(1, "us", ANOTHER_ORIGINAL_PARTITION, "another")); + assertHistoricalValueTag( + recoveredKvTablet, + ANOTHER_ORIGINAL_PARTITION, + tieredPrimaryKey, + anotherPartitionAppend.lastOffset()); + assertHistoricalValueTag( + recoveredKvTablet, + ORIGINAL_PARTITION, + deletedPrimaryKey, + deleteAppend.lastOffset()); } finally { historicalPartitionManager.close(); } @@ -957,11 +1318,42 @@ private static void assertHistoricalValue( new ValueDecoder( schemaGetter(tableInfo), tableInfo.getTableConfig().getKvFormat(), - KvValueLayout.fromTableConfig(tableInfo.getTableConfig())) + KvValueLayout.TAGGED) .decodeValue(result.value()); assertThatRow(value.row).withSchema(tableInfo.getRowType()).isEqualTo(expectedRow); } + private static void assertHistoricalValueTag( + KvTablet kvTablet, String originalPartition, byte[] primaryKey, long expectedLogOffset) + throws IOException { + byte[] rawValue = historicalRawValue(kvTablet, originalPartition, primaryKey); + assertThat(rawValue).isNotNull(); + assertThat(KvValueLayout.TAGGED.readValueTag(MemorySegment.wrap(rawValue))) + .isEqualTo(expectedLogOffset); + } + + private static byte[] historicalRawValue( + KvTablet kvTablet, String originalPartition, byte[] primaryKey) throws IOException { + return kvTablet.getRocksDBKv() + .get(HistoricalKvKeyEncoder.encode(originalPartition, primaryKey)); + } + + private static void compactHistoricalKv(KvTablet kvTablet) throws Exception { + try (FlushOptions flushOptions = new FlushOptions().setWaitForFlush(true)) { + kvTablet.getRocksDBKv().getDb().flush(flushOptions); + kvTablet.getRocksDBKv().getDb().compactRange(); + } + } + + private static void deleteHistoricalRocksDbKey(KvTablet kvTablet, byte[] primaryKey) { + try { + kvTablet.getRocksDBKv() + .delete(HistoricalKvKeyEncoder.encode(ORIGINAL_PARTITION, primaryKey)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + private static SchemaGetter schemaGetter(TableInfo tableInfo) { return new TestingSchemaGetter( new SchemaInfo(tableInfo.getSchema(), tableInfo.getSchemaId())); @@ -970,8 +1362,10 @@ private static SchemaGetter schemaGetter(TableInfo tableInfo) { private final class TestingHistoricalLakeLookupManager extends HistoricalLakeLookupManager { private final AtomicInteger lookupCount = new AtomicInteger(); private final AtomicInteger lookupBatchCount = new AtomicInteger(); + private final AtomicInteger requiredSnapshotCount = new AtomicInteger(); private final Map lakeValuesByPartition = new HashMap<>(); private volatile @Nullable Runnable lookupHook; + private volatile @Nullable Runnable requireSnapshotHook; private TestingHistoricalLakeLookupManager(Configuration configuration) { super( @@ -992,6 +1386,20 @@ private void setLookupHook(Runnable lookupHook) { this.lookupHook = lookupHook; } + private void setRequireSnapshotHook(Runnable requireSnapshotHook) { + this.requireSnapshotHook = requireSnapshotHook; + } + + @Override + void requireLakeSnapshot(long tableId, long snapshotId) { + super.requireLakeSnapshot(tableId, snapshotId); + requiredSnapshotCount.incrementAndGet(); + Runnable hook = requireSnapshotHook; + if (hook != null) { + hook.run(); + } + } + @Override List lookup( LookupDataForBucket lookupData, diff --git a/fluss-server/src/test/java/org/apache/fluss/server/utils/HistoricalPartitionTableValidationTest.java b/fluss-server/src/test/java/org/apache/fluss/server/utils/HistoricalPartitionTableValidationTest.java index cb427c23ec8..9b6aa6913ea 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/utils/HistoricalPartitionTableValidationTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/utils/HistoricalPartitionTableValidationTest.java @@ -54,7 +54,6 @@ void testReportsAllUnmetHistoricalPartitionRequirements() { + "'table.datalake.enabled' must be set to true; " + "'table.datalake.format' must be set to 'paimon' " + "(currently not set); " - + "the table must define a primary key; " + "the table must define exactly one partition key (found 0)."); // Case 2: Aggregate requirements before related validators can report only one failure. @@ -80,7 +79,6 @@ void testReportsAllUnmetHistoricalPartitionRequirements() { "'table.datalake.historical-partition.enabled' has unmet requirements: " + "'table.datalake.format' must be set to 'paimon' " + "(currently 'iceberg'); " - + "the table must define a primary key; " + "the table must define exactly one partition key (found 0)."); } } diff --git a/fluss-server/src/test/java/org/apache/fluss/server/utils/ServerRpcMessageUtilsTest.java b/fluss-server/src/test/java/org/apache/fluss/server/utils/ServerRpcMessageUtilsTest.java index 9f318eded4c..c5ed0c48a6b 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/utils/ServerRpcMessageUtilsTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/utils/ServerRpcMessageUtilsTest.java @@ -21,12 +21,18 @@ import org.apache.fluss.record.KvRecordBatch; import org.apache.fluss.row.encode.KvValueLayout; import org.apache.fluss.rpc.entity.LookupResultForBucket; +import org.apache.fluss.rpc.entity.ProduceLogResultForBucket; import org.apache.fluss.rpc.entity.PutKvResultForBucket; import org.apache.fluss.rpc.messages.LookupResponse; +import org.apache.fluss.rpc.messages.PbProduceLogReqForBucket; +import org.apache.fluss.rpc.messages.PbProduceLogRespForBucket; import org.apache.fluss.rpc.messages.PbPutKvReqForBucket; import org.apache.fluss.rpc.messages.PbPutKvRespForBucket; +import org.apache.fluss.rpc.messages.ProduceLogRequest; +import org.apache.fluss.rpc.messages.ProduceLogResponse; import org.apache.fluss.rpc.messages.PutKvRequest; import org.apache.fluss.rpc.messages.PutKvResponse; +import org.apache.fluss.server.entity.ProduceLogDataForBucket; import org.apache.fluss.server.entity.PutKvDataForBucket; import org.junit.jupiter.api.Test; @@ -64,6 +70,52 @@ void testLookupResponseSupportsMixedValueLayouts() { .containsExactly(new byte[] {1, 0, 11, 12}, new byte[] {1, 0, 21, 22}); } + @Test + void testHistoricalProduceLogRequestAndResponsePreserveOriginalPartitions() { + long tableId = 1L; + long partitionId = 2L; + PbProduceLogReqForBucket firstBucketRequest = + new PbProduceLogReqForBucket() + .setPartitionId(partitionId) + .setBucketId(0) + .setRecords(new byte[0]) + .setOriginalPartitionName("dt=2025-01-01"); + ProduceLogRequest request = + new ProduceLogRequest().setTableId(tableId).setAcks(1).setTimeoutMs(10_000); + request.addAllBucketsReqs( + Arrays.asList( + firstBucketRequest, + new PbProduceLogReqForBucket() + .copyFrom(firstBucketRequest) + .setOriginalPartitionName("dt=2025-01-02"))); + + TableBucket tableBucket = new TableBucket(tableId, partitionId, 0); + List decoded = + ServerRpcMessageUtils.toProduceLogDataForBuckets(request); + assertThat(decoded) + .extracting(ProduceLogDataForBucket::tableBucket) + .containsOnly(tableBucket); + assertThat(decoded) + .extracting(ProduceLogDataForBucket::originalPartitionName) + .containsExactly("dt=2025-01-01", "dt=2025-01-02"); + + ProduceLogResponse response = + ServerRpcMessageUtils.makeProduceLogResponse( + Arrays.asList( + ProduceLogResultForBucket.historicalSuccess( + tableBucket, 0L, 1L, "dt=2025-01-01"), + ProduceLogResultForBucket.historicalSuccess( + tableBucket, 1L, 2L, "dt=2025-01-02"))); + assertThat(response.getBucketsRespsList()) + .extracting(PbProduceLogRespForBucket::getOriginalPartitionName) + .containsExactly("dt=2025-01-01", "dt=2025-01-02"); + + request.addAllBucketsReqs(Collections.singletonList(firstBucketRequest)); + assertThatThrownBy(() -> ServerRpcMessageUtils.toProduceLogDataForBuckets(request)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("duplicate table bucket"); + } + @Test void testHistoricalPutKvRequestAndResponsePreserveOriginalPartitions() throws Exception { long tableId = 1L; diff --git a/fluss-test-coverage/pom.xml b/fluss-test-coverage/pom.xml index d8428c4a013..226a89eeea6 100644 --- a/fluss-test-coverage/pom.xml +++ b/fluss-test-coverage/pom.xml @@ -491,6 +491,9 @@ org.apache.fluss.lake.source.* org.apache.fluss.lake.lakestorage.LakeStorage + + org.apache.fluss.lake.lakestorage.LakeTableLookuper + org.apache.fluss.lake.lakestorage.LakeStorage.LookuperContext