From 466e462e0e03d1508d752e743f210756376c4846 Mon Sep 17 00:00:00 2001 From: luoyuxia Date: Thu, 27 Aug 2026 07:19:58 +0800 Subject: [PATCH 1/4] [client][server][paimon] Support historical partition writes Route writes for expired partitions through internal historical targets while preserving original partition metadata across PUT_KV and PRODUCE_LOG. Tier historical KV and log records back to their original Paimon partitions, fail writes to confirmed missing targets, and safely clean fully tiered historical KV overlays with leader-epoch and offset guards. Co-Authored-By: Codex AI-Model: gpt-5 AI-Contributed/Feature: 1111/1469 AI-Contributed/UT: 1829/2215 --- .../client/utils/ClientRpcMessageUtils.java | 9 + .../write/AbstractRowLogWriteBatch.java | 12 +- .../client/write/ArrowLogWriteBatch.java | 10 +- .../client/write/CompactedLogWriteBatch.java | 3 + .../client/write/IndexedLogWriteBatch.java | 3 + .../fluss/client/write/KvWriteBatch.java | 2 + .../fluss/client/write/RecordAccumulator.java | 128 ++++- .../org/apache/fluss/client/write/Sender.java | 189 ++++++-- .../apache/fluss/client/write/WriteBatch.java | 15 + .../fluss/client/write/WriterClient.java | 81 +++- .../utils/ClientRpcMessageUtilsTest.java | 1 + .../client/write/ArrowLogWriteBatchTest.java | 3 + .../write/CompactedLogWriteBatchTest.java | 1 + .../write/IndexedLogWriteBatchTest.java | 1 + .../fluss/client/write/KvWriteBatchTest.java | 2 + .../client/write/RecordAccumulatorTest.java | 88 ++++ .../apache/fluss/client/write/SenderTest.java | 453 +++++++++++++++++- .../apache/fluss/config/ConfigOptions.java | 17 +- .../org/apache/fluss/config/TableConfig.java | 2 +- .../enumerator/TieringSourceEnumerator.java | 2 +- .../source/split/TieringSplitGenerator.java | 44 +- .../paimon/tiering/PaimonLakeCommitter.java | 5 +- .../lake/paimon/tiering/PaimonLakeWriter.java | 14 +- .../paimon/tiering/PaimonWriteResult.java | 22 +- .../tiering/PaimonWriteResultSerializer.java | 17 +- .../lake/paimon/tiering/RecordWriter.java | 40 +- .../append/AppendOnlyArrowBatchHelper.java | 30 +- .../tiering/append/AppendOnlyWriter.java | 13 +- .../tiering/mergetree/MergeTreeWriter.java | 20 +- ...se.java => HistoricalPartitionITCase.java} | 190 +++++++- .../paimon/tiering/PaimonTieringTest.java | 175 +++++++ .../rpc/entity/ProduceLogResultForBucket.java | 49 +- .../rpc/netty/client/ServerConnection.java | 41 ++ .../apache/fluss/rpc/protocol/ApiKeys.java | 3 +- .../fluss/rpc/util/CommonRpcMessageUtils.java | 12 + fluss-rpc/src/main/proto/FlussApi.proto | 4 + .../netty/client/ServerConnectionTest.java | 118 ++++- fluss-rust/crates/fluss/proto/FlussApi.proto | 4 + fluss-rust/crates/fluss/src/proto/fluss.rs | 6 + fluss-rust/crates/fluss/src/rpc/api_key.rs | 3 +- .../fluss/src/rpc/message/produce_log.rs | 1 + .../crates/fluss/src/rpc/server_connection.rs | 4 +- .../fluss/server/DynamicServerConfig.java | 2 + .../HistoricalLookupCacheConfigValidator.java | 11 +- .../entity/ProduceLogDataForBucket.java | 51 ++ .../apache/fluss/server/replica/Replica.java | 105 +++- .../fluss/server/replica/ReplicaManager.java | 85 +++- .../HistoricalPartitionManager.java | 272 ++++++++++- .../HistoricalPartitionTaskExecutor.java | 64 ++- .../fluss/server/tablet/TabletService.java | 35 +- .../server/utils/ServerRpcMessageUtils.java | 35 +- .../utils/TableDescriptorValidation.java | 4 - .../fluss/server/DynamicConfigChangeTest.java | 31 ++ .../server/replica/ReplicaManagerTest.java | 27 ++ .../HistoricalPartitionManagerTest.java | 325 +++++++++++++ .../HistoricalPartitionTaskExecutorTest.java | 31 ++ ...istoricalPartitionTableValidationTest.java | 29 +- .../utils/ServerRpcMessageUtilsTest.java | 52 ++ 58 files changed, 2814 insertions(+), 182 deletions(-) rename fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/{HistoricalPartitionLookupITCase.java => HistoricalPartitionITCase.java} (64%) create mode 100644 fluss-server/src/main/java/org/apache/fluss/server/entity/ProduceLogDataForBucket.java diff --git a/fluss-client/src/main/java/org/apache/fluss/client/utils/ClientRpcMessageUtils.java b/fluss-client/src/main/java/org/apache/fluss/client/utils/ClientRpcMessageUtils.java index 3c7512945dc..1a40afc92be 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/utils/ClientRpcMessageUtils.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/utils/ClientRpcMessageUtils.java @@ -147,6 +147,10 @@ public static ProduceLogRequest makeProduceLogRequest( if (tableBucket.getPartitionId() != null) { pbProduceLogReqForBucket.setPartitionId(tableBucket.getPartitionId()); } + if (readyBatch.writeBatch().getOriginalPartitionName() != null) { + pbProduceLogReqForBucket.setOriginalPartitionName( + readyBatch.writeBatch().getOriginalPartitionName()); + } }); return request; } @@ -202,6 +206,11 @@ public static PutKvRequest makePutKvRequest( if (tableBucket.getPartitionId() != null) { pbPutKvReqForBucket.setPartitionId(tableBucket.getPartitionId()); } + KvWriteBatch kvWriteBatch = (KvWriteBatch) readyBatch.writeBatch(); + if (kvWriteBatch.getOriginalPartitionName() != null) { + pbPutKvReqForBucket.setOriginalPartitionName( + kvWriteBatch.getOriginalPartitionName()); + } }); return request; } diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/AbstractRowLogWriteBatch.java b/fluss-client/src/main/java/org/apache/fluss/client/write/AbstractRowLogWriteBatch.java index 104f8f29e33..d9afe697600 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/AbstractRowLogWriteBatch.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/AbstractRowLogWriteBatch.java @@ -26,6 +26,8 @@ import org.apache.fluss.record.bytesview.BytesView; import org.apache.fluss.row.InternalRow; +import javax.annotation.Nullable; + import java.io.IOException; import java.util.List; @@ -49,11 +51,19 @@ protected AbstractRowLogWriteBatch( PhysicalTablePath physicalTablePath, int schemaId, WriteFormat writeFormat, + @Nullable String originalPartitionName, long createdMs, AbstractPagedOutputView outputView, MemoryLogRecordsRowBuilder recordsBuilder, String buildErrorMessage) { - super(tableId, bucketId, physicalTablePath, schemaId, writeFormat, createdMs); + super( + tableId, + bucketId, + physicalTablePath, + schemaId, + writeFormat, + originalPartitionName, + createdMs); this.outputView = outputView; this.recordsBuilder = recordsBuilder; this.buildErrorMessage = buildErrorMessage; diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/ArrowLogWriteBatch.java b/fluss-client/src/main/java/org/apache/fluss/client/write/ArrowLogWriteBatch.java index a6894b98b2e..4c4f293aa8f 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/ArrowLogWriteBatch.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/ArrowLogWriteBatch.java @@ -58,9 +58,17 @@ public ArrowLogWriteBatch( int schemaId, ArrowWriter arrowWriter, AbstractPagedOutputView outputView, + @Nullable String originalPartitionName, long createdMs, @Nullable LogRecordBatchStatisticsCollector statisticsCollector) { - super(tableId, bucketId, physicalTablePath, schemaId, WriteFormat.ARROW_LOG, createdMs); + super( + tableId, + bucketId, + physicalTablePath, + schemaId, + WriteFormat.ARROW_LOG, + originalPartitionName, + createdMs); this.outputView = outputView; this.recordsBuilder = MemoryLogRecordsArrowBuilder.builder( diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/CompactedLogWriteBatch.java b/fluss-client/src/main/java/org/apache/fluss/client/write/CompactedLogWriteBatch.java index 81bedff0fa0..1e3fc0b8dbc 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/CompactedLogWriteBatch.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/CompactedLogWriteBatch.java @@ -25,6 +25,7 @@ import org.apache.fluss.row.compacted.CompactedRow; import org.apache.fluss.rpc.messages.ProduceLogRequest; +import javax.annotation.Nullable; import javax.annotation.concurrent.NotThreadSafe; import static org.apache.fluss.utils.Preconditions.checkArgument; @@ -46,6 +47,7 @@ public CompactedLogWriteBatch( int schemaId, int writeLimit, AbstractPagedOutputView outputView, + @Nullable String originalPartitionName, long createdMs) { super( tableId, @@ -53,6 +55,7 @@ public CompactedLogWriteBatch( physicalTablePath, schemaId, WriteFormat.COMPACTED_LOG, + originalPartitionName, createdMs, outputView, MemoryLogRecordsCompactedBuilder.builder(schemaId, writeLimit, outputView, true), diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/IndexedLogWriteBatch.java b/fluss-client/src/main/java/org/apache/fluss/client/write/IndexedLogWriteBatch.java index 2bb496cbfe3..2b50402218b 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/IndexedLogWriteBatch.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/IndexedLogWriteBatch.java @@ -24,6 +24,7 @@ import org.apache.fluss.row.indexed.IndexedRow; import org.apache.fluss.rpc.messages.ProduceLogRequest; +import javax.annotation.Nullable; import javax.annotation.concurrent.NotThreadSafe; import static org.apache.fluss.utils.Preconditions.checkArgument; @@ -45,6 +46,7 @@ public IndexedLogWriteBatch( int schemaId, int writeLimit, AbstractPagedOutputView outputView, + @Nullable String originalPartitionName, long createdMs) { super( tableId, @@ -52,6 +54,7 @@ public IndexedLogWriteBatch( physicalTablePath, schemaId, WriteFormat.INDEXED_LOG, + originalPartitionName, createdMs, outputView, MemoryLogRecordsIndexedBuilder.builder(schemaId, writeLimit, outputView, true), diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/KvWriteBatch.java b/fluss-client/src/main/java/org/apache/fluss/client/write/KvWriteBatch.java index 4ca01e133b2..0a5c16db688 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/KvWriteBatch.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/KvWriteBatch.java @@ -63,6 +63,7 @@ public KvWriteBatch( AbstractPagedOutputView outputView, @Nullable int[] targetColumns, MergeMode mergeMode, + @Nullable String originalPartitionName, long createdMs) { super( tableId, @@ -70,6 +71,7 @@ public KvWriteBatch( physicalTablePath, schemaId, WriteFormat.fromKvFormat(kvFormat), + originalPartitionName, createdMs); this.outputView = outputView; this.recordsBuilder = diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java b/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java index ffa18ad9455..7546e48b8e5 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java @@ -32,6 +32,7 @@ import org.apache.fluss.metadata.PhysicalTablePath; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; import org.apache.fluss.metrics.MetricNames; import org.apache.fluss.record.LogRecordBatchStatisticsCollector; import org.apache.fluss.row.arrow.ArrowWriter; @@ -67,6 +68,7 @@ import static org.apache.fluss.record.LogRecordBatchFormat.NO_BATCH_SEQUENCE; import static org.apache.fluss.record.LogRecordBatchFormat.NO_WRITER_ID; import static org.apache.fluss.shaded.arrow.org.apache.arrow.memory.BufferAllocatorUtil.createBufferAllocator; +import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; import static org.apache.fluss.utils.Preconditions.checkNotNull; /* This file is based on source code of Apache Kafka Project (https://kafka.apache.org/), licensed by the Apache @@ -115,6 +117,9 @@ public final class RecordAccumulator { private final ConcurrentMap writeBatches = new CopyOnWriteMap<>(); + /** Tables observed by this writer with historical partition support enabled. */ + private final Set historicalPartitionEnabledTables = ConcurrentHashMap.newKeySet(); + private final IncompleteBatches incomplete; private final Map nodesDrainIndex; @@ -198,6 +203,9 @@ public RecordAppendResult append( throws Exception { PhysicalTablePath physicalTablePath = writeRecord.getPhysicalTablePath(); TableInfo tableInfo = writeRecord.getTableInfo(); + if (tableInfo.getTableConfig().isHistoricalPartitionEnabled()) { + historicalPartitionEnabledTables.add(tableInfo.getTablePath()); + } // The metadata may return null for the partition id, but it is fine to pass null here, // because we will fill the partitionId in bucketReady() before send the batch. Optional partitionIdOpt = cluster.getPartitionId(physicalTablePath); @@ -206,7 +214,9 @@ public RecordAppendResult append( physicalTablePath, k -> new BucketAndWriteBatches( - partitionIdOpt.orElse(null), tableInfo.isPartitioned())); + partitionIdOpt.orElse(null), + tableInfo.isPartitioned(), + physicalTablePath)); // We keep track of the number of appending thread to make sure we do not miss batches in // abortIncompleteBatches(). @@ -331,6 +341,49 @@ public void reEnqueue(ReadyWriteBatch readyWriteBatch) { } } + /** + * Tries to route writes for an original partition path to the given physical target. + * + *

The accumulator keeps queues keyed by {@code originalPath}, while metadata lookup, leader + * discovery, and RPC sending use {@code targetPath}. The target may therefore be either the + * original partition itself or the shared historical partition. + * + *

The first queue creation fixes the target for that original path. A later call succeeds + * only when it selects the same target; this method never moves queued or inflight batches + * between physical partitions. + * + * @return true if the target was installed or already matches, false if a different target was + * fixed previously + */ + boolean tryRouteWritesTo( + PhysicalTablePath originalPath, PhysicalTablePath targetPath, long targetPartitionId) { + BucketAndWriteBatches resolvedTarget = + new BucketAndWriteBatches(targetPartitionId, true, targetPath); + // Install the route atomically before append can create the first queue for this path. + BucketAndWriteBatches existing = writeBatches.putIfAbsent(originalPath, resolvedTarget); + if (existing == null) { + return true; + } + + // An append may already have fixed this path to a target. Keep that target and only accept + // the metadata result when it describes the same physical partition. + if (!existing.targetPath.equals(targetPath)) { + return false; + } + existing.partitionId = targetPartitionId; + return true; + } + + /** Returns whether a write target has already been chosen for this original path. */ + boolean hasWriteTarget(PhysicalTablePath originalPath) { + return writeBatches.containsKey(originalPath); + } + + /** Returns whether the target belongs to a table with historical partition support enabled. */ + boolean isHistoricalPartitionEnabled(PhysicalTablePath targetPath) { + return historicalPartitionEnabledTables.contains(targetPath.getTablePath()); + } + /** Abort all incomplete batches (whether they have been sent or not). */ public void abortAllBatches(final Exception reason) { for (WriteBatch batch : incomplete.copyAll()) { @@ -357,7 +410,8 @@ private Deque getOrCreateDeque( k -> new BucketAndWriteBatches( tableBucket.getPartitionId(), - physicalTablePath.getPartitionName() != null)); + physicalTablePath.getPartitionName() != null, + physicalTablePath)); return bucketAndWriteBatches.batches.computeIfAbsent( tableBucket.getBucket(), k -> new ArrayDeque<>()); } @@ -485,8 +539,9 @@ private long bucketReady( Cluster cluster, long nextReadyCheckDelayMs) { // first check this table has partitionId. + PhysicalTablePath targetPath = bucketAndWriteBatches.targetPath; if (bucketAndWriteBatches.isPartitionedTable && bucketAndWriteBatches.partitionId == null) { - Optional optionIdOpt = cluster.getPartitionId(physicalTablePath); + Optional optionIdOpt = cluster.getPartitionId(targetPath); if (optionIdOpt.isPresent()) { bucketAndWriteBatches.partitionId = optionIdOpt.get(); } else { @@ -495,7 +550,7 @@ private long bucketReady( physicalTablePath); // TODO: we shouldn't add unready partitions to unknownLeaderTables, // because it cases PartitionNotExistException later - unknownLeaderTables.add(physicalTablePath); + unknownLeaderTables.add(targetPath); return nextReadyCheckDelayMs; } } @@ -531,10 +586,10 @@ private long bucketReady( int bucketId = entry.getKey(); Optional tableIdOpt = cluster.getTableId(physicalTablePath.getTablePath()); if (!tableIdOpt.isPresent()) { - unknownLeaderTables.add(physicalTablePath); + unknownLeaderTables.add(targetPath); } else { TableBucket tableBucket = - cluster.getTableBucket(tableIdOpt.get(), physicalTablePath, bucketId); + cluster.getTableBucket(tableIdOpt.get(), targetPath, bucketId); // If this bucket is throttled, don't mark its node as ready. // Instead, factor the remaining throttle time into the next check delay. @@ -556,7 +611,7 @@ private long bucketReady( // This is a bucket for which leader is not known, but messages are // available to send. Note that entries are currently not removed from // batches when deque is empty. - unknownLeaderTables.add(physicalTablePath); + unknownLeaderTables.add(targetPath); } else { nextReadyCheckDelayMs = batchReady( @@ -627,6 +682,15 @@ private RecordAppendResult appendNewBatch( PreAllocatedPagedOutputView outputView = new PreAllocatedPagedOutputView(segments); int schemaId = tableInfo.getSchemaId(); WriteFormat writeFormat = writeRecord.getWriteFormat(); + BucketAndWriteBatches bucketAndWriteBatches = + checkNotNull( + writeBatches.get(physicalTablePath), + "Write batches for %s must exist.", + physicalTablePath); + String originalPartitionName = + bucketAndWriteBatches.isHistoricalWriteTarget() + ? checkNotNull(physicalTablePath.getPartitionName()) + : null; final WriteBatch batch = createWriteBatch( writeRecord, @@ -635,7 +699,8 @@ private RecordAppendResult appendNewBatch( writeFormat, physicalTablePath, outputView, - schemaId); + schemaId, + originalPartitionName); batch.tryAppend(writeRecord, callback); deque.addLast(batch); @@ -650,7 +715,8 @@ private WriteBatch createWriteBatch( WriteFormat writeFormat, PhysicalTablePath physicalTablePath, PreAllocatedPagedOutputView outputView, - int schemaId) { + int schemaId, + @Nullable String originalPartitionName) { // If the table is kv table we need to create a kv batch, otherwise we create a log batch. switch (writeFormat) { case COMPACTED_KV: @@ -665,6 +731,7 @@ private WriteBatch createWriteBatch( outputView, writeRecord.getTargetColumns(), writeRecord.getMergeMode(), + originalPartitionName, clock.milliseconds()); case ARROW_LOG: @@ -688,6 +755,7 @@ private WriteBatch createWriteBatch( tableInfo.getSchemaId(), arrowWriter, outputView, + originalPartitionName, clock.milliseconds(), statisticsCollector); @@ -699,6 +767,7 @@ private WriteBatch createWriteBatch( schemaId, outputView.getPreAllocatedSize(), outputView, + originalPartitionName, clock.milliseconds()); case INDEXED_LOG: @@ -709,6 +778,7 @@ private WriteBatch createWriteBatch( tableInfo.getSchemaId(), outputView.getPreAllocatedSize(), outputView, + originalPartitionName, clock.milliseconds()); default: @@ -1013,6 +1083,10 @@ private List getAllBucketsInCurrentNode(Integer currentNode, Clu List buckets = new ArrayList<>(); Set physicalTablePaths = cluster.getBucketLocationsByPath().keySet(); for (PhysicalTablePath path : physicalTablePaths) { + BucketAndWriteBatches bucketAndWriteBatches = writeBatches.get(path); + if (bucketAndWriteBatches != null && bucketAndWriteBatches.isHistoricalWriteTarget()) { + continue; + } List bucketsForTable = cluster.getAvailableBucketsForPhysicalTablePath(path); for (BucketLocation bucket : bucketsForTable) { @@ -1023,6 +1097,29 @@ private List getAllBucketsInCurrentNode(Integer currentNode, Clu } } } + + // Historical queues remain keyed by their original partition path. Add a location using + // that queue key while retaining the historical bucket as the RPC target. + for (Map.Entry entry : writeBatches.entrySet()) { + BucketAndWriteBatches bucketAndWriteBatches = entry.getValue(); + PhysicalTablePath originalPath = entry.getKey(); + if (!bucketAndWriteBatches.isHistoricalWriteTarget()) { + continue; + } + for (BucketLocation bucketLocation : + cluster.getAvailableBucketsForPhysicalTablePath( + bucketAndWriteBatches.targetPath)) { + if (bucketLocation.getLeader() != null + && Objects.equals(currentNode, bucketLocation.getLeader())) { + buckets.add( + new BucketLocation( + originalPath, + bucketLocation.getTableBucket(), + bucketLocation.getLeader(), + bucketLocation.getReplicas())); + } + } + } return buckets; } @@ -1162,13 +1259,24 @@ public void destroyResources() { /** Per table bucket and write batches. */ private static class BucketAndWriteBatches { public final boolean isPartitionedTable; + /** The physical partition used for metadata lookup, leader discovery, and write RPCs. */ + private final PhysicalTablePath targetPath; + public volatile @Nullable Long partitionId; // Write batches for each bucket in queue. public final Map> batches = new CopyOnWriteMap<>(); - public BucketAndWriteBatches(@Nullable Long partitionId, boolean isPartitionedTable) { + private BucketAndWriteBatches( + @Nullable Long partitionId, + boolean isPartitionedTable, + PhysicalTablePath targetPath) { this.partitionId = partitionId; this.isPartitionedTable = isPartitionedTable; + this.targetPath = targetPath; + } + + public boolean isHistoricalWriteTarget() { + return HISTORICAL_PARTITION_VALUE.equals(targetPath.getPartitionName()); } } } diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java b/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java index b16780035f7..2c71c584717 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java @@ -45,6 +45,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nullable; import javax.annotation.concurrent.GuardedBy; import java.util.ArrayList; @@ -52,10 +53,12 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import static org.apache.fluss.client.utils.ClientRpcMessageUtils.makeProduceLogRequest; import static org.apache.fluss.client.utils.ClientRpcMessageUtils.makePutKvRequest; +import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; import static org.apache.fluss.utils.Preconditions.checkArgument; import static org.apache.fluss.utils.Preconditions.checkNotNull; @@ -228,8 +231,8 @@ private void sendWriteData() throws Exception { // TODO: this try-catch is not needed when we don't update metadata for // unready partitions Throwable t = ExceptionUtils.stripExecutionException(e); - if (t.getCause() instanceof PartitionNotExistException) { - // ignore this exception, this is probably happen because the partition + if (t instanceof PartitionNotExistException) { + abortIfHistoricalWriteTargetMissing(readyCheckResult.unknownLeaderTables); } else { throw e; } @@ -250,10 +253,8 @@ private void sendWriteData() throws Exception { // get the list of batches prepare to send. Map> batches = accumulator.drain(clusterSnapshot, readyNodes, maxRequestSize); - if (!batches.isEmpty()) { addToInflightBatches(batches); - // TODO add logic for batch expire. sendWriteRequests(batches); @@ -393,25 +394,76 @@ private void sendWriteRequest(int destination, short acks, List } else { writeBatchByTable.forEach( (tableId, writeBatches) -> { - if (isLogBatches(writeBatches)) { - sendProduceLogRequestAndHandleResponse( - gateway, - makeProduceLogRequest( - tableId, acks, maxRequestTimeoutMs, writeBatches), - tableId, - writeBatches); - } else { - sendPutKvRequestAndHandleResponse( - gateway, - makePutKvRequest( - tableId, acks, maxRequestTimeoutMs, writeBatches), - tableId, - writeBatches); + boolean logBatches = isLogBatches(writeBatches); + for (List requestGroup : packRequestGroups(writeBatches)) { + if (logBatches) { + sendProduceLogRequestAndHandleResponse( + gateway, + makeProduceLogRequest( + tableId, acks, maxRequestTimeoutMs, requestGroup), + tableId, + requestGroup); + } else { + sendPutKvRequestAndHandleResponse( + gateway, + makePutKvRequest( + tableId, acks, maxRequestTimeoutMs, requestGroup), + tableId, + requestGroup); + } } }); } } + /** + * Splits normal and historical batches into separate requests. + * + *

Normal and historical writes cannot share a request. Both write protocols correlate + * historical responses by {@link TableBucket} and original partition name, so different + * original partitions targeting the same historical table bucket can remain in one request. + */ + private static List> packRequestGroups( + List writeBatches) { + List normalBatches = new ArrayList<>(); + List historicalBatches = new ArrayList<>(); + + for (ReadyWriteBatch readyWriteBatch : writeBatches) { + if (readyWriteBatch.writeBatch().getOriginalPartitionName() == null) { + normalBatches.add(readyWriteBatch); + } else { + historicalBatches.add(readyWriteBatch); + } + } + + List> requestGroups = new ArrayList<>(2); + if (!normalBatches.isEmpty()) { + requestGroups.add(normalBatches); + } + if (!historicalBatches.isEmpty()) { + requestGroups.add(historicalBatches); + } + return requestGroups; + } + + private static Map toBatchesByKey( + List writeBatches) { + Map recordsByKey = new HashMap<>(); + for (ReadyWriteBatch readyWriteBatch : writeBatches) { + WriteBatch writeBatch = readyWriteBatch.writeBatch(); + WriteBatchKey key = + new WriteBatchKey( + readyWriteBatch.tableBucket(), writeBatch.getOriginalPartitionName()); + ReadyWriteBatch previous = recordsByKey.put(key, readyWriteBatch); + checkArgument( + previous == null, + "A write request contains duplicate table bucket %s and original partition %s.", + readyWriteBatch.tableBucket(), + writeBatch.getOriginalPartitionName()); + } + return recordsByKey; + } + /** * Check whether the given batches are log batches. We assume all the batches are of the same * type. @@ -430,8 +482,7 @@ private void sendProduceLogRequestAndHandleResponse( ProduceLogRequest request, long tableId, List writeBatches) { - Map recordsByBucket = new HashMap<>(); - writeBatches.forEach(batch -> recordsByBucket.put(batch.tableBucket(), batch)); + Map recordsByKey = toBatchesByKey(writeBatches); long startTime = System.currentTimeMillis(); gateway.produceLog(request) .whenComplete( @@ -441,8 +492,7 @@ private void sendProduceLogRequestAndHandleResponse( if (e != null) { handleWriteRequestException(e, writeBatches); } else { - handleProduceLogResponse( - produceLogResponse, tableId, recordsByBucket); + handleProduceLogResponse(produceLogResponse, tableId, recordsByKey); } }); } @@ -452,8 +502,7 @@ private void sendPutKvRequestAndHandleResponse( PutKvRequest request, long tableId, List writeBatches) { - Map recordsByBucket = new HashMap<>(); - writeBatches.forEach(batch -> recordsByBucket.put(batch.tableBucket(), batch)); + Map recordsByKey = toBatchesByKey(writeBatches); long startTime = System.currentTimeMillis(); gateway.putKv(request) .whenComplete( @@ -463,7 +512,7 @@ private void sendPutKvRequestAndHandleResponse( if (e != null) { handleWriteRequestException(e, writeBatches); } else { - handlePutKvResponse(putKvResponse, tableId, recordsByBucket); + handlePutKvResponse(putKvResponse, tableId, recordsByKey); } }); } @@ -471,7 +520,7 @@ private void sendPutKvRequestAndHandleResponse( private void handleProduceLogResponse( ProduceLogResponse response, long tableId, - Map recordsByBucket) { + Map recordsByKey) { Set invalidMetadataTablesSet = new HashSet<>(); for (PbProduceLogRespForBucket logRespForBucket : response.getBucketsRespsList()) { TableBucket tb = @@ -481,7 +530,13 @@ private void handleProduceLogResponse( ? logRespForBucket.getPartitionId() : null, logRespForBucket.getBucketId()); - ReadyWriteBatch writeBatch = recordsByBucket.get(tb); + ReadyWriteBatch writeBatch = + recordsByKey.get( + new WriteBatchKey( + tb, + logRespForBucket.hasOriginalPartitionName() + ? logRespForBucket.getOriginalPartitionName() + : null)); if (logRespForBucket.hasErrorCode()) { Set invalidMetadataTables = handleWriteBatchException( @@ -497,7 +552,7 @@ private void handleProduceLogResponse( private void handlePutKvResponse( PutKvResponse putKvResponse, long tableId, - Map recordsByBucket) { + Map recordsByKey) { Set invalidMetadataTablesSet = new HashSet<>(); for (PbPutKvRespForBucket respForBucket : putKvResponse.getBucketsRespsList()) { TableBucket tb = @@ -511,7 +566,13 @@ private void handlePutKvResponse( accumulator.updateThrottle(tb, respForBucket.getPressure()); } - ReadyWriteBatch writeBatch = recordsByBucket.get(tb); + ReadyWriteBatch writeBatch = + recordsByKey.get( + new WriteBatchKey( + tb, + respForBucket.hasOriginalPartitionName() + ? respForBucket.getOriginalPartitionName() + : null)); if (writeBatch == null) { continue; } @@ -549,6 +610,14 @@ private Set handleWriteBatchException( ReadyWriteBatch readyWriteBatch, ApiError error) { Set invalidMetadataTables = new HashSet<>(); WriteBatch writeBatch = readyWriteBatch.writeBatch(); + // Historical queues use the original path as their accumulator key, so capture the actual + // RPC target before any retry handling. + PhysicalTablePath requestTargetPath = + writeBatch.getOriginalPartitionName() == null + ? writeBatch.physicalTablePath() + : PhysicalTablePath.of( + writeBatch.physicalTablePath().getTablePath(), + HISTORICAL_PARTITION_VALUE); if (error.exception() instanceof StorageBackpressureException) { // Hard rejection: the storage engine reached its slowdown trigger and rejected the // write. Map it to full pressure (internal hard-rejection value 1.0f) so the bucket is @@ -617,7 +686,10 @@ private Set handleWriteBatchException( readyWriteBatch.tableBucket(), error.exception()); } - invalidMetadataTables.add(writeBatch.physicalTablePath()); + // A historical batch remains keyed by its original partition path in the + // accumulator, but its RPC is sent to the internal historical partition. Invalidate + // the actual RPC target so the retry refreshes the historical bucket metadata. + invalidMetadataTables.add(requestTargetPath); } } else { LOG.warn( @@ -632,6 +704,35 @@ private Set handleWriteBatchException( return invalidMetadataTables; } + private void abortIfHistoricalWriteTargetMissing(Set unknownLeaderTables) + throws Exception { + for (PhysicalTablePath targetPath : unknownLeaderTables) { + if (!accumulator.isHistoricalPartitionEnabled(targetPath)) { + continue; + } + try { + metadataUpdater.checkAndUpdatePartitionMetadata(targetPath); + } catch (Exception e) { + Throwable t = ExceptionUtils.stripExecutionException(e); + if (t instanceof PartitionNotExistException) { + // Retrying a historical-enabled table without a leader would leave its + // batches queued indefinitely. Fail only after checking the target itself so + // ordinary writes in the bulk metadata request keep their existing behavior. + PartitionNotExistException missingTargetException = + new PartitionNotExistException( + "Write target " + + targetPath + + " for a historical-partition-enabled table no " + + "longer exists according to refreshed metadata."); + missingTargetException.initCause(t); + maybeAbortBatches(missingTargetException); + return; + } + throw e; + } + } + } + private void updateWriterMetrics(Map> batches) { batches.values() .forEach( @@ -702,4 +803,32 @@ private void awaitNextReadyCheck(long delayMs) throws InterruptedException { void destroyResources() { accumulator.destroyResources(); } + + private static final class WriteBatchKey { + private final TableBucket tableBucket; + private final @Nullable String originalPartitionName; + + private WriteBatchKey(TableBucket tableBucket, @Nullable String originalPartitionName) { + this.tableBucket = tableBucket; + this.originalPartitionName = originalPartitionName; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof WriteBatchKey)) { + return false; + } + WriteBatchKey that = (WriteBatchKey) o; + return tableBucket.equals(that.tableBucket) + && Objects.equals(originalPartitionName, that.originalPartitionName); + } + + @Override + public int hashCode() { + return Objects.hash(tableBucket, originalPartitionName); + } + } } diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/WriteBatch.java b/fluss-client/src/main/java/org/apache/fluss/client/write/WriteBatch.java index 754f9caa706..6c730bbb5e5 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/WriteBatch.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/WriteBatch.java @@ -54,6 +54,14 @@ public abstract class WriteBatch { protected final List callbacks = new ArrayList<>(); private final AtomicReference finalState = new AtomicReference<>(null); private final AtomicInteger attempts = new AtomicInteger(0); + /** + * The original partition name for a batch targeting the historical system partition. + * + *

It is null for a normal write and contains the logical partition namespace for a + * historical write. + */ + private final @Nullable String originalPartitionName; + protected boolean reopened; protected int recordCount; private long drainedMs; @@ -64,12 +72,14 @@ public WriteBatch( PhysicalTablePath physicalTablePath, int schemaId, WriteFormat writeFormat, + @Nullable String originalPartitionName, long createdMs) { this.physicalTablePath = physicalTablePath; this.createdMs = createdMs; this.tableId = tableId; this.schemaId = schemaId; this.writeFormat = checkNotNull(writeFormat, "write format must be not null"); + this.originalPartitionName = originalPartitionName; this.bucketId = bucketId; this.requestFuture = new RequestFuture(); this.recordCount = 0; @@ -193,6 +203,11 @@ public PhysicalTablePath physicalTablePath() { return physicalTablePath; } + /** Returns the original partition name for a historical write, or null for a normal write. */ + public @Nullable String getOriginalPartitionName() { + return originalPartitionName; + } + public RequestFuture getRequestFuture() { return requestFuture; } diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/WriterClient.java b/fluss-client/src/main/java/org/apache/fluss/client/write/WriterClient.java index ad8c7870547..cb46c627bed 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/WriterClient.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/WriterClient.java @@ -28,10 +28,12 @@ import org.apache.fluss.config.Configuration; import org.apache.fluss.exception.FlussRuntimeException; import org.apache.fluss.exception.IllegalConfigurationException; +import org.apache.fluss.exception.PartitionNotExistException; import org.apache.fluss.metadata.PhysicalTablePath; import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.rpc.gateway.TabletServerGateway; import org.apache.fluss.rpc.metrics.ClientMetricGroup; +import org.apache.fluss.utils.AutoPartitionStrategy; import org.apache.fluss.utils.CopyOnWriteMap; import org.apache.fluss.utils.clock.SystemClock; import org.apache.fluss.utils.concurrent.ExecutorThreadFactory; @@ -42,6 +44,9 @@ import javax.annotation.concurrent.ThreadSafe; import java.time.Duration; +import java.time.Instant; +import java.time.ZonedDateTime; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.concurrent.ExecutorService; @@ -51,6 +56,9 @@ import static org.apache.fluss.config.ConfigOptions.NoKeyAssigner.ROUND_ROBIN; import static org.apache.fluss.config.ConfigOptions.NoKeyAssigner.STICKY; import static org.apache.fluss.utils.ExceptionUtils.toException; +import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; +import static org.apache.fluss.utils.PartitionUtils.generateAutoPartitionTime; +import static org.apache.fluss.utils.PartitionUtils.isPastAutoPartition; /** * A client that write records to server. @@ -70,6 +78,7 @@ public class WriterClient { private static final Logger LOG = LoggerFactory.getLogger(WriterClient.class); public static final String SENDER_THREAD_PREFIX = "fluss-write-sender"; + private static final Duration MAX_DEFAULT_TIME_ZONE_DIFFERENCE = Duration.ofHours(26); /** * {@link ConfigOptions#CLIENT_WRITER_MAX_INFLIGHT_REQUESTS_PER_BUCKET} should be less than or * equal to this value when idempotence producer enabled to ensure message ordering. @@ -194,7 +203,12 @@ private void doSend(WriteRecord record, WriteCallback callback) { PhysicalTablePath physicalTablePath = record.getPhysicalTablePath(); // Skip the call entirely on non-partitioned tables; there is no partition to create. if (tableInfo.isPartitioned()) { - dynamicPartitionCreator.checkAndCreatePartitionAsync(physicalTablePath, tableInfo); + if (mayBeExpiredHistoricalPartition(physicalTablePath, tableInfo, Instant.now())) { + resolveHistoricalWriteTarget(physicalTablePath); + } else { + dynamicPartitionCreator.checkAndCreatePartitionAsync( + physicalTablePath, tableInfo); + } } // maybe create bucket assigner. @@ -240,6 +254,71 @@ private void doSend(WriteRecord record, WriteCallback callback) { } } + static boolean mayBeExpiredHistoricalPartition( + PhysicalTablePath physicalTablePath, TableInfo tableInfo, Instant now) { + String partitionName = physicalTablePath.getPartitionName(); + AutoPartitionStrategy strategy = tableInfo.getTableConfig().getAutoPartitionStrategy(); + if (partitionName == null + || !tableInfo.getTableConfig().isHistoricalPartitionEnabled() + || strategy.numToRetain() < 0) { + return false; + } + + // The table's default time zone is not persisted. Shift the expiration boundary by the + // largest IANA time-zone difference, then apply retention in the table's partition unit. + Instant latestPotentialServerTime = now.plus(MAX_DEFAULT_TIME_ZONE_DIFFERENCE); + if (!isPastAutoPartition(partitionName, strategy, latestPotentialServerTime)) { + return false; + } + ZonedDateTime latestPotentialServerDateTime = + ZonedDateTime.ofInstant(latestPotentialServerTime, strategy.timeZone().toZoneId()); + String earliestPotentialRetainedPartition = + generateAutoPartitionTime( + latestPotentialServerDateTime, + -strategy.numToRetain(), + strategy.timeUnit(), + strategy); + return partitionName.compareTo(earliestPotentialRetainedPartition) < 0; + } + + private synchronized void resolveHistoricalWriteTarget(PhysicalTablePath originalPath) { + if (accumulator.hasWriteTarget(originalPath)) { + return; + } + + // The time check only limits metadata traffic. Invalidate a potentially stale cached route + // and authoritatively choose the target before the record enters the queue. + metadataUpdater.invalidPhysicalTableBucketAndPartitionMeta( + Collections.singleton(originalPath)); + PhysicalTablePath targetPath = originalPath; + try { + if (!metadataUpdater.checkAndUpdatePartitionMetadata(originalPath)) { + throw new FlussRuntimeException( + "Failed to resolve write target for " + originalPath + '.'); + } + } catch (PartitionNotExistException ignored) { + targetPath = + PhysicalTablePath.of(originalPath.getTablePath(), HISTORICAL_PARTITION_VALUE); + // TODO: Activate this target only after Server retirement guarantees that all accepted + // original writes have been tiered to the lake. + if (!metadataUpdater.checkAndUpdatePartitionMetadata(targetPath)) { + throw new PartitionNotExistException( + "Historical partition " + targetPath + " does not exist."); + } + } + + if (!accumulator.tryRouteWritesTo( + originalPath, targetPath, metadataUpdater.getPartitionIdOrElseThrow(targetPath))) { + throw new FlussRuntimeException( + "Cannot route writes for " + + originalPath + + " to " + + targetPath + + " because the accumulator already contains writes for a different " + + "physical target."); + } + } + private void maybeAbortBatches(Throwable t) { if (accumulator.hasIncomplete()) { LOG.error("Aborting all pending write batches due to fatal error", t); diff --git a/fluss-client/src/test/java/org/apache/fluss/client/utils/ClientRpcMessageUtilsTest.java b/fluss-client/src/test/java/org/apache/fluss/client/utils/ClientRpcMessageUtilsTest.java index 3ed17da7da5..153c3dc13f0 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/utils/ClientRpcMessageUtilsTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/utils/ClientRpcMessageUtilsTest.java @@ -140,6 +140,7 @@ private KvWriteBatch createKvWriteBatch(int bucketId, MergeMode mergeMode) throw outputView, null, mergeMode, + null, System.currentTimeMillis()); } } diff --git a/fluss-client/src/test/java/org/apache/fluss/client/write/ArrowLogWriteBatchTest.java b/fluss-client/src/test/java/org/apache/fluss/client/write/ArrowLogWriteBatchTest.java index f87f0c01a35..c9b98e1dfca 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/write/ArrowLogWriteBatchTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/write/ArrowLogWriteBatchTest.java @@ -137,6 +137,7 @@ void testAppendWithPreAllocatedMemorySegments() throws Exception { DATA1_ROW_TYPE, DEFAULT_COMPRESSION), new PreAllocatedPagedOutputView(memorySegmentList), + null, System.currentTimeMillis(), null); assertThat(arrowLogWriteBatch.pooledMemorySegments()).isEqualTo(memorySegmentList); @@ -213,6 +214,7 @@ void testArrowCompressionRatioEstimated() throws Exception { DATA1_TABLE_INFO.getSchemaId(), arrowWriter, new PreAllocatedPagedOutputView(memorySegmentList), + null, System.currentTimeMillis(), null); @@ -315,6 +317,7 @@ private ArrowLogWriteBatch createArrowLogWriteBatch(TableBucket tb, int maxSizeI DATA1_ROW_TYPE, DEFAULT_COMPRESSION), new UnmanagedPagedOutputView(128), + null, System.currentTimeMillis(), null); } diff --git a/fluss-client/src/test/java/org/apache/fluss/client/write/CompactedLogWriteBatchTest.java b/fluss-client/src/test/java/org/apache/fluss/client/write/CompactedLogWriteBatchTest.java index dc8fbce9d32..2461c2a7134 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/write/CompactedLogWriteBatchTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/write/CompactedLogWriteBatchTest.java @@ -254,6 +254,7 @@ private CompactedLogWriteBatch createLogWriteBatch( DATA1_TABLE_INFO.getSchemaId(), writeLimit, new PreAllocatedPagedOutputView(Collections.singletonList(memorySegment)), + null, System.currentTimeMillis()); } diff --git a/fluss-client/src/test/java/org/apache/fluss/client/write/IndexedLogWriteBatchTest.java b/fluss-client/src/test/java/org/apache/fluss/client/write/IndexedLogWriteBatchTest.java index 331be4209f6..a6269ad64b3 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/write/IndexedLogWriteBatchTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/write/IndexedLogWriteBatchTest.java @@ -216,6 +216,7 @@ private IndexedLogWriteBatch createLogWriteBatch( DATA1_TABLE_INFO.getSchemaId(), writeLimit, new PreAllocatedPagedOutputView(Collections.singletonList(memorySegment)), + null, System.currentTimeMillis()); } diff --git a/fluss-client/src/test/java/org/apache/fluss/client/write/KvWriteBatchTest.java b/fluss-client/src/test/java/org/apache/fluss/client/write/KvWriteBatchTest.java index 7b61976a038..b0faf06e679 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/write/KvWriteBatchTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/write/KvWriteBatchTest.java @@ -230,6 +230,7 @@ private KvWriteBatch createKvWriteBatch( outputView, null, MergeMode.DEFAULT, + null, System.currentTimeMillis()); } @@ -326,6 +327,7 @@ private KvWriteBatch createKvWriteBatchWithMergeMode(TableBucket tb, MergeMode m outputView, null, mergeMode, + null, System.currentTimeMillis()); } diff --git a/fluss-client/src/test/java/org/apache/fluss/client/write/RecordAccumulatorTest.java b/fluss-client/src/test/java/org/apache/fluss/client/write/RecordAccumulatorTest.java index acd1e4e2911..21939d16888 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/write/RecordAccumulatorTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/write/RecordAccumulatorTest.java @@ -43,6 +43,7 @@ import org.apache.fluss.row.BinaryRow; import org.apache.fluss.row.GenericRow; import org.apache.fluss.row.arrow.ArrowWriter; +import org.apache.fluss.row.encode.CompactedKeyEncoder; import org.apache.fluss.row.indexed.IndexedRow; import org.apache.fluss.rpc.GatewayClientProxy; import org.apache.fluss.rpc.RpcClient; @@ -71,14 +72,21 @@ import static org.apache.fluss.record.LogRecordBatch.CURRENT_LOG_MAGIC_VALUE; import static org.apache.fluss.record.LogRecordBatchFormat.recordBatchHeaderSize; import static org.apache.fluss.record.TestData.DATA1_PHYSICAL_TABLE_PATH; +import static org.apache.fluss.record.TestData.DATA1_PHYSICAL_TABLE_PATH_PK_PA_2024; import static org.apache.fluss.record.TestData.DATA1_ROW_TYPE; import static org.apache.fluss.record.TestData.DATA1_SCHEMA; +import static org.apache.fluss.record.TestData.DATA1_SCHEMA_PK; import static org.apache.fluss.record.TestData.DATA1_TABLE_ID; +import static org.apache.fluss.record.TestData.DATA1_TABLE_ID_PK; import static org.apache.fluss.record.TestData.DATA1_TABLE_INFO; +import static org.apache.fluss.record.TestData.DATA1_TABLE_INFO_PK; import static org.apache.fluss.record.TestData.DATA1_TABLE_PATH; +import static org.apache.fluss.record.TestData.DATA1_TABLE_PATH_PK; import static org.apache.fluss.record.TestData.DEFAULT_REMOTE_DATA_DIR; +import static org.apache.fluss.testutils.DataTestUtils.compactedRow; import static org.apache.fluss.testutils.DataTestUtils.indexedRow; import static org.apache.fluss.testutils.DataTestUtils.row; +import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -191,6 +199,41 @@ void testDrainBatches() throws Exception { verifyTableBucketInBatches(batches3, tb1, tb3); } + @Test + void testAppendAfterHistoricalTargetResolved() throws Exception { + long originalPartitionId = 11L; + long historicalPartitionId = 22L; + PhysicalTablePath originalPath = DATA1_PHYSICAL_TABLE_PATH_PK_PA_2024; + PhysicalTablePath anotherOriginalPath = PhysicalTablePath.of(DATA1_TABLE_PATH_PK, "2023"); + PhysicalTablePath historicalPath = + PhysicalTablePath.of(DATA1_TABLE_PATH_PK, HISTORICAL_PARTITION_VALUE); + TableBucket originalBucket = new TableBucket(DATA1_TABLE_ID_PK, originalPartitionId, 0); + TableBucket historicalBucket = new TableBucket(DATA1_TABLE_ID_PK, historicalPartitionId, 0); + cluster = + createPartitionedKvCluster( + originalPath, originalBucket, historicalPath, historicalBucket); + + RecordAccumulator accum = createTestRecordAccumulator(1024, 10L * 1024); + accum.tryRouteWritesTo(originalPath, historicalPath, historicalPartitionId); + accum.tryRouteWritesTo(anotherOriginalPath, historicalPath, historicalPartitionId); + accum.append(createKvRecord(originalPath), writeCallback, cluster, 0, false); + accum.append(createKvRecord(originalPath), writeCallback, cluster, 0, false); + accum.append(createKvRecord(anotherOriginalPath), writeCallback, cluster, 0, false); + + List drainedBatches = + accum.drain(cluster, Collections.singleton(node1.id()), Integer.MAX_VALUE) + .get(node1.id()); + + assertThat(drainedBatches).hasSize(2); + assertThat(drainedBatches) + .allSatisfy(batch -> assertThat(batch.tableBucket()).isEqualTo(historicalBucket)); + assertThat(drainedBatches) + .extracting(batch -> ((KvWriteBatch) batch.writeBatch()).getOriginalPartitionName()) + .containsExactlyInAnyOrder( + originalPath.getPartitionName(), anotherOriginalPath.getPartitionName()); + drainedBatches.forEach(batch -> accum.deallocate(batch.writeBatch())); + } + @Test void testDrainCompressedBatches() throws Exception { int batchSize = 10 * 1024; @@ -584,6 +627,21 @@ private WriteRecord createRecord(IndexedRow row, TableInfo tableInfo) { return WriteRecord.forIndexedAppend(tableInfo, DATA1_PHYSICAL_TABLE_PATH, row, null); } + private WriteRecord createKvRecord(PhysicalTablePath physicalTablePath) { + BinaryRow row = compactedRow(DATA1_ROW_TYPE, new Object[] {1, "a"}); + byte[] key = + new CompactedKeyEncoder(DATA1_ROW_TYPE, DATA1_SCHEMA_PK.getPrimaryKeyIndexes()) + .encodeKey(row); + return WriteRecord.forUpsert( + DATA1_TABLE_INFO_PK, + physicalTablePath, + row, + key, + key, + WriteFormat.COMPACTED_KV, + null); + } + private TableInfo withSchemaId(int schemaId) { return new TableInfo( DATA1_TABLE_INFO.getTablePath(), @@ -622,6 +680,36 @@ private Cluster updateCluster(List bucketLocations) { Collections.emptyMap()); } + private Cluster createPartitionedKvCluster( + PhysicalTablePath originalPath, + TableBucket originalBucket, + PhysicalTablePath historicalPath, + TableBucket historicalBucket) { + Map aliveTabletServersById = new HashMap<>(); + aliveTabletServersById.put(node1.id(), node1); + + Map> bucketsByPath = new HashMap<>(); + bucketsByPath.put( + originalPath, + Collections.singletonList( + new BucketLocation(originalPath, originalBucket, node1.id(), serverNodes))); + bucketsByPath.put( + historicalPath, + Collections.singletonList( + new BucketLocation( + historicalPath, historicalBucket, node1.id(), serverNodes))); + + Map partitionIdsByPath = new HashMap<>(); + partitionIdsByPath.put(originalPath, originalBucket.getPartitionId()); + partitionIdsByPath.put(historicalPath, historicalBucket.getPartitionId()); + return new Cluster( + aliveTabletServersById, + new ServerNode(0, "localhost", 89, ServerType.COORDINATOR), + bucketsByPath, + Collections.singletonMap(DATA1_TABLE_PATH_PK, DATA1_TABLE_ID_PK), + partitionIdsByPath); + } + private void delayedInterrupt(final Thread thread, final long delayMs) { Thread t = new Thread( diff --git a/fluss-client/src/test/java/org/apache/fluss/client/write/SenderTest.java b/fluss-client/src/test/java/org/apache/fluss/client/write/SenderTest.java index 69d9c669c9e..3dc684a13a3 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/write/SenderTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/write/SenderTest.java @@ -19,23 +19,30 @@ import org.apache.fluss.client.metadata.TestingMetadataUpdater; import org.apache.fluss.client.metrics.TestingWriterMetricGroup; +import org.apache.fluss.cluster.BucketLocation; import org.apache.fluss.cluster.Cluster; import org.apache.fluss.cluster.ServerNode; +import org.apache.fluss.config.AutoPartitionTimeUnit; import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; import org.apache.fluss.config.MemorySize; import org.apache.fluss.exception.AuthorizationException; import org.apache.fluss.exception.NetworkException; +import org.apache.fluss.exception.PartitionNotExistException; import org.apache.fluss.exception.TableNotExistException; import org.apache.fluss.exception.TimeoutException; +import org.apache.fluss.metadata.DataLakeFormat; import org.apache.fluss.metadata.PhysicalTablePath; +import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.metadata.TablePath; import org.apache.fluss.record.MemoryLogRecords; import org.apache.fluss.row.BinaryRow; import org.apache.fluss.row.GenericRow; import org.apache.fluss.row.encode.CompactedKeyEncoder; +import org.apache.fluss.row.indexed.IndexedRow; import org.apache.fluss.rpc.entity.ProduceLogResultForBucket; import org.apache.fluss.rpc.entity.PutKvResultForBucket; import org.apache.fluss.rpc.messages.ApiMessage; @@ -44,7 +51,9 @@ import org.apache.fluss.rpc.messages.PutKvRequest; import org.apache.fluss.rpc.messages.PutKvResponse; import org.apache.fluss.rpc.protocol.Errors; +import org.apache.fluss.server.entity.ProduceLogDataForBucket; import org.apache.fluss.server.tablet.TestTabletServerGateway; +import org.apache.fluss.types.DataTypes; import org.apache.fluss.utils.clock.SystemClock; import org.junit.jupiter.api.AfterEach; @@ -52,10 +61,13 @@ import org.junit.jupiter.api.Test; import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Deque; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -76,12 +88,14 @@ import static org.apache.fluss.record.TestData.DATA2_TABLE_ID; import static org.apache.fluss.record.TestData.DEFAULT_REMOTE_DATA_DIR; import static org.apache.fluss.rpc.protocol.Errors.SCHEMA_NOT_EXIST; -import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getProduceLogData; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.makeProduceLogResponse; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.makePutKvResponse; +import static org.apache.fluss.server.utils.ServerRpcMessageUtils.toProduceLogDataForBuckets; import static org.apache.fluss.testutils.DataTestUtils.compactedRow; +import static org.apache.fluss.testutils.DataTestUtils.indexedRow; import static org.apache.fluss.testutils.DataTestUtils.row; import static org.apache.fluss.testutils.common.CommonTestUtils.retry; +import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; import static org.assertj.core.api.Assertions.assertThat; /** ITCase for {@link Sender}. */ @@ -114,6 +128,263 @@ public void teardown() throws Exception { sender.destroyResources(); } + @Test + void testSendsHistoricalPutWhenTargetResolvedBeforeAppend() throws Exception { + sender.destroyResources(); + String originalPartitionName = "20000101"; + TableInfo tableInfo = createHistoricalTableInfo(); + PhysicalTablePath originalPath = + PhysicalTablePath.of(tableInfo.getTablePath(), originalPartitionName); + PhysicalTablePath historicalPath = + PhysicalTablePath.of(tableInfo.getTablePath(), HISTORICAL_PARTITION_VALUE); + long historicalPartitionId = 22L; + TableBucket historicalBucket = + new TableBucket(tableInfo.getTableId(), historicalPartitionId, 0); + metadataUpdater = + new TestingMetadataUpdater( + Collections.singletonMap(tableInfo.getTablePath(), tableInfo)); + metadataUpdater.updateCluster( + partitionedCluster( + tableInfo, Collections.singletonMap(historicalPath, historicalBucket))); + sender = setupWithIdempotenceState(); + accumulator.tryRouteWritesTo(originalPath, historicalPath, historicalPartitionId); + + CompletableFuture future = + appendKvRecord(tableInfo, originalPath, 1, metadataUpdater.getCluster()); + + sender.runOnce(); + + assertThat(sender.numOfInFlightBatches(historicalBucket)).isOne(); + TestTabletServerGateway gateway = node1Gateway(); + PutKvRequest request = (PutKvRequest) gateway.getRequest(0); + assertThat(request.getBucketsReqAt(0).getPartitionId()).isEqualTo(historicalPartitionId); + assertThat(request.getBucketsReqAt(0).getOriginalPartitionName()) + .isEqualTo(originalPartitionName); + + gateway.response( + 0, createHistoricalPutKvResponse(historicalBucket, 1L, originalPartitionName)); + assertThat(future.get()).isNull(); + } + + @Test + void testPotentialExpirationUsesAutoPartitionTimeUnit() { + TableInfo tableInfo = createHistoricalTableInfo(AutoPartitionTimeUnit.HOUR, 48); + Instant now = Instant.parse("2026-08-24T00:00:00Z"); + + assertThat( + WriterClient.mayBeExpiredHistoricalPartition( + PhysicalTablePath.of(tableInfo.getTablePath(), "2026082301"), + tableInfo, + now)) + .isTrue(); + assertThat( + WriterClient.mayBeExpiredHistoricalPartition( + PhysicalTablePath.of(tableInfo.getTablePath(), "2026082302"), + tableInfo, + now)) + .isFalse(); + } + + @Test + void testFailsWriteAfterMetadataConfirmsPartitionMissing() throws Exception { + sender.destroyResources(); + TableInfo tableInfo = createHistoricalTableInfo(); + PhysicalTablePath originalPath = PhysicalTablePath.of(tableInfo.getTablePath(), "20990101"); + TableBucket originalBucket = new TableBucket(tableInfo.getTableId(), 21L, 0); + metadataUpdater = missingPartitionMetadataUpdater(tableInfo); + metadataUpdater.updateCluster( + partitionedCluster( + tableInfo, Collections.singletonMap(originalPath, originalBucket))); + sender = setupWithIdempotenceState(); + + CompletableFuture future = + appendKvRecord(tableInfo, originalPath, 1, metadataUpdater.getCluster()); + sender.runOnce(); + + TestTabletServerGateway gateway = node1Gateway(); + gateway.response( + 0, createPutKvResponse(originalBucket, Errors.UNKNOWN_TABLE_OR_BUCKET_EXCEPTION)); + assertThat(future).isNotDone(); + + sender.runOnce(); + assertThat(future.get()) + .isInstanceOf(PartitionNotExistException.class) + .hasMessageContaining(originalPath.toString()) + .hasCauseInstanceOf(PartitionNotExistException.class); + } + + @Test + void testMissingPartitionDoesNotAbortNormalWrites() throws Exception { + sender.destroyResources(); + TableInfo tableInfo = createNormalPartitionedTableInfo(); + PhysicalTablePath partitionPath = + PhysicalTablePath.of(tableInfo.getTablePath(), "20990101"); + TableBucket tableBucket = new TableBucket(tableInfo.getTableId(), 21L, 0); + metadataUpdater = missingPartitionMetadataUpdater(tableInfo); + metadataUpdater.updateCluster( + partitionedCluster( + tableInfo, Collections.singletonMap(partitionPath, tableBucket))); + sender = setupWithIdempotenceState(); + + CompletableFuture future = + appendKvRecord(tableInfo, partitionPath, 1, metadataUpdater.getCluster()); + sender.runOnce(); + + TestTabletServerGateway gateway = node1Gateway(); + gateway.response( + 0, createPutKvResponse(tableBucket, Errors.UNKNOWN_TABLE_OR_BUCKET_EXCEPTION)); + sender.runOnce(); + + assertThat(future).isNotDone(); + accumulator.abortAllBatches(new RuntimeException("Test cleanup.")); + } + + @Test + void testPackNormalAndHistoricalPutRequests() throws Exception { + sender.destroyResources(); + TableInfo tableInfo = createHistoricalTableInfo(); + PhysicalTablePath activePath = PhysicalTablePath.of(tableInfo.getTablePath(), "20990101"); + PhysicalTablePath firstOriginalPath = + PhysicalTablePath.of(tableInfo.getTablePath(), "20000101"); + PhysicalTablePath secondOriginalPath = + PhysicalTablePath.of(tableInfo.getTablePath(), "20000102"); + PhysicalTablePath historicalPath = + PhysicalTablePath.of(tableInfo.getTablePath(), HISTORICAL_PARTITION_VALUE); + TableBucket activeBucket = new TableBucket(tableInfo.getTableId(), 21L, 0); + TableBucket historicalBucket = new TableBucket(tableInfo.getTableId(), 22L, 0); + + Map tableBucketsByPath = new HashMap<>(); + tableBucketsByPath.put(activePath, activeBucket); + tableBucketsByPath.put(historicalPath, historicalBucket); + metadataUpdater = + new TestingMetadataUpdater( + Collections.singletonMap(tableInfo.getTablePath(), tableInfo)); + metadataUpdater.updateCluster(partitionedCluster(tableInfo, tableBucketsByPath)); + sender = setupWithIdempotenceState(); + + CompletableFuture activeFuture = + appendKvRecord(tableInfo, activePath, 1, metadataUpdater.getCluster()); + accumulator.tryRouteWritesTo( + firstOriginalPath, historicalPath, historicalBucket.getPartitionId()); + accumulator.tryRouteWritesTo( + secondOriginalPath, historicalPath, historicalBucket.getPartitionId()); + CompletableFuture firstHistoricalFuture = + appendKvRecord(tableInfo, firstOriginalPath, 2, metadataUpdater.getCluster()); + CompletableFuture secondHistoricalFuture = + appendKvRecord(tableInfo, secondOriginalPath, 3, metadataUpdater.getCluster()); + + sender.runOnce(); + + TestTabletServerGateway gateway = node1Gateway(); + assertThat(gateway.pendingRequestSize()).isEqualTo(2); + + PutKvRequest normalRequest = (PutKvRequest) gateway.getRequest(0); + assertThat(normalRequest.getBucketsReqsCount()).isOne(); + assertThat(normalRequest.getBucketsReqAt(0).getPartitionId()) + .isEqualTo(activeBucket.getPartitionId()); + assertThat(normalRequest.getBucketsReqAt(0).hasOriginalPartitionName()).isFalse(); + + PutKvRequest historicalRequest = (PutKvRequest) gateway.getRequest(1); + assertThat(historicalRequest.getBucketsReqsCount()).isEqualTo(2); + Set originalPartitionNames = new HashSet<>(); + for (int i = 0; i < historicalRequest.getBucketsReqsCount(); i++) { + assertThat(historicalRequest.getBucketsReqAt(i).getPartitionId()) + .isEqualTo(historicalBucket.getPartitionId()); + originalPartitionNames.add( + historicalRequest.getBucketsReqAt(i).getOriginalPartitionName()); + } + assertThat(originalPartitionNames) + .containsExactlyInAnyOrder( + firstOriginalPath.getPartitionName(), + secondOriginalPath.getPartitionName()); + + gateway.response(0, createPutKvResponse(activeBucket, 1L)); + gateway.response( + 0, + makePutKvResponse( + Arrays.asList( + PutKvResultForBucket.historicalSuccess( + historicalBucket, + 1L, + secondOriginalPath.getPartitionName()), + PutKvResultForBucket.historicalSuccess( + historicalBucket, + 1L, + firstOriginalPath.getPartitionName())))); + assertThat(activeFuture).isDone(); + assertThat(firstHistoricalFuture).isDone(); + assertThat(secondHistoricalFuture).isDone(); + assertThat(activeFuture.get()).isNull(); + assertThat(firstHistoricalFuture.get()).isNull(); + assertThat(secondHistoricalFuture.get()).isNull(); + } + + @Test + void testPackHistoricalProduceLogRequests() throws Exception { + sender.destroyResources(); + TableInfo tableInfo = createHistoricalLogTableInfo(); + PhysicalTablePath firstOriginalPath = + PhysicalTablePath.of(tableInfo.getTablePath(), "20000101"); + PhysicalTablePath secondOriginalPath = + PhysicalTablePath.of(tableInfo.getTablePath(), "20000102"); + PhysicalTablePath historicalPath = + PhysicalTablePath.of(tableInfo.getTablePath(), HISTORICAL_PARTITION_VALUE); + TableBucket historicalBucket = new TableBucket(tableInfo.getTableId(), 22L, 0); + + metadataUpdater = + new TestingMetadataUpdater( + Collections.singletonMap(tableInfo.getTablePath(), tableInfo)); + metadataUpdater.updateCluster( + partitionedCluster( + tableInfo, Collections.singletonMap(historicalPath, historicalBucket))); + sender = setupWithIdempotenceState(); + + accumulator.tryRouteWritesTo( + firstOriginalPath, historicalPath, historicalBucket.getPartitionId()); + accumulator.tryRouteWritesTo( + secondOriginalPath, historicalPath, historicalBucket.getPartitionId()); + CompletableFuture firstFuture = + appendLogRecord(tableInfo, firstOriginalPath, 1, metadataUpdater.getCluster()); + CompletableFuture secondFuture = + appendLogRecord(tableInfo, secondOriginalPath, 2, metadataUpdater.getCluster()); + + sender.runOnce(); + + TestTabletServerGateway gateway = node1Gateway(); + assertThat(gateway.pendingRequestSize()).isOne(); + ProduceLogRequest request = (ProduceLogRequest) gateway.getRequest(0); + assertThat(request.getBucketsReqsCount()).isEqualTo(2); + Set originalPartitionNames = new HashSet<>(); + for (int i = 0; i < request.getBucketsReqsCount(); i++) { + assertThat(request.getBucketsReqAt(i).getPartitionId()) + .isEqualTo(historicalBucket.getPartitionId()); + originalPartitionNames.add(request.getBucketsReqAt(i).getOriginalPartitionName()); + } + assertThat(originalPartitionNames) + .containsExactlyInAnyOrder( + firstOriginalPath.getPartitionName(), + secondOriginalPath.getPartitionName()); + + gateway.response( + 0, + makeProduceLogResponse( + Arrays.asList( + ProduceLogResultForBucket.historicalSuccess( + historicalBucket, + 1L, + 2L, + secondOriginalPath.getPartitionName()), + ProduceLogResultForBucket.historicalSuccess( + historicalBucket, + 0L, + 1L, + firstOriginalPath.getPartitionName())))); + assertThat(firstFuture).isDone(); + assertThat(secondFuture).isDone(); + assertThat(firstFuture.get()).isNull(); + assertThat(secondFuture.get()).isNull(); + } + @Test void testSimple() throws Exception { long offset = 0; @@ -1124,6 +1395,159 @@ private void resetTableInfosWith(TableInfo tableInfo) { metadataUpdater.updateTableInfos(tableInfos); } + private static TableInfo createHistoricalTableInfo() { + return createHistoricalTableInfo(AutoPartitionTimeUnit.DAY, 7); + } + + private static TableInfo createHistoricalTableInfo( + AutoPartitionTimeUnit timeUnit, int numToRetain) { + return createPartitionedKvTableInfo(timeUnit, numToRetain, true); + } + + private static TableInfo createNormalPartitionedTableInfo() { + return createPartitionedKvTableInfo(AutoPartitionTimeUnit.DAY, 7, false); + } + + private static TableInfo createPartitionedKvTableInfo( + AutoPartitionTimeUnit timeUnit, int numToRetain, boolean historicalPartitionEnabled) { + Schema schema = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("dt", DataTypes.STRING()) + .primaryKey("id", "dt") + .build(); + TableDescriptor descriptor = + TableDescriptor.builder() + .schema(schema) + .partitionedBy("dt") + .distributedBy(1) + .property(ConfigOptions.TABLE_AUTO_PARTITION_ENABLED, true) + .property(ConfigOptions.TABLE_AUTO_PARTITION_TIME_UNIT, timeUnit) + .property(ConfigOptions.TABLE_AUTO_PARTITION_NUM_RETENTION, numToRetain) + .property(ConfigOptions.TABLE_AUTO_PARTITION_TIMEZONE, "UTC") + .property(ConfigOptions.TABLE_DATALAKE_ENABLED, true) + .property(ConfigOptions.TABLE_DATALAKE_FORMAT, DataLakeFormat.PAIMON) + .property( + ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_ENABLED, + historicalPartitionEnabled) + .build(); + return TableInfo.of( + DATA1_TABLE_PATH_PK, + DATA1_TABLE_ID_PK, + 1, + descriptor, + DEFAULT_REMOTE_DATA_DIR, + 1L, + 1L); + } + + private static TestingMetadataUpdater missingPartitionMetadataUpdater(TableInfo tableInfo) { + return new TestingMetadataUpdater( + Collections.singletonMap(tableInfo.getTablePath(), tableInfo)) { + @Override + public void updatePhysicalTableMetadata(Set physicalTablePaths) { + throw new PartitionNotExistException("Partition does not exist."); + } + + @Override + public boolean checkAndUpdatePartitionMetadata(PhysicalTablePath physicalTablePath) { + throw new PartitionNotExistException("Partition does not exist."); + } + }; + } + + private static TableInfo createHistoricalLogTableInfo() { + Schema schema = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("dt", DataTypes.STRING()) + .build(); + TableDescriptor descriptor = + TableDescriptor.builder() + .schema(schema) + .partitionedBy("dt") + .distributedBy(1, "id") + .property(ConfigOptions.TABLE_AUTO_PARTITION_ENABLED, true) + .property(ConfigOptions.TABLE_DATALAKE_ENABLED, true) + .property(ConfigOptions.TABLE_DATALAKE_FORMAT, DataLakeFormat.PAIMON) + .property(ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_ENABLED, true) + .build(); + return TableInfo.of( + DATA1_TABLE_PATH, DATA1_TABLE_ID, 1, descriptor, DEFAULT_REMOTE_DATA_DIR, 1L, 1L); + } + + private static Cluster partitionedCluster( + TableInfo tableInfo, Map tableBucketsByPath) { + int[] replicas = new int[] {TestingMetadataUpdater.NODE1.id()}; + Map> bucketLocationsByPath = new HashMap<>(); + Map partitionIdsByPath = new HashMap<>(); + tableBucketsByPath.forEach( + (physicalTablePath, tableBucket) -> { + bucketLocationsByPath.put( + physicalTablePath, + Collections.singletonList( + new BucketLocation( + physicalTablePath, + tableBucket, + TestingMetadataUpdater.NODE1.id(), + replicas))); + partitionIdsByPath.put(physicalTablePath, tableBucket.getPartitionId()); + }); + return new Cluster( + Collections.singletonMap( + TestingMetadataUpdater.NODE1.id(), TestingMetadataUpdater.NODE1), + TestingMetadataUpdater.COORDINATOR, + bucketLocationsByPath, + Collections.singletonMap(tableInfo.getTablePath(), tableInfo.getTableId()), + partitionIdsByPath); + } + + private CompletableFuture appendKvRecord( + TableInfo tableInfo, PhysicalTablePath physicalTablePath, int id, Cluster cluster) + throws Exception { + BinaryRow row = + compactedRow( + tableInfo.getRowType(), + new Object[] {id, physicalTablePath.getPartitionName()}); + byte[] key = + new CompactedKeyEncoder( + tableInfo.getRowType(), + tableInfo.getSchema().getPrimaryKeyIndexes()) + .encodeKey(row); + CompletableFuture future = new CompletableFuture<>(); + accumulator.append( + WriteRecord.forUpsert( + tableInfo, + physicalTablePath, + row, + key, + key, + WriteFormat.COMPACTED_KV, + null), + (tableBucket, logEndOffset, error) -> future.complete(error), + cluster, + 0, + false); + return future; + } + + private CompletableFuture appendLogRecord( + TableInfo tableInfo, PhysicalTablePath physicalTablePath, int id, Cluster cluster) + throws Exception { + IndexedRow row = + indexedRow( + tableInfo.getRowType(), + new Object[] {id, physicalTablePath.getPartitionName()}); + CompletableFuture future = new CompletableFuture<>(); + accumulator.append( + WriteRecord.forIndexedAppend(tableInfo, physicalTablePath, row, null), + (tableBucket, logEndOffset, error) -> future.complete(error), + cluster, + 0, + false); + return future; + } + private void appendToAccumulator(TableBucket tb, GenericRow row, WriteCallback writeCallback) throws Exception { appendToAccumulator(DATA1_TABLE_INFO, tb, row, writeCallback); @@ -1169,6 +1593,11 @@ private void appendKvToAccumulator( false); } + private TestTabletServerGateway node1Gateway() { + return (TestTabletServerGateway) + metadataUpdater.newTabletServerClientForNode(TestingMetadataUpdater.NODE1.id()); + } + private ApiMessage getRequest(TableBucket tb, int index) { TestTabletServerGateway gateway = (TestTabletServerGateway) @@ -1261,6 +1690,14 @@ private PutKvResponse createPutKvResponse(TableBucket tb, long endOffset) { Collections.singletonList(new PutKvResultForBucket(tb, endOffset))); } + private PutKvResponse createHistoricalPutKvResponse( + TableBucket tb, long endOffset, String originalPartitionName) { + return makePutKvResponse( + Collections.singletonList( + PutKvResultForBucket.historicalSuccess( + tb, endOffset, originalPartitionName))); + } + private PutKvResponse createPutKvResponse(TableBucket tb, long endOffset, float pressure) { return makePutKvResponse( Collections.singletonList(new PutKvResultForBucket(tb, endOffset, pressure))); @@ -1309,14 +1746,24 @@ private IdempotenceManager createIdempotenceManager(boolean idempotenceEnabled) } private static boolean hasIdempotentRecords(TableBucket tb, ProduceLogRequest request) { - MemoryLogRecords memoryLogRecords = getProduceLogData(request).get(tb); + MemoryLogRecords memoryLogRecords = getProduceLogRecords(request, tb); return memoryLogRecords.batchIterator().next().writerId() != NO_WRITER_ID; } private static void assertBatchSequenceEquals( TableBucket tb, ProduceLogRequest request, int expectedBatchSequence) { - MemoryLogRecords memoryLogRecords = getProduceLogData(request).get(tb); + MemoryLogRecords memoryLogRecords = getProduceLogRecords(request, tb); assertThat(memoryLogRecords.batchIterator().next().batchSequence()) .isEqualTo(expectedBatchSequence); } + + private static MemoryLogRecords getProduceLogRecords( + ProduceLogRequest request, TableBucket tableBucket) { + for (ProduceLogDataForBucket bucketData : toProduceLogDataForBuckets(request)) { + if (bucketData.tableBucket().equals(tableBucket)) { + return bucketData.records(); + } + } + throw new IllegalArgumentException("No records found for table bucket " + tableBucket); + } } diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java index 16301f4612f..6685381cb0e 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java @@ -439,6 +439,14 @@ public class ConfigOptions { .withDescription( "The duration after which an idle historical partition table lookuper is removed from the cache."); + public static final ConfigOption SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME = + key("server.historical-partition.kv-cleanup.idle-time") + .durationType() + .defaultValue(Duration.ofMinutes(30)) + .withDescription( + "The historical KV write idle time after which a fully tiered local overlay can be cleaned. " + + "Set to 0 to disable idle cleanup."); + public static final ConfigOption SERVER_DATA_DISK_WRITE_LIMIT_RATIO = key("server.data-disk.write-limit-ratio") .doubleType() @@ -1925,11 +1933,12 @@ public class ConfigOptions { .booleanType() .defaultValue(false) .withDescription( - "Whether to enable historical partition lookup for the table. " + "Whether to enable historical partition access for the table. " + "When enabled, the coordinator creates and retains a system partition " - + "for routing lookups of expired partitions to lake storage. " - + "Currently, this option only supports auto-partitioned Paimon primary " - + "key tables with a single partition key. Disabled by default. " + + "for routing writes to expired partitions and, for primary-key tables, " + + "lookups of expired partitions to lake storage. Currently, this option " + + "only supports auto-partitioned Paimon tables with a single partition " + + "key. Disabled by default. " + "After changing this option, restart existing lookup jobs that need " + "to look up historical partition data so that their clients load the " + "updated table configuration."); diff --git a/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java b/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java index ea046e3b11d..d1b41983768 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java @@ -129,7 +129,7 @@ public boolean isDataLakeEnabled() { return config.get(ConfigOptions.TABLE_DATALAKE_ENABLED); } - /** Whether historical partition lookup is enabled. */ + /** Whether historical partition access is enabled. */ public boolean isHistoricalPartitionEnabled() { return config.get(ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_ENABLED); } diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/enumerator/TieringSourceEnumerator.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/enumerator/TieringSourceEnumerator.java index f1bf98527af..eb27397c9c6 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/enumerator/TieringSourceEnumerator.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/enumerator/TieringSourceEnumerator.java @@ -158,7 +158,7 @@ public void start() { this.coordinatorGateway = GatewayClientProxy.createGatewayProxy( metadataUpdater::getCoordinatorServer, rpcClient, CoordinatorGateway.class); - this.splitGenerator = new TieringSplitGenerator(flussAdmin); + this.splitGenerator = new TieringSplitGenerator(flussAdmin, metadataUpdater); LOG.info("Starting register Tiering Service to Fluss Coordinator..."); try { diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/split/TieringSplitGenerator.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/split/TieringSplitGenerator.java index a4b2638f309..fc2cbd9a03d 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/split/TieringSplitGenerator.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/split/TieringSplitGenerator.java @@ -22,8 +22,10 @@ import org.apache.fluss.client.initializer.OffsetsInitializer.BucketOffsetsRetriever; import org.apache.fluss.client.metadata.KvSnapshots; import org.apache.fluss.client.metadata.LakeSnapshot; +import org.apache.fluss.client.metadata.MetadataUpdater; import org.apache.fluss.exception.LakeTableSnapshotNotExistException; import org.apache.fluss.metadata.PartitionInfo; +import org.apache.fluss.metadata.PhysicalTablePath; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.metadata.TablePath; @@ -36,6 +38,7 @@ import javax.annotation.Nullable; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; @@ -43,6 +46,7 @@ import java.util.stream.IntStream; import static org.apache.fluss.client.table.scanner.log.LogScanner.EARLIEST_OFFSET; +import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; import static org.apache.fluss.utils.Preconditions.checkState; /** A generator for lake splits. */ @@ -51,9 +55,11 @@ public class TieringSplitGenerator { private static final Logger LOG = LoggerFactory.getLogger(TieringSplitGenerator.class); private final Admin flussAdmin; + private final MetadataUpdater metadataUpdater; - public TieringSplitGenerator(Admin flussAdmin) { + public TieringSplitGenerator(Admin flussAdmin, MetadataUpdater metadataUpdater) { this.flussAdmin = flussAdmin; + this.metadataUpdater = metadataUpdater; } public List generateTableSplits(TableInfo tableInfo) throws Exception { @@ -92,6 +98,21 @@ public List generateTableSplits(TableInfo tableInfo) throws Except Collectors.toMap( PartitionInfo::getPartitionId, PartitionInfo::getPartitionName)); + if (tableInfo.getTableConfig().isHistoricalPartitionEnabled()) { + // The internal historical partition is intentionally omitted from + // listPartitionInfos(), but tiering must consume it to synchronize historical + // writes to the lake table. Resolve it explicitly and include it in the splits. + PhysicalTablePath historicalPath = + PhysicalTablePath.of(tablePath, HISTORICAL_PARTITION_VALUE); + // Partition metadata is decoded using the tableId-to-path mapping already present + // in the Cluster, so initialize the table metadata before requesting the internal + // partition directly. + metadataUpdater.checkAndUpdateTableMetadata(Collections.singleton(tablePath)); + metadataUpdater.checkAndUpdatePartitionMetadata(historicalPath); + partitionNameById.put( + metadataUpdater.getPartitionIdOrElseThrow(historicalPath), + HISTORICAL_PARTITION_VALUE); + } return generatePartitionTableSplit( tableInfo, partitionNameById, bucketOffsetsRetriever, lakeSnapshotInfo); @@ -112,6 +133,7 @@ private List generatePartitionTableSplit( for (Map.Entry partitionNameByIdEntry : partitionNameById.entrySet()) { long partitionId = partitionNameByIdEntry.getKey(); String partitionName = partitionNameByIdEntry.getValue(); + boolean historicalPartition = HISTORICAL_PARTITION_VALUE.equals(partitionName); Map latestBucketsOffset = bucketOffsetsRetriever.latestOffsets( partitionName, @@ -119,7 +141,7 @@ private List generatePartitionTableSplit( .boxed() .collect(Collectors.toList())); KvSnapshots latestKvSnapshots = null; - if (tableInfo.hasPrimaryKey()) { + if (tableInfo.hasPrimaryKey() && !historicalPartition) { // get the table partition latest kv snapshot info try { latestKvSnapshots = @@ -134,6 +156,8 @@ private List generatePartitionTableSplit( ExceptionUtils.stripCompletionException(e)); } } + // Historical KV replicas do not create regular KV snapshots. Their lake snapshot is + // the durable base, so tier them from the retained WAL like log tables. splits.addAll( generateTableSplit( @@ -142,7 +166,8 @@ private List generatePartitionTableSplit( partitionName, lakeSnapshotInfo, latestKvSnapshots, - latestBucketsOffset)); + latestBucketsOffset, + historicalPartition)); } return splits; } @@ -172,7 +197,13 @@ private List generateNonPartitionedTableSplit( } return generateTableSplit( - tableInfo, null, null, lakeSnapshotInfo, latestKvSnapshots, latestBucketsOffset); + tableInfo, + null, + null, + lakeSnapshotInfo, + latestKvSnapshots, + latestBucketsOffset, + false); } private List generateTableSplit( @@ -181,10 +212,11 @@ private List generateTableSplit( @Nullable String partitionName, @Nullable LakeSnapshot lakeSnapshotInfo, @Nullable KvSnapshots latestKvSnapshots, - Map latestBucketsOffset) { + Map latestBucketsOffset, + boolean historicalPartition) { List splits = new ArrayList<>(); - if (tableInfo.hasPrimaryKey()) { + if (tableInfo.hasPrimaryKey() && !historicalPartition) { // it's primary key table checkState(latestKvSnapshots != null); for (int bucket = 0; bucket < tableInfo.getNumBuckets(); bucket++) { diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeCommitter.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeCommitter.java index c23cc373cda..7b16280eaf8 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeCommitter.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeCommitter.java @@ -35,6 +35,7 @@ import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.TableSnapshot; import org.apache.paimon.table.sink.CommitCallback; +import org.apache.paimon.table.sink.CommitMessage; import org.apache.paimon.table.sink.TableCommitImpl; import org.apache.paimon.utils.SnapshotManager; import org.slf4j.Logger; @@ -91,7 +92,9 @@ public PaimonCommittable toCommittable(List paimonWriteResult throws IOException { ManifestCommittable committable = new ManifestCommittable(COMMIT_IDENTIFIER); for (PaimonWriteResult paimonWriteResult : paimonWriteResults) { - committable.addFileCommittable(paimonWriteResult.commitMessage()); + for (CommitMessage commitMessage : paimonWriteResult.commitMessages()) { + committable.addFileCommittable(commitMessage); + } } return new PaimonCommittable(committable); } diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeWriter.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeWriter.java index 9b82403b6ce..a4bb0d44a41 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeWriter.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeWriter.java @@ -32,7 +32,6 @@ import org.apache.paimon.CoreOptions; import org.apache.paimon.catalog.Catalog; import org.apache.paimon.table.FileStoreTable; -import org.apache.paimon.table.sink.CommitMessage; import java.io.IOException; import java.util.Collections; @@ -40,6 +39,7 @@ import java.util.Map; import static org.apache.fluss.lake.paimon.utils.PaimonConversions.toPaimon; +import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; /** Implementation of {@link LakeWriter} for Paimon. */ public class PaimonLakeWriter implements LakeWriter, SupportsRecordBatchWrite { @@ -58,6 +58,8 @@ public PaimonLakeWriter( List partitionKeys = fileStoreTable.partitionKeys(); RowType flussRowType = writerInitContext.tableInfo().getRowType(); + boolean historicalPartition = + HISTORICAL_PARTITION_VALUE.equals(writerInitContext.partition()); // FIP-27: detect whether the target Paimon table is a clean table (only user columns) or a // legacy table (carrying the three Fluss system columns). Writers emit system columns only @@ -72,7 +74,8 @@ public PaimonLakeWriter( writerInitContext.partition(), partitionKeys, flussRowType, - paimonIncludingSystemColumns) + paimonIncludingSystemColumns, + historicalPartition) : new MergeTreeWriter( fileStoreTable, writerInitContext.tableBucket(), @@ -80,7 +83,8 @@ public PaimonLakeWriter( partitionKeys, flussRowType, writerInitContext.ioTmpDirs(), - paimonIncludingSystemColumns); + paimonIncludingSystemColumns, + historicalPartition); } @Override @@ -113,13 +117,11 @@ public void write(RecordBatch recordBatch) throws IOException { @Override public PaimonWriteResult complete() throws IOException { - CommitMessage commitMessage; try { - commitMessage = recordWriter.complete(); + return new PaimonWriteResult(recordWriter.complete()); } catch (Exception e) { throw new IOException("Failed to complete Paimon write.", e); } - return new PaimonWriteResult(commitMessage); } @Override diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonWriteResult.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonWriteResult.java index 70575c00e16..b2d40fbb25f 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonWriteResult.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonWriteResult.java @@ -20,19 +20,29 @@ import org.apache.paimon.table.sink.CommitMessage; import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; /** The write result of Paimon lake writer to pass to committer to commit. */ -public class PaimonWriteResult implements Serializable { +public final class PaimonWriteResult implements Serializable { private static final long serialVersionUID = 1L; - private final CommitMessage commitMessage; + private final List commitMessages; - public PaimonWriteResult(CommitMessage commitMessage) { - this.commitMessage = commitMessage; + /** Creates a write result containing all commit messages produced by one lake writer. */ + public PaimonWriteResult(List commitMessages) { + checkNotNull(commitMessages, "commitMessages must not be null"); + checkArgument(!commitMessages.isEmpty(), "commitMessages must not be empty"); + this.commitMessages = Collections.unmodifiableList(new ArrayList<>(commitMessages)); } - public CommitMessage commitMessage() { - return commitMessage; + /** Returns all commit messages produced by the lake writer. */ + public List commitMessages() { + return commitMessages; } } diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonWriteResultSerializer.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonWriteResultSerializer.java index 7efb3d37548..3ffecc9855d 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonWriteResultSerializer.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonWriteResultSerializer.java @@ -19,10 +19,14 @@ import org.apache.fluss.lake.serializer.SimpleVersionedSerializer; +import org.apache.paimon.io.DataInputViewStreamWrapper; +import org.apache.paimon.io.DataOutputSerializer; import org.apache.paimon.table.sink.CommitMessage; import org.apache.paimon.table.sink.CommitMessageSerializer; +import java.io.ByteArrayInputStream; import java.io.IOException; +import java.util.List; /** The {@link SimpleVersionedSerializer} for {@link PaimonWriteResult}. */ public class PaimonWriteResultSerializer implements SimpleVersionedSerializer { @@ -38,8 +42,9 @@ public int getVersion() { @Override public byte[] serialize(PaimonWriteResult paimonWriteResult) throws IOException { - CommitMessage commitMessage = paimonWriteResult.commitMessage(); - return messageSer.serialize(commitMessage); + DataOutputSerializer output = new DataOutputSerializer(64); + messageSer.serializeList(paimonWriteResult.commitMessages(), output); + return output.getCopyOfBuffer(); } @Override @@ -52,7 +57,11 @@ public PaimonWriteResult deserialize(int version, byte[] serialized) throws IOEx + version + "."); } - CommitMessage commitMessage = messageSer.deserialize(messageSer.getVersion(), serialized); - return new PaimonWriteResult(commitMessage); + try (DataInputViewStreamWrapper input = + new DataInputViewStreamWrapper(new ByteArrayInputStream(serialized))) { + List commitMessages = + messageSer.deserializeList(messageSer.getVersion(), input); + return new PaimonWriteResult(commitMessages); + } } } diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/RecordWriter.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/RecordWriter.java index 2260d553bc9..6e762a96141 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/RecordWriter.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/RecordWriter.java @@ -31,6 +31,7 @@ import java.util.List; import static org.apache.fluss.lake.paimon.utils.PaimonConversions.toPaimonPartition; +import static org.apache.fluss.utils.Preconditions.checkNotNull; import static org.apache.fluss.utils.Preconditions.checkState; /** A base interface to write {@link LogRecord} to Paimon. */ @@ -40,7 +41,8 @@ public abstract class RecordWriter implements AutoCloseable { protected final RowType tableRowType; protected final int bucket; protected final List partitionKeys; - protected final BinaryRow partition; + protected final boolean historicalPartition; + protected final @Nullable BinaryRow fixedPartition; protected final FlussRecordAsPaimonRow flussRecordAsPaimonRow; public RecordWriter( @@ -50,17 +52,21 @@ public RecordWriter( @Nullable String partition, List partitionKeys, org.apache.fluss.types.RowType flussRowType, - boolean paimonIncludingSystemColumns) { + boolean paimonIncludingSystemColumns, + boolean historicalPartition) { this.tableWrite = tableWrite; this.tableRowType = tableRowType; this.bucket = tableBucket.getBucket(); this.partitionKeys = partitionKeys; - if (partition == null || partitionKeys.isEmpty()) { + this.historicalPartition = historicalPartition; + if (historicalPartition) { + this.fixedPartition = null; + } else if (partition == null || partitionKeys.isEmpty()) { // non-partitioned table - this.partition = BinaryRow.EMPTY_ROW; + this.fixedPartition = BinaryRow.EMPTY_ROW; } else { // eagerly resolve BinaryRow partition from partition name string - this.partition = resolvePartition(partition, partitionKeys, flussRowType); + this.fixedPartition = resolvePartition(partition, partitionKeys, flussRowType); } this.flussRecordAsPaimonRow = new FlussRecordAsPaimonRow( @@ -69,19 +75,31 @@ public RecordWriter( public abstract void write(LogRecord record) throws Exception; - CommitMessage complete() throws Exception { + List complete() throws Exception { List commitMessages = tableWrite.prepareCommit(); - checkState( - commitMessages.size() == 1, - "The size of CommitMessage must be 1, but got %s.", - commitMessages); - return commitMessages.get(0); + // A normal writer targets one fixed partition, while a historical writer may write to + // multiple original partitions and therefore produce multiple commit messages. + if (!historicalPartition) { + checkState( + commitMessages.size() == 1, + "The size of CommitMessage must be 1, but got %s.", + commitMessages); + } + return commitMessages; } public void close() throws Exception { tableWrite.close(); } + /** Sets the current Fluss record and returns the Paimon partition it should be written to. */ + protected BinaryRow prepareRecordAndGetPartition(LogRecord record) { + flussRecordAsPaimonRow.setFlussRecord(record); + return historicalPartition + ? tableWrite.getPartition(flussRecordAsPaimonRow) + : checkNotNull(fixedPartition); + } + /** * Resolves a Paimon {@link BinaryRow} partition from the partition name string by parsing each * partition value to its typed Fluss representation, constructing a synthetic row, and diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyArrowBatchHelper.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyArrowBatchHelper.java index ac3b75cad3b..83518179ddf 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyArrowBatchHelper.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyArrowBatchHelper.java @@ -44,6 +44,8 @@ import java.util.ArrayList; import java.util.List; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + /** * Helper class that encapsulates Arrow-dependent batch writing logic for append-only tables. * @@ -106,7 +108,11 @@ class AppendOnlyArrowBatchHelper implements AutoCloseable { * system columns (__bucket, __offset, __timestamp) and uses Paimon's {@link ArrowBundleRecords} * for efficient batch writing. */ - void writeArrowBatch(ArrowBatchData arrowBatchData, BinaryRow partition) throws Exception { + void writeArrowBatch( + ArrowBatchData arrowBatchData, + @Nullable BinaryRow fixedPartition, + boolean historicalPartition) + throws Exception { int writtenBucket = bucket; if (fileStoreTable.store().bucketMode() == BucketMode.BUCKET_UNAWARE) { writtenBucket = 0; @@ -119,7 +125,7 @@ void writeArrowBatch(ArrowBatchData arrowBatchData, BinaryRow partition) throws // the Paimon table schema. Write it directly without enriching system columns. ArrowBundleRecords cleanRecords = new ArrowBundleRecords(originalRoot, tableRowType, CASE_SENSITIVE); - tableWrite.writeBundle(partition, writtenBucket, cleanRecords); + writeArrowBundle(cleanRecords, fixedPartition, historicalPartition, writtenBucket); return; } @@ -133,7 +139,25 @@ void writeArrowBatch(ArrowBatchData arrowBatchData, BinaryRow partition) throws ArrowBundleRecords arrowBundleRecords = new ArrowBundleRecords(enrichedRoot, tableRowType, CASE_SENSITIVE); - tableWrite.writeBundle(partition, writtenBucket, arrowBundleRecords); + writeArrowBundle(arrowBundleRecords, fixedPartition, historicalPartition, writtenBucket); + } + + private void writeArrowBundle( + ArrowBundleRecords arrowBundleRecords, + @Nullable BinaryRow fixedPartition, + boolean historicalPartition, + int writtenBucket) + throws Exception { + if (historicalPartition) { + // writeBundle accepts one fixed partition, but a historical batch may contain rows + // from multiple original partitions. + for (InternalRow row : arrowBundleRecords) { + BinaryRow partition = tableWrite.getPartition(row); + tableWrite.getWrite().write(partition, writtenBucket, row); + } + } else { + tableWrite.writeBundle(checkNotNull(fixedPartition), writtenBucket, arrowBundleRecords); + } } /** diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyWriter.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyWriter.java index 23f61a33171..a6d4f4292fd 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyWriter.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyWriter.java @@ -23,6 +23,7 @@ import org.apache.fluss.record.LogRecord; import org.apache.fluss.types.RowType; +import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.table.BucketMode; import org.apache.paimon.table.FileStoreTable; @@ -54,7 +55,8 @@ public AppendOnlyWriter( @Nullable String partition, List partitionKeys, RowType flussRowType, - boolean paimonIncludingSystemColumns) { + boolean paimonIncludingSystemColumns, + boolean historicalPartition) { //noinspection unchecked super( (TableWriteImpl) @@ -65,14 +67,15 @@ public AppendOnlyWriter( partition, partitionKeys, flussRowType, - paimonIncludingSystemColumns); + paimonIncludingSystemColumns, + historicalPartition); this.fileStoreTable = fileStoreTable; this.paimonIncludingSystemColumns = paimonIncludingSystemColumns; } @Override public void write(LogRecord record) throws Exception { - flussRecordAsPaimonRow.setFlussRecord(record); + BinaryRow targetPartition = prepareRecordAndGetPartition(record); // hacky, call internal method tableWrite.getWrite() to support // to write to given partition, otherwise, it'll always extract a partition from Paimon row @@ -82,7 +85,7 @@ public void write(LogRecord record) throws Exception { if (fileStoreTable.store().bucketMode() == BucketMode.BUCKET_UNAWARE) { writtenBucket = 0; } - tableWrite.getWrite().write(partition, writtenBucket, flussRecordAsPaimonRow); + tableWrite.getWrite().write(targetPartition, writtenBucket, flussRecordAsPaimonRow); } /** @@ -104,7 +107,7 @@ public void writeArrowBatch(ArrowBatchData arrowBatchData) throws Exception { } else { helper = (AppendOnlyArrowBatchHelper) arrowBatchHelper; } - helper.writeArrowBatch(arrowBatchData, partition); + helper.writeArrowBatch(arrowBatchData, fixedPartition, historicalPartition); } @Override diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/mergetree/MergeTreeWriter.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/mergetree/MergeTreeWriter.java index 37aeef7afe6..7d48c850e9a 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/mergetree/MergeTreeWriter.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/mergetree/MergeTreeWriter.java @@ -23,6 +23,7 @@ import org.apache.fluss.types.RowType; import org.apache.paimon.KeyValue; +import org.apache.paimon.data.BinaryRow; import org.apache.paimon.disk.IOManager; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.sink.RowKeyExtractor; @@ -58,7 +59,8 @@ public MergeTreeWriter( partitionKeys, flussRowType, (String[]) null, - paimonIncludingSystemColumns); + paimonIncludingSystemColumns, + false); } public MergeTreeWriter( @@ -68,7 +70,8 @@ public MergeTreeWriter( List partitionKeys, RowType flussRowType, @Nullable String[] ioTmpDirs, - boolean paimonIncludingSystemColumns) { + boolean paimonIncludingSystemColumns, + boolean historicalPartition) { this( fileStoreTable, createIOManager(ioTmpDirs), @@ -76,7 +79,8 @@ public MergeTreeWriter( partition, partitionKeys, flussRowType, - paimonIncludingSystemColumns); + paimonIncludingSystemColumns, + historicalPartition); } MergeTreeWriter( @@ -86,7 +90,8 @@ public MergeTreeWriter( @Nullable String partition, List partitionKeys, RowType flussRowType, - boolean paimonIncludingSystemColumns) { + boolean paimonIncludingSystemColumns, + boolean historicalPartition) { super( createTableWrite(fileStoreTable, ioManager), fileStoreTable.rowType(), @@ -94,7 +99,8 @@ public MergeTreeWriter( partition, partitionKeys, flussRowType, - paimonIncludingSystemColumns); + paimonIncludingSystemColumns, + historicalPartition); this.rowKeyExtractor = fileStoreTable.createRowKeyExtractor(); this.ioManager = ioManager; } @@ -128,7 +134,7 @@ public void close() throws Exception { @Override public void write(LogRecord record) throws Exception { - flussRecordAsPaimonRow.setFlussRecord(record); + BinaryRow targetPartition = prepareRecordAndGetPartition(record); rowKeyExtractor.setRecord(flussRecordAsPaimonRow); keyValue.replace( @@ -139,6 +145,6 @@ public void write(LogRecord record) throws Exception { // hacky, call internal method tableWrite.getWrite() to support // to write to given partition, otherwise, it'll always extract a partition from Paimon row // which may be costly - tableWrite.getWrite().write(partition, bucket, keyValue); + tableWrite.getWrite().write(targetPartition, bucket, keyValue); } } diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/HistoricalPartitionLookupITCase.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/HistoricalPartitionITCase.java similarity index 64% rename from fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/HistoricalPartitionLookupITCase.java rename to fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/HistoricalPartitionITCase.java index 091c72bc21b..a135d520162 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/HistoricalPartitionLookupITCase.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/HistoricalPartitionITCase.java @@ -38,12 +38,15 @@ import org.apache.fluss.types.DataTypes; import org.apache.flink.core.execution.JobClient; +import org.apache.paimon.utils.CloseableIterator; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; import java.time.Duration; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -56,8 +59,8 @@ import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; import static org.assertj.core.api.Assertions.assertThat; -/** End-to-end IT case for looking up expired Fluss partitions from Paimon. */ -class HistoricalPartitionLookupITCase extends FlinkPaimonTieringTestBase { +/** End-to-end IT case for historical partition writes, tiering, recovery, and lookup. */ +class HistoricalPartitionITCase extends FlinkPaimonTieringTestBase { private static final String EXPIRED_PARTITION_NAME = "20240101"; private static final String SECOND_EXPIRED_PARTITION_NAME = "20240102"; @@ -76,6 +79,87 @@ protected static void beforeAll() { FlinkPaimonTieringTestBase.beforeAll(FLUSS_CLUSTER_EXTENSION.getClientConfig()); } + @Test + void testWriteAndTierHistoricalKvToPaimon() throws Exception { + TablePath tablePath = TablePath.of(DEFAULT_DB, "historical_write_tiering"); + Schema schema = partitionedPkSchema(true); + long tableId = + createTable( + tablePath, + partitionedPkDescriptor(schema, true, EXPIRED_PARTITION_RETENTION)); + + try { + long historicalPartitionId = waitUntilHistoricalPartitionReady(tablePath, tableId); + + InternalRow expectedRow = dataRow(true, 1, "unused", "Alice"); + assertThat(admin.listPartitionInfos(tablePath).get()) + .noneMatch(p -> EXPIRED_PARTITION_NAME.equals(p.getPartitionName())); + writeRows(tablePath, Collections.singletonList(expectedRow), false); + // Historical writes must not recreate the expired original partition. + assertThat(admin.listPartitionInfos(tablePath).get()) + .noneMatch(p -> EXPIRED_PARTITION_NAME.equals(p.getPartitionName())); + + TableBucket historicalBucket = new TableBucket(tableId, historicalPartitionId, 0); + assertThat(getLeaderReplica(historicalBucket).getLocalLogEndOffset()).isEqualTo(1); + JobClient jobClient = buildTieringJob(execEnv); + try { + assertReplicaStatus(historicalBucket, 1); + checkFlussOffsetsInSnapshot( + tablePath, Collections.singletonMap(historicalBucket, 1L)); + assertThat(readPaimonRows(tablePath)) + .containsExactly("1|" + EXPIRED_PARTITION_NAME + "|Alice"); + } finally { + jobClient.cancel().get(); + } + + restartLeaderAndVerifyLookup(tablePath, historicalBucket, schema, expectedRow); + } finally { + dropTable(tablePath); + } + } + + @Test + void testWriteAndTierHistoricalLogToPaimon() throws Exception { + TablePath tablePath = TablePath.of(DEFAULT_DB, "historical_log_write_tiering"); + Schema schema = partitionedLogSchema(); + long tableId = createTable(tablePath, partitionedLogDescriptor(schema)); + + try { + long historicalPartitionId = waitUntilHistoricalPartitionReady(tablePath, tableId); + + List expectedRows = + Arrays.asList( + row(1, EXPIRED_PARTITION_NAME, "Alice"), + row(2, SECOND_EXPIRED_PARTITION_NAME, "Bob")); + writeRows(tablePath, expectedRows, true); + assertThat(admin.listPartitionInfos(tablePath).get()) + .noneMatch( + partitionInfo -> + EXPIRED_PARTITION_NAME.equals(partitionInfo.getPartitionName()) + || SECOND_EXPIRED_PARTITION_NAME.equals( + partitionInfo.getPartitionName())); + + TableBucket historicalBucket = new TableBucket(tableId, historicalPartitionId, 0); + assertThat(getLeaderReplica(historicalBucket).getLocalLogEndOffset()).isEqualTo(2); + + JobClient jobClient = buildTieringJob(execEnv); + try { + assertReplicaStatus(historicalBucket, 2); + checkFlussOffsetsInSnapshot( + tablePath, Collections.singletonMap(historicalBucket, 2L)); + + assertThat(readPaimonRows(tablePath)) + .containsExactlyInAnyOrder( + "1|" + EXPIRED_PARTITION_NAME + "|Alice", + "2|" + SECOND_EXPIRED_PARTITION_NAME + "|Bob"); + } finally { + jobClient.cancel().get(); + } + } finally { + dropTable(tablePath); + } + } + @ParameterizedTest(name = "defaultBucketKey={0}") @ValueSource(booleans = {true, false}) void testLookupExpiredPartitionFromPaimon(boolean defaultBucketKey) throws Exception { @@ -86,7 +170,7 @@ void testLookupExpiredPartitionFromPaimon(boolean defaultBucketKey) throws Excep ? "historical_lookup_default_bucket" : "historical_lookup_bucket_subset"); Schema oldSchema = partitionedPkSchema(defaultBucketKey); - long tableId = createTable(tablePath, partitionedPkDescriptor(oldSchema)); + long tableId = createTable(tablePath, partitionedPkDescriptor(oldSchema, false)); // Enable historical lookup through ALTER TABLE to cover dynamic creation of the // coordinator-owned historical system partition. @@ -232,6 +316,59 @@ protected FlussClusterExtension getFlussClusterExtension() { return FLUSS_CLUSTER_EXTENSION; } + private static long waitUntilHistoricalPartitionReady(TablePath tablePath, long tableId) + throws Exception { + Optional historicalPartition = + FLUSS_CLUSTER_EXTENSION + .getZooKeeperClient() + .getPartition(tablePath, HISTORICAL_PARTITION_VALUE); + assertThat(historicalPartition).isPresent(); + long partitionId = historicalPartition.get().getPartitionId(); + FLUSS_CLUSTER_EXTENSION.waitUntilTablePartitionReady(tableId, partitionId); + return partitionId; + } + + private List readPaimonRows(TablePath tablePath) throws Exception { + List actualRows = new ArrayList<>(); + try (CloseableIterator rows = + getPaimonRowCloseableIterator(tablePath)) { + while (rows.hasNext()) { + org.apache.paimon.data.InternalRow row = rows.next(); + actualRows.add(row.getInt(0) + "|" + row.getString(1) + "|" + row.getString(2)); + } + } + return actualRows; + } + + private void restartLeaderAndVerifyLookup( + TablePath tablePath, + TableBucket historicalBucket, + Schema schema, + InternalRow expectedRow) + throws Exception { + int tabletServerId = FLUSS_CLUSTER_EXTENSION.waitAndGetLeader(historicalBucket); + FLUSS_CLUSTER_EXTENSION.stopTabletServer(tabletServerId); + try { + FLUSS_CLUSTER_EXTENSION.startTabletServer(tabletServerId); + FLUSS_CLUSTER_EXTENSION.waitAndGetLeaderReplica(historicalBucket); + + try (Connection connection = ConnectionFactory.createConnection(clientConf); + Table table = connection.getTable(tablePath)) { + InternalRow actualRow = + table.newLookup() + .createLookuper() + .lookup(lookupKey(true, 1, "unused")) + .get() + .getSingletonRow(); + assertThatRow(actualRow).withSchema(schema.getRowType()).isEqualTo(expectedRow); + } + } finally { + if (FLUSS_CLUSTER_EXTENSION.getTabletServerById(tabletServerId) == null) { + FLUSS_CLUSTER_EXTENSION.startTabletServer(tabletServerId); + } + } + } + private static Schema partitionedPkSchema(boolean defaultBucketKey) { if (defaultBucketKey) { return Schema.newBuilder() @@ -250,6 +387,14 @@ private static Schema partitionedPkSchema(boolean defaultBucketKey) { .build(); } + private static Schema partitionedLogSchema() { + return Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("dt", DataTypes.STRING()) + .column("name", DataTypes.STRING()) + .build(); + } + private static Schema evolvedPartitionedPkSchema(boolean defaultBucketKey) { if (defaultBucketKey) { return Schema.newBuilder() @@ -270,11 +415,41 @@ private static Schema evolvedPartitionedPkSchema(boolean defaultBucketKey) { .build(); } - private static TableDescriptor partitionedPkDescriptor(Schema schema) { + private static TableDescriptor partitionedPkDescriptor( + Schema schema, boolean historicalPartitionEnabled) { + return partitionedPkDescriptor( + schema, historicalPartitionEnabled, INITIAL_PARTITION_RETENTION); + } + + private static TableDescriptor partitionedPkDescriptor( + Schema schema, boolean historicalPartitionEnabled, int partitionRetention) { + TableDescriptor.Builder builder = + TableDescriptor.builder() + .schema(schema) + // This is the default bucket key for (id, dt), and a strict subset of the + // physical primary key for (id, sub_id, dt). + .distributedBy(1, "id") + .partitionedBy("dt") + .property(ConfigOptions.TABLE_AUTO_PARTITION_ENABLED, true) + .property(ConfigOptions.TABLE_AUTO_PARTITION_KEY, "dt") + .property( + ConfigOptions.TABLE_AUTO_PARTITION_TIME_UNIT, + AutoPartitionTimeUnit.DAY) + .property( + ConfigOptions.TABLE_AUTO_PARTITION_NUM_RETENTION, + partitionRetention) + .property(ConfigOptions.TABLE_AUTO_PARTITION_TIMEZONE, "UTC") + .property(ConfigOptions.TABLE_DATALAKE_ENABLED, true) + .property(ConfigOptions.TABLE_DATALAKE_FRESHNESS, Duration.ofMillis(500)); + if (historicalPartitionEnabled) { + builder.property(ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_ENABLED, true); + } + return builder.build(); + } + + private static TableDescriptor partitionedLogDescriptor(Schema schema) { return TableDescriptor.builder() .schema(schema) - // This is the default bucket key for (id, dt), and a strict subset of the physical - // primary key for (id, sub_id, dt). .distributedBy(1, "id") .partitionedBy("dt") .property(ConfigOptions.TABLE_AUTO_PARTITION_ENABLED, true) @@ -282,10 +457,11 @@ private static TableDescriptor partitionedPkDescriptor(Schema schema) { .property(ConfigOptions.TABLE_AUTO_PARTITION_TIME_UNIT, AutoPartitionTimeUnit.DAY) .property( ConfigOptions.TABLE_AUTO_PARTITION_NUM_RETENTION, - INITIAL_PARTITION_RETENTION) + EXPIRED_PARTITION_RETENTION) .property(ConfigOptions.TABLE_AUTO_PARTITION_TIMEZONE, "UTC") .property(ConfigOptions.TABLE_DATALAKE_ENABLED, true) .property(ConfigOptions.TABLE_DATALAKE_FRESHNESS, Duration.ofMillis(500)) + .property(ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_ENABLED, true) .build(); } diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringTest.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringTest.java index a433af06c82..e689778cf12 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringTest.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringTest.java @@ -17,25 +17,34 @@ package org.apache.fluss.lake.paimon.tiering; +import org.apache.fluss.config.AutoPartitionTimeUnit; import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; +import org.apache.fluss.lake.batch.ArrowRecordBatch; import org.apache.fluss.lake.committer.CommittedLakeSnapshot; import org.apache.fluss.lake.committer.CommitterInitContext; import org.apache.fluss.lake.committer.LakeCommitter; import org.apache.fluss.lake.serializer.SimpleVersionedSerializer; import org.apache.fluss.lake.writer.LakeWriter; +import org.apache.fluss.lake.writer.SupportsRecordBatchWrite; import org.apache.fluss.lake.writer.WriterInitContext; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.record.ArrowBatchData; import org.apache.fluss.record.ChangeType; import org.apache.fluss.record.GenericRecord; import org.apache.fluss.record.LogRecord; import org.apache.fluss.row.BinaryString; import org.apache.fluss.row.GenericRow; +import org.apache.fluss.utils.UnshadedArrowReadUtils; import org.apache.fluss.utils.types.Tuple2; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.paimon.CoreOptions; import org.apache.paimon.catalog.Catalog; import org.apache.paimon.catalog.CatalogContext; @@ -58,11 +67,13 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; import javax.annotation.Nullable; import java.io.File; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -82,6 +93,7 @@ import static org.apache.fluss.record.ChangeType.UPDATE_AFTER; import static org.apache.fluss.record.ChangeType.UPDATE_BEFORE; import static org.apache.fluss.record.TestData.DEFAULT_REMOTE_DATA_DIR; +import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; import static org.assertj.core.api.Assertions.assertThat; /** The UT for tiering to Paimon via {@link PaimonLakeTieringFactory}. */ @@ -212,6 +224,76 @@ void testTieringWriteTable(boolean isPrimaryKeyTable, boolean isPartitioned) thr } } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testHistoricalPartitionTiering(boolean isPrimaryKeyTable) throws Exception { + TablePath tablePath = + TablePath.of( + "paimon", "test_historical_" + (isPrimaryKeyTable ? "primary_key" : "log")); + TableInfo tableInfo = createHistoricalTable(tablePath, isPrimaryKeyTable); + long timestamp = 1_000L; + List records = + Arrays.asList( + historicalRecord( + 0L, timestamp, 1, "partition-1", "20240101", isPrimaryKeyTable), + historicalRecord( + 1L, timestamp, 1, "partition-2", "20240102", isPrimaryKeyTable)); + + PaimonWriteResult writeResult; + try (LakeWriter lakeWriter = + createLakeWriter(tablePath, 0, HISTORICAL_PARTITION_VALUE, 1L, tableInfo)) { + for (LogRecord record : records) { + lakeWriter.write(record); + } + writeResult = lakeWriter.complete(); + } + + assertThat(writeResult.commitMessages()).hasSize(2); + SimpleVersionedSerializer serializer = + paimonLakeTieringFactory.getWriteResultSerializer(); + assertThat(serializer.getVersion()).isEqualTo(1); + byte[] serialized = serializer.serialize(writeResult); + writeResult = serializer.deserialize(serializer.getVersion(), serialized); + assertThat(writeResult.commitMessages()).hasSize(2); + + commitWriteResults(tablePath, tableInfo, Collections.singletonList(writeResult)); + verifyHistoricalRecords(tablePath, isPrimaryKeyTable, records); + } + + @Test + void testHistoricalArrowBatchTiering() throws Exception { + TablePath tablePath = TablePath.of("paimon", "test_historical_arrow"); + TableInfo tableInfo = createHistoricalTable(tablePath, false); + long baseOffset = 10L; + long timestamp = 1_000L; + List records = + Arrays.asList( + historicalRecord( + baseOffset, timestamp, 1, "partition-1", "20240101", false), + historicalRecord( + baseOffset + 1, timestamp, 2, "partition-2", "20240102", false)); + + PaimonWriteResult writeResult; + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE); + LakeWriter lakeWriter = + createLakeWriter(tablePath, 0, HISTORICAL_PARTITION_VALUE, 1L, tableInfo)) { + VectorSchemaRoot root = + VectorSchemaRoot.create( + UnshadedArrowReadUtils.toArrowSchema(tableInfo.getRowType()), + allocator); + try (ArrowRecordBatch arrowRecordBatch = + new ArrowRecordBatch(new ArrowBatchData(root, baseOffset, timestamp, 1))) { + writeArrowRows(root, records); + ((SupportsRecordBatchWrite) lakeWriter).write(arrowRecordBatch); + } + writeResult = lakeWriter.complete(); + } + + assertThat(writeResult.commitMessages()).hasSize(2); + commitWriteResults(tablePath, tableInfo, Collections.singletonList(writeResult)); + verifyHistoricalRecords(tablePath, false, records); + } + @Test void testEmptyCommitCreatesSnapshot() throws Exception { TablePath tablePath = TablePath.of("paimon", "test_empty_commit"); @@ -593,6 +675,23 @@ private void verifyLogTableRecordsThreePartition( actualRecords.close(); } + private void verifyHistoricalRecords( + TablePath tablePath, boolean isPrimaryKeyTable, List records) + throws Exception { + List partitions = Arrays.asList("20240101", "20240102"); + assertThat(paimonCatalog.listPartitions(toPaimon(tablePath))) + .extracting(partition -> partition.spec().get("c3")) + .containsExactlyInAnyOrderElementsOf(partitions); + for (int i = 0; i < partitions.size(); i++) { + String partition = partitions.get(i); + verifyTableRecords( + getPaimonRows(tablePath, partition, isPrimaryKeyTable, 0), + Collections.singletonList(records.get(i)), + 0, + partition); + } + } + private void verifyTableRecords( CloseableIterator actualRecords, List expectRecords, @@ -737,6 +836,36 @@ private GenericRecord toRecord(long offset, GenericRow row, ChangeType changeTyp return new GenericRecord(offset, System.currentTimeMillis(), changeType, row); } + private LogRecord historicalRecord( + long offset, + long timestamp, + int key, + String value, + String partition, + boolean isPrimaryKeyTable) { + GenericRow row = new GenericRow(3); + row.setField(0, key); + row.setField(1, BinaryString.fromString(value)); + row.setField(2, BinaryString.fromString(partition)); + return new GenericRecord( + offset, timestamp, isPrimaryKeyTable ? INSERT : ChangeType.APPEND_ONLY, row); + } + + private void writeArrowRows(VectorSchemaRoot root, List records) { + root.allocateNew(); + IntVector keyVector = (IntVector) root.getVector("c1"); + VarCharVector valueVector = (VarCharVector) root.getVector("c2"); + VarCharVector partitionVector = (VarCharVector) root.getVector("c3"); + for (int i = 0; i < records.size(); i++) { + org.apache.fluss.row.InternalRow row = records.get(i).getRow(); + keyVector.setSafe(i, row.getInt(0)); + valueVector.setSafe(i, row.getString(1).toString().getBytes(StandardCharsets.UTF_8)); + partitionVector.setSafe( + i, row.getString(2).toString().getBytes(StandardCharsets.UTF_8)); + } + root.setRowCount(records.size()); + } + private CloseableIterator getPaimonRows( TablePath tablePath, @Nullable String partition, boolean isPrimaryKeyTable, int bucket) throws Exception { @@ -911,6 +1040,52 @@ private void createTable( doCreatePaimonTable(tablePath, builder); } + private TableInfo createHistoricalTable(TablePath tablePath, boolean isPrimaryKeyTable) + throws Exception { + createTable( + tablePath, + isPrimaryKeyTable, + true, + isPrimaryKeyTable ? 1 : null, + Collections.emptyMap()); + + org.apache.fluss.metadata.Schema.Builder schemaBuilder = + org.apache.fluss.metadata.Schema.newBuilder() + .column("c1", org.apache.fluss.types.DataTypes.INT()) + .column("c2", org.apache.fluss.types.DataTypes.STRING()) + .column("c3", org.apache.fluss.types.DataTypes.STRING()); + if (isPrimaryKeyTable) { + schemaBuilder.primaryKey("c1", "c3"); + } + TableDescriptor descriptor = + TableDescriptor.builder() + .schema(schemaBuilder.build()) + .partitionedBy("c3") + .distributedBy(1) + .property(ConfigOptions.TABLE_DATALAKE_ENABLED, true) + .property(ConfigOptions.TABLE_AUTO_PARTITION_ENABLED, true) + .property(ConfigOptions.TABLE_AUTO_PARTITION_KEY, "c3") + .property( + ConfigOptions.TABLE_AUTO_PARTITION_TIME_UNIT, + AutoPartitionTimeUnit.DAY) + .build(); + return TableInfo.of(tablePath, 0, 1, descriptor, DEFAULT_REMOTE_DATA_DIR, 1L, 1L); + } + + private void commitWriteResults( + TablePath tablePath, TableInfo tableInfo, List writeResults) + throws Exception { + try (LakeCommitter committer = + createLakeCommitter(tablePath, tableInfo, new Configuration())) { + PaimonCommittable committable = committer.toCommittable(writeResults); + assertThat( + committer + .commit(committable, Collections.emptyMap()) + .getCommittedSnapshotId()) + .isOne(); + } + } + private void createMultiPartitionTable(TablePath tablePath) throws Exception { Schema.Builder builder = Schema.newBuilder() diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/ProduceLogResultForBucket.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/ProduceLogResultForBucket.java index 37efad6c646..98812e97447 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/ProduceLogResultForBucket.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/ProduceLogResultForBucket.java @@ -23,35 +23,68 @@ import org.apache.fluss.rpc.protocol.ApiError; import org.apache.fluss.rpc.protocol.Errors; +import javax.annotation.Nullable; + +import java.util.Objects; + /** Result of {@link ProduceLogRequest} for each table bucket. */ @Internal public class ProduceLogResultForBucket extends WriteResultForBucket { private final long baseOffset; + private final @Nullable String originalPartitionName; public ProduceLogResultForBucket(TableBucket tableBucket, long baseOffset, long endOffset) { - this(tableBucket, baseOffset, endOffset, ApiError.NONE); + this(tableBucket, baseOffset, endOffset, ApiError.NONE, null); } public ProduceLogResultForBucket(TableBucket tableBucket, ApiError error) { - this(tableBucket, -1L, -1L, error); + this(tableBucket, -1L, -1L, error, null); + } + + public static ProduceLogResultForBucket historicalSuccess( + TableBucket tableBucket, + long baseOffset, + long endOffset, + String originalPartitionName) { + return new ProduceLogResultForBucket( + tableBucket, baseOffset, endOffset, ApiError.NONE, originalPartitionName); + } + + public static ProduceLogResultForBucket historicalFailure( + TableBucket tableBucket, ApiError error, String originalPartitionName) { + return new ProduceLogResultForBucket(tableBucket, -1L, -1L, error, originalPartitionName); } private ProduceLogResultForBucket( - TableBucket tableBucket, long baseOffset, long endOffset, ApiError error) { + TableBucket tableBucket, + long baseOffset, + long endOffset, + ApiError error, + @Nullable String originalPartitionName) { super(tableBucket, endOffset, error); this.baseOffset = baseOffset; + this.originalPartitionName = originalPartitionName; } public long getBaseOffset() { return baseOffset; } + /** Returns the original partition name for a historical write, or null for a normal write. */ + public @Nullable String getOriginalPartitionName() { + return originalPartitionName; + } + @Override public T copy(Errors newError) { //noinspection unchecked return (T) new ProduceLogResultForBucket( - tableBucket, baseOffset, getWriteLogEndOffset(), newError.toApiError()); + tableBucket, + baseOffset, + getWriteLogEndOffset(), + newError.toApiError(), + originalPartitionName); } @Override @@ -66,6 +99,12 @@ public boolean equals(Object o) { return false; } ProduceLogResultForBucket that = (ProduceLogResultForBucket) o; - return baseOffset == that.baseOffset; + return baseOffset == that.baseOffset + && Objects.equals(originalPartitionName, that.originalPartitionName); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), baseOffset, originalPartitionName); } } diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/client/ServerConnection.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/client/ServerConnection.java index 46ec0806110..f28cf1e9b76 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/client/ServerConnection.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/client/ServerConnection.java @@ -25,11 +25,14 @@ import org.apache.fluss.exception.InvalidServerTypeException; import org.apache.fluss.exception.NetworkException; import org.apache.fluss.exception.RetriableAuthenticationException; +import org.apache.fluss.exception.UnsupportedVersionException; import org.apache.fluss.rpc.messages.ApiMessage; import org.apache.fluss.rpc.messages.ApiVersionsRequest; import org.apache.fluss.rpc.messages.ApiVersionsResponse; import org.apache.fluss.rpc.messages.AuthenticateRequest; import org.apache.fluss.rpc.messages.AuthenticateResponse; +import org.apache.fluss.rpc.messages.ProduceLogRequest; +import org.apache.fluss.rpc.messages.PutKvRequest; import org.apache.fluss.rpc.metrics.ClientMetricGroup; import org.apache.fluss.rpc.metrics.ConnectionMetrics; import org.apache.fluss.rpc.protocol.ApiKeys; @@ -62,12 +65,16 @@ import java.util.concurrent.TimeUnit; import java.util.function.BiConsumer; +import static org.apache.fluss.rpc.util.CommonRpcMessageUtils.hasHistoricalProduce; +import static org.apache.fluss.rpc.util.CommonRpcMessageUtils.hasHistoricalPut; import static org.apache.fluss.utils.IOUtils.closeQuietly; /** Connection to a Netty server used by the {@link NettyClient}. */ @ThreadSafe final class ServerConnection { private static final Logger LOG = LoggerFactory.getLogger(ServerConnection.class); + private static final short HISTORICAL_PRODUCE_LOG_MIN_VERSION = 1; + private static final short HISTORICAL_PUT_KV_MIN_VERSION = 3; private final ServerNode node; @@ -309,6 +316,7 @@ private CompletableFuture doSend( if (serverApiVersions != null) { try { version = serverApiVersions.highestAvailableVersion(apiKey); + validateVersionCompatibility(apiKey, version, rawRequest); } catch (Exception e) { responseFuture.completeExceptionally(e); return responseFuture; @@ -361,6 +369,39 @@ private CompletableFuture doSend( } } + private void validateVersionCompatibility( + ApiKeys apiKey, short version, ApiMessage rawRequest) { + if (apiKey == ApiKeys.PRODUCE_LOG && version < HISTORICAL_PRODUCE_LOG_MIN_VERSION) { + ProduceLogRequest produceLogRequest = (ProduceLogRequest) rawRequest; + if (hasHistoricalProduce(produceLogRequest)) { + throw new UnsupportedVersionException( + "Historical partition writes require PRODUCE_LOG version " + + HISTORICAL_PRODUCE_LOG_MIN_VERSION + + " or newer, but server " + + node + + " negotiated version " + + version + + '.'); + } + } + + if (apiKey != ApiKeys.PUT_KV || version >= HISTORICAL_PUT_KV_MIN_VERSION) { + return; + } + + PutKvRequest putKvRequest = (PutKvRequest) rawRequest; + if (hasHistoricalPut(putKvRequest)) { + throw new UnsupportedVersionException( + "Historical partition writes require PUT_KV version " + + HISTORICAL_PUT_KV_MIN_VERSION + + " or newer, but server " + + node + + " negotiated version " + + version + + '.'); + } + } + private void handleApiVersionsResponse(ApiMessage response, Throwable cause) { if (cause != null) { close(cause); diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/ApiKeys.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/ApiKeys.java index baf4256650e..ffdd1978de9 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/ApiKeys.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/ApiKeys.java @@ -42,7 +42,8 @@ public enum ApiKeys { GET_TABLE_SCHEMA(1011, 0, 0, PUBLIC), GET_METADATA(1012, 0, 0, PUBLIC), UPDATE_METADATA(1013, 0, 0, PRIVATE), - PRODUCE_LOG(1014, 0, 0, PUBLIC), + // Version 1: Supports original_partition_name in requests and responses for historical writes. + PRODUCE_LOG(1014, 0, 1, PUBLIC), FETCH_LOG(1015, 0, 0, PUBLIC), // Version 0: Uses lake's encoder for primary key encoding (legacy behavior). diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/util/CommonRpcMessageUtils.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/util/CommonRpcMessageUtils.java index 219b51087a4..befd3fc7f3e 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/util/CommonRpcMessageUtils.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/util/CommonRpcMessageUtils.java @@ -34,6 +34,7 @@ import org.apache.fluss.rpc.messages.PbPartitionSpec; import org.apache.fluss.rpc.messages.PbRemoteLogFetchInfo; import org.apache.fluss.rpc.messages.PbRemoteLogSegment; +import org.apache.fluss.rpc.messages.ProduceLogRequest; import org.apache.fluss.rpc.messages.PutKvRequest; import org.apache.fluss.rpc.protocol.ApiError; import org.apache.fluss.security.acl.AccessControlEntry; @@ -83,6 +84,17 @@ public static boolean hasHistoricalPut(PutKvRequest putKvRequest) { && putKvRequest.getBucketsReqAt(0).hasOriginalPartitionName(); } + /** + * Returns whether the produce-log request is for historical partition writes. + * + *

Normal and historical write buckets cannot be mixed in the same request, so the first + * bucket determines the request type. + */ + public static boolean hasHistoricalProduce(ProduceLogRequest produceLogRequest) { + return produceLogRequest.getBucketsReqsCount() > 0 + && produceLogRequest.getBucketsReqAt(0).hasOriginalPartitionName(); + } + public static List toPbAclInfos(Collection aclBindings) { return aclBindings.stream() .map(CommonRpcMessageUtils::toPbAclInfo) diff --git a/fluss-rpc/src/main/proto/FlussApi.proto b/fluss-rpc/src/main/proto/FlussApi.proto index 9d5e05fbdae..1ae6b1f126b 100644 --- a/fluss-rpc/src/main/proto/FlussApi.proto +++ b/fluss-rpc/src/main/proto/FlussApi.proto @@ -886,6 +886,8 @@ message PbProduceLogReqForBucket { optional int64 partition_id = 1; required int32 bucket_id = 2; required bytes records = 3; + // The original partition name for a historical write; unset for a normal write. + optional string original_partition_name = 4; } message PbProduceLogRespForBucket { @@ -894,6 +896,8 @@ message PbProduceLogRespForBucket { optional int32 error_code = 3; optional string error_message = 4; optional int64 base_offset = 5; + // The original partition name echoed from a historical write request; unset for a normal write. + optional string original_partition_name = 6; } message PbFetchLogReqForTable { diff --git a/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/client/ServerConnectionTest.java b/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/client/ServerConnectionTest.java index 4368872ec73..541c6f50ee2 100644 --- a/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/client/ServerConnectionTest.java +++ b/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/client/ServerConnectionTest.java @@ -23,6 +23,7 @@ import org.apache.fluss.config.Configuration; import org.apache.fluss.exception.DisconnectException; import org.apache.fluss.exception.InvalidServerTypeException; +import org.apache.fluss.exception.UnsupportedVersionException; import org.apache.fluss.metrics.Gauge; import org.apache.fluss.metrics.Metric; import org.apache.fluss.metrics.MetricType; @@ -33,11 +34,18 @@ import org.apache.fluss.rpc.TestingGatewayService; import org.apache.fluss.rpc.TestingTabletGatewayService; import org.apache.fluss.rpc.messages.ApiMessage; +import org.apache.fluss.rpc.messages.ApiVersionsRequest; +import org.apache.fluss.rpc.messages.ApiVersionsResponse; import org.apache.fluss.rpc.messages.GetTableSchemaRequest; import org.apache.fluss.rpc.messages.ListDatabasesRequest; import org.apache.fluss.rpc.messages.LookupRequest; +import org.apache.fluss.rpc.messages.PbApiVersion; import org.apache.fluss.rpc.messages.PbLookupReqForBucket; import org.apache.fluss.rpc.messages.PbTablePath; +import org.apache.fluss.rpc.messages.ProduceLogRequest; +import org.apache.fluss.rpc.messages.ProduceLogResponse; +import org.apache.fluss.rpc.messages.PutKvRequest; +import org.apache.fluss.rpc.messages.PutKvResponse; import org.apache.fluss.rpc.metrics.ClientMetricGroup; import org.apache.fluss.rpc.metrics.TestingClientMetricGroup; import org.apache.fluss.rpc.netty.client.ServerConnection.ConnectionState; @@ -62,6 +70,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import static org.apache.fluss.metrics.MetricNames.CLIENT_BYTES_IN_RATE_AVG; import static org.apache.fluss.metrics.MetricNames.CLIENT_BYTES_IN_RATE_TOTAL; @@ -239,7 +248,80 @@ public ChannelFuture connect(String host, int port) { .isInstanceOf(DisconnectException.class); } + @Test + void testRejectHistoricalWritesForOldServer() throws Exception { + nettyServer.close(); + OldWriteGatewayService oldGatewayService = new OldWriteGatewayService(); + buildNettyServer(oldGatewayService); + + ServerConnection connection = + new ServerConnection( + bootstrap, + serverNode, + TestingClientMetricGroup.newInstance(), + clientAuthenticator, + (con, ignore) -> {}); + try { + assertThat(connection.send(ApiKeys.PUT_KV, putKvRequest(null)).get()) + .isInstanceOf(PutKvResponse.class); + assertThat(oldGatewayService.putKvRequests).hasValue(1); + + assertThatThrownBy( + () -> + connection + .send(ApiKeys.PUT_KV, putKvRequest("dt=20260823")) + .get()) + .rootCause() + .isInstanceOf(UnsupportedVersionException.class) + .hasMessageContaining("require PUT_KV version 3 or newer") + .hasMessageContaining("negotiated version 2"); + assertThat(oldGatewayService.putKvRequests).hasValue(1); + + assertThat(connection.send(ApiKeys.PRODUCE_LOG, produceLogRequest(null)).get()) + .isInstanceOf(ProduceLogResponse.class); + assertThat(oldGatewayService.produceLogRequests).hasValue(1); + + assertThatThrownBy( + () -> + connection + .send( + ApiKeys.PRODUCE_LOG, + produceLogRequest("dt=20260823")) + .get()) + .rootCause() + .isInstanceOf(UnsupportedVersionException.class) + .hasMessageContaining("require PRODUCE_LOG version 1 or newer") + .hasMessageContaining("negotiated version 0"); + assertThat(oldGatewayService.produceLogRequests).hasValue(1); + } finally { + connection.close().get(); + } + } + + private static PutKvRequest putKvRequest(String originalPartitionName) { + PutKvRequest request = new PutKvRequest().setTableId(1L).setAcks(1).setTimeoutMs(10_000); + request.addBucketsReq().setBucketId(0).setRecords(new byte[0]); + if (originalPartitionName != null) { + request.getBucketsReqAt(0).setOriginalPartitionName(originalPartitionName); + } + return request; + } + + private static ProduceLogRequest produceLogRequest(String originalPartitionName) { + ProduceLogRequest request = + new ProduceLogRequest().setTableId(1L).setAcks(1).setTimeoutMs(10_000); + request.addBucketsReq().setBucketId(0).setRecords(new byte[0]); + if (originalPartitionName != null) { + request.getBucketsReqAt(0).setOriginalPartitionName(originalPartitionName); + } + return request; + } + private void buildNettyServer() throws Exception { + buildNettyServer(new TestingTabletGatewayService()); + } + + private void buildNettyServer(TestingGatewayService gatewayService) throws Exception { try (NetUtils.Port availablePort = getAvailablePort(); NetUtils.Port availablePort2 = getAvailablePort()) { serverNode = @@ -248,7 +330,7 @@ private void buildNettyServer() throws Exception { serverNode2 = new ServerNode( 2, "localhost", availablePort2.getPort(), ServerType.TABLET_SERVER); - service = new TestingTabletGatewayService(); + service = gatewayService; MetricGroup metricGroup = NOPMetricsGroup.newInstance(); nettyServer = new NettyServer( @@ -263,6 +345,40 @@ private void buildNettyServer() throws Exception { } } + private static class OldWriteGatewayService extends TestingTabletGatewayService { + + private final AtomicInteger putKvRequests = new AtomicInteger(); + private final AtomicInteger produceLogRequests = new AtomicInteger(); + + @Override + public CompletableFuture apiVersions(ApiVersionsRequest request) { + return super.apiVersions(request) + .thenApply( + response -> { + for (PbApiVersion apiVersion : response.getApiVersionsList()) { + if (apiVersion.getApiKey() == ApiKeys.PUT_KV.id) { + apiVersion.setMaxVersion(2); + } else if (apiVersion.getApiKey() == ApiKeys.PRODUCE_LOG.id) { + apiVersion.setMaxVersion(0); + } + } + return response; + }); + } + + @Override + public CompletableFuture putKv(PutKvRequest request) { + putKvRequests.incrementAndGet(); + return CompletableFuture.completedFuture(new PutKvResponse()); + } + + @Override + public CompletableFuture produceLog(ProduceLogRequest request) { + produceLogRequests.incrementAndGet(); + return CompletableFuture.completedFuture(new ProduceLogResponse()); + } + } + private static class MockMetricRegistry extends NOPMetricRegistry { Map registeredMetrics = new HashMap<>(); diff --git a/fluss-rust/crates/fluss/proto/FlussApi.proto b/fluss-rust/crates/fluss/proto/FlussApi.proto index 9d5e05fbdae..1ae6b1f126b 100644 --- a/fluss-rust/crates/fluss/proto/FlussApi.proto +++ b/fluss-rust/crates/fluss/proto/FlussApi.proto @@ -886,6 +886,8 @@ message PbProduceLogReqForBucket { optional int64 partition_id = 1; required int32 bucket_id = 2; required bytes records = 3; + // The original partition name for a historical write; unset for a normal write. + optional string original_partition_name = 4; } message PbProduceLogRespForBucket { @@ -894,6 +896,8 @@ message PbProduceLogRespForBucket { optional int32 error_code = 3; optional string error_message = 4; optional int64 base_offset = 5; + // The original partition name echoed from a historical write request; unset for a normal write. + optional string original_partition_name = 6; } message PbFetchLogReqForTable { diff --git a/fluss-rust/crates/fluss/src/proto/fluss.rs b/fluss-rust/crates/fluss/src/proto/fluss.rs index 95e7045c57e..e29dbaea874 100644 --- a/fluss-rust/crates/fluss/src/proto/fluss.rs +++ b/fluss-rust/crates/fluss/src/proto/fluss.rs @@ -1165,6 +1165,9 @@ pub struct PbProduceLogReqForBucket { pub bucket_id: i32, #[prost(bytes = "bytes", required, tag = "3")] pub records: ::prost::bytes::Bytes, + /// The original partition name for a historical write; unset for a normal write. + #[prost(string, optional, tag = "4")] + pub original_partition_name: ::core::option::Option<::prost::alloc::string::String>, } #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct PbProduceLogRespForBucket { @@ -1178,6 +1181,9 @@ pub struct PbProduceLogRespForBucket { pub error_message: ::core::option::Option<::prost::alloc::string::String>, #[prost(int64, optional, tag = "5")] pub base_offset: ::core::option::Option, + /// The original partition name echoed from a historical write request; unset for a normal write. + #[prost(string, optional, tag = "6")] + pub original_partition_name: ::core::option::Option<::prost::alloc::string::String>, } #[derive(Clone, PartialEq, ::prost::Message)] pub struct PbFetchLogReqForTable { diff --git a/fluss-rust/crates/fluss/src/rpc/api_key.rs b/fluss-rust/crates/fluss/src/rpc/api_key.rs index e9720a4e2b6..1341d1e6180 100644 --- a/fluss-rust/crates/fluss/src/rpc/api_key.rs +++ b/fluss-rust/crates/fluss/src/rpc/api_key.rs @@ -93,7 +93,6 @@ impl ApiKey { | ApiKey::TableExists | ApiKey::GetTableSchema | ApiKey::MetaData - | ApiKey::ProduceLog | ApiKey::FetchLog | ApiKey::ListOffsets | ApiKey::GetLatestKvSnapshots @@ -129,6 +128,8 @@ impl ApiKey { | ApiKey::GetClusterHealth | ApiKey::ListRemoteLogManifests | ApiKey::ListKvSnapshots => Some(ApiVersionRange::new(ApiVersion(0), ApiVersion(0))), + // ProduceLog v1 adds historical partition context to requests and responses. + ApiKey::ProduceLog => Some(ApiVersionRange::new(ApiVersion(0), ApiVersion(1))), // PutKv v2 adds the storage backpressure error code; v3 adds historical partition // context to requests and responses. ApiKey::PutKv => Some(ApiVersionRange::new(ApiVersion(0), ApiVersion(3))), diff --git a/fluss-rust/crates/fluss/src/rpc/message/produce_log.rs b/fluss-rust/crates/fluss/src/rpc/message/produce_log.rs index de9dce118d2..041e0adde01 100644 --- a/fluss-rust/crates/fluss/src/rpc/message/produce_log.rs +++ b/fluss-rust/crates/fluss/src/rpc/message/produce_log.rs @@ -49,6 +49,7 @@ impl ProduceLogRequest { partition_id: ready_batch.table_bucket.partition_id(), bucket_id: ready_batch.table_bucket.bucket_id(), records: ready_batch.write_batch.build()?, + original_partition_name: None, }) } diff --git a/fluss-rust/crates/fluss/src/rpc/server_connection.rs b/fluss-rust/crates/fluss/src/rpc/server_connection.rs index a8b36cbecc2..66825ab265a 100644 --- a/fluss-rust/crates/fluss/src/rpc/server_connection.rs +++ b/fluss-rust/crates/fluss/src/rpc/server_connection.rs @@ -1196,7 +1196,7 @@ mod tests { min_version: 0, max_version: 3, }, - // ProduceLog: server v0..v2, client v0 only → negotiated v0 + // ProduceLog: server v0..v2, client v0..v1 → negotiated v1 PbApiVersion { api_key: 1014, min_version: 0, @@ -1238,7 +1238,7 @@ mod tests { negotiated .highest_available_version(ApiKey::ProduceLog) .unwrap(), - ApiVersion(0) + ApiVersion(1) ); // Disjoint range → error diff --git a/fluss-server/src/main/java/org/apache/fluss/server/DynamicServerConfig.java b/fluss-server/src/main/java/org/apache/fluss/server/DynamicServerConfig.java index 98d721e152e..5f6e860dbe4 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/DynamicServerConfig.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/DynamicServerConfig.java @@ -56,6 +56,7 @@ import static org.apache.fluss.config.ConfigOptions.REMOTE_DATA_DIRS_WEIGHTS; import static org.apache.fluss.config.ConfigOptions.SERVER_DATA_DISK_WRITE_LIMIT_RATIO; import static org.apache.fluss.config.ConfigOptions.SERVER_DATA_DISK_WRITE_RECOVER_RATIO; +import static org.apache.fluss.config.ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME; import static org.apache.fluss.config.ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUPER_CACHE_EXPIRE_AFTER_ACCESS; import static org.apache.fluss.config.ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO; import static org.apache.fluss.config.ConfigOptions.SERVER_SASL_CREDENTIALS; @@ -84,6 +85,7 @@ class DynamicServerConfig { KV_SNAPSHOT_INTERVAL.key(), SERVER_DATA_DISK_WRITE_RECOVER_RATIO.key(), SERVER_DATA_DISK_WRITE_LIMIT_RATIO.key(), + SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME.key(), SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO.key(), SERVER_HISTORICAL_PARTITION_LOOKUPER_CACHE_EXPIRE_AFTER_ACCESS.key(), // Config options for remote.data.dirs diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/HistoricalLookupCacheConfigValidator.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/HistoricalLookupCacheConfigValidator.java index c9d763f410a..337b9542d45 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/HistoricalLookupCacheConfigValidator.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/HistoricalLookupCacheConfigValidator.java @@ -24,11 +24,20 @@ import java.time.Duration; -/** Validates dynamic historical lookup cache settings. */ +/** Validates dynamic historical partition settings used outside the coordinator. */ final class HistoricalLookupCacheConfigValidator implements ServerReconfigurable { @Override public void validate(Configuration newConfig) throws ConfigException { + Duration newCleanupIdleTime = + newConfig.get(ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME); + if (newCleanupIdleTime.isNegative()) { + throw new ConfigException( + String.format( + "Invalid configuration for %s, it must not be negative.", + ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME.key())); + } + double newMaxRatio = newConfig.get( ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/entity/ProduceLogDataForBucket.java b/fluss-server/src/main/java/org/apache/fluss/server/entity/ProduceLogDataForBucket.java new file mode 100644 index 00000000000..50897fea1dc --- /dev/null +++ b/fluss-server/src/main/java/org/apache/fluss/server/entity/ProduceLogDataForBucket.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.entity; + +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.record.MemoryLogRecords; + +import javax.annotation.Nullable; + +/** Log records for one normal or historical partition bucket. */ +public class ProduceLogDataForBucket { + private final TableBucket tableBucket; + private final MemoryLogRecords records; + private final @Nullable String originalPartitionName; + + public ProduceLogDataForBucket( + TableBucket tableBucket, + MemoryLogRecords records, + @Nullable String originalPartitionName) { + this.tableBucket = tableBucket; + this.records = records; + this.originalPartitionName = originalPartitionName; + } + + public TableBucket tableBucket() { + return tableBucket; + } + + public MemoryLogRecords records() { + return records; + } + + public @Nullable String originalPartitionName() { + return originalPartitionName; + } +} diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java index ee28567756f..0f578468a23 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java @@ -222,6 +222,8 @@ public final class Replica { private volatile @Nullable KvTablet kvTablet; private volatile @Nullable CloseableRegistry closeableRegistryForKv; private @Nullable PeriodicSnapshotManager kvSnapshotManager; + // The lake log end offset used as the durable base of the current historical KV overlay. + private volatile long historicalKvBaseOffset = -1L; /** * Server-wide {@link ScannerManager}. Active sessions for this bucket are closed in {@link @@ -380,6 +382,58 @@ public long getLakeLogEndOffset() { return logTablet.getLakeLogEndOffset(); } + /** Returns whether the lake and local log end offsets match for historical KV cleanup. */ + public boolean isHistoricalKvCleanupReady() { + return inReadLock( + leaderIsrUpdateLock, + () -> { + long localLogEndOffset = logTablet.localLogEndOffset(); + return isHistoricalKvCleanupReady(localLogEndOffset); + }); + } + + /** + * Drops and recreates a fully tiered historical KV overlay if leadership and offsets still + * match. + * + * @param expectedLeaderEpoch leader epoch captured when cleanup was scheduled + * @param logEndOffset matching lake and local log end offset that triggered cleanup + * @param prepareLakeLookup marks the covering lake snapshot before the overlay is dropped + * @return whether the overlay was cleaned + */ + public boolean cleanupHistoricalKv( + int expectedLeaderEpoch, long logEndOffset, Runnable prepareLakeLookup) { + checkNotNull(prepareLakeLookup, "prepareLakeLookup must not be null"); + return inWriteLock( + leaderIsrUpdateLock, + () -> { + long localLogEndOffset = logTablet.localLogEndOffset(); + if (leaderEpoch != expectedLeaderEpoch + || localLogEndOffset != logEndOffset + || !isHistoricalKvCleanupReady(localLogEndOffset)) { + return false; + } + + LOG.info( + "Cleaning historical KV overlay for {} at local log end offset {} " + + "covered by lake log end offset {}.", + tableBucket, + localLogEndOffset, + logTablet.getLakeLogEndOffset()); + try { + // A lookup started after the rebuilt empty overlay is published must open + // a lake view that covers the state removed by this cleanup. + prepareLakeLookup.run(); + dropKv(); + createHistoricalKvAfterCleanup(); + return true; + } catch (RuntimeException e) { + fatalErrorHandler.onFatalError(e); + throw e; + } + }); + } + public boolean isDataLakeEnabled() { return getTableConfig().isDataLakeEnabled(); } @@ -725,6 +779,16 @@ private void logTableConfigChanges(TableInfo oldTableInfo, TableInfo newTableInf } } + private boolean isHistoricalKvCleanupReady(long localLogEndOffset) { + return isLeader() + && isHistoricalPartition() + && isKvTable() + && kvTablet != null + && historicalKvBaseOffset >= 0L + && historicalKvBaseOffset < localLogEndOffset + && logTablet.getLakeLogEndOffset() == localLogEndOffset; + } + private void createKv() { try { // create a closeable registry for the closable related to kv @@ -754,10 +818,7 @@ private void createKv() { } // A historical KV tablet is a disposable overlay over the lake snapshot. It is recovered // by replaying WAL from the lake log end offset and does not create its own KV snapshots. - if (isHistoricalPartition()) { - // TODO: Clean up historical KV state after the corresponding WAL is fully tiered to - // lake storage. - } else { + if (!isHistoricalPartition()) { startPeriodicKvSnapshot(snapshotUsed.orElse(null)); } } @@ -777,6 +838,26 @@ private void dropKv() { kvManager.dropKv(tableBucket); kvTablet = null; } + historicalKvBaseOffset = -1L; + } + + private void createHistoricalKvAfterCleanup() { + checkState(isHistoricalPartition(), "Only a historical KV overlay can be cleaned."); + try { + closeableRegistryForKv = new CloseableRegistry(); + closeableRegistry.registerCloseable(closeableRegistryForKv); + initKvTablet(); + } catch (Exception e) { + try { + dropKv(); + } catch (Exception cleanupError) { + e.addSuppressed(cleanupError); + } + throw new KvStorageException( + String.format( + "Failed to recreate historical KV overlay for bucket %s.", tableBucket), + e); + } } private void mayFlushKv(long newHighWatermark) { @@ -898,6 +979,9 @@ private Optional initKvTablet() { logTablet.updateMinRetainOffset(restoreStartOffset); recoverKvTablet(restoreStartOffset, rowCount, autoIncIDRange); + if (isHistoricalPartition()) { + historicalKvBaseOffset = restoreStartOffset; + } } catch (Exception e) { throw new KvStorageException( String.format( @@ -1044,8 +1128,10 @@ private void recoverKvTablet( private long historicalRecoveryStartOffset() { long lakeLogEndOffset = logTablet.getLakeLogEndOffset(); + long localLogEndOffset = logTablet.localLogEndOffset(); long logStartOffset = logTablet.logStartOffset(); - long recoveryStartOffset = lakeLogEndOffset >= 0 ? lakeLogEndOffset : 0L; + long recoveryStartOffset = + lakeLogEndOffset >= 0 ? Math.min(lakeLogEndOffset, localLogEndOffset) : 0L; checkState( recoveryStartOffset >= logStartOffset, "Cannot recover historical KV state: recovery start offset %s is before the " @@ -1161,9 +1247,14 @@ public LogAppendInfo appendRecordsToLeader(MemoryLogRecords memoryLogRecords, in "Leader not local for bucket %s on tabletServer %d", tableBucket, localTabletServerId)); } - if (isHistoricalPartition()) { + // Historical primary-key writes must go through PUT_KV so the server can + // preserve the original partition namespace and consult the lake on a local + // miss. Append-only records already contain their partition columns, so a log + // table can append them directly to its historical system partition. + if (isHistoricalPartition() && isKvTable()) { throw new InvalidPartitionException( - "Normal write request must not target a historical partition."); + "Produce-log request must not target the historical partition of " + + "a primary-key table."); } validateInSyncReplicaSize(requiredAcks); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java index b3357019230..67fc81a3350 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java @@ -82,6 +82,7 @@ import org.apache.fluss.server.entity.NotifyLeaderAndIsrData; import org.apache.fluss.server.entity.NotifyLeaderAndIsrResultForBucket; import org.apache.fluss.server.entity.NotifyRemoteLogOffsetsData; +import org.apache.fluss.server.entity.ProduceLogDataForBucket; import org.apache.fluss.server.entity.PutKvDataForBucket; import org.apache.fluss.server.entity.StopReplicaData; import org.apache.fluss.server.entity.StopReplicaResultForBucket; @@ -373,7 +374,8 @@ public ReplicaManager( localDiskManager, dataDir, dataDirVolumeBytes, - scheduler); + scheduler, + clock); registerMetrics(); } @@ -417,6 +419,7 @@ public int getCoordinatorEpoch() { public void validate(Configuration newConfig) throws ConfigException { // Type validation is already handled by DynamicServerConfig. // Here we only do basic sanity checks. + historicalPartitionManager.validate(newConfig); int newMinInSyncReplicas = newConfig.get(ConfigOptions.LOG_REPLICA_MIN_IN_SYNC_REPLICAS_NUMBER); if (newMinInSyncReplicas <= 0) { @@ -688,6 +691,51 @@ public void appendRecordsToLog( timeoutMs, requiredAcks, entriesPerBucket.size(), appendResult, responseCallback); } + /** Appends historical log batches while preserving each original partition in the response. */ + public void appendHistoricalRecordsToLog( + int timeoutMs, + int requiredAcks, + Collection entriesPerBucket, + @Nullable UserContext userContext, + Consumer> responseCallback) { + if (entriesPerBucket.isEmpty()) { + responseCallback.accept(Collections.emptyList()); + return; + } + + List results = Collections.synchronizedList(new ArrayList<>()); + AtomicInteger remaining = new AtomicInteger(entriesPerBucket.size()); + for (ProduceLogDataForBucket bucketData : entriesPerBucket) { + String originalPartitionName = + checkNotNull( + bucketData.originalPartitionName(), + "originalPartitionName must not be null"); + appendRecordsToLog( + timeoutMs, + requiredAcks, + Collections.singletonMap(bucketData.tableBucket(), bucketData.records()), + userContext, + bucketResults -> { + ProduceLogResultForBucket result = bucketResults.get(0); + ProduceLogResultForBucket historicalResult = + result.failed() + ? ProduceLogResultForBucket.historicalFailure( + result.getTableBucket(), + result.getError(), + originalPartitionName) + : ProduceLogResultForBucket.historicalSuccess( + result.getTableBucket(), + result.getBaseOffset(), + result.getWriteLogEndOffset(), + originalPartitionName); + results.add(historicalResult); + if (remaining.decrementAndGet() == 0) { + responseCallback.accept(new ArrayList<>(results)); + } + }); + } + } + /** * Fetch records from a replica. Currently, we will return the fetched records immediately. * @@ -1350,7 +1398,8 @@ public void notifyLakeTableOffset( lakeBucketOffsets.entrySet()) { TableBucket tb = lakeBucketOffsetEntry.getKey(); LakeBucketOffset lakeBucketOffset = lakeBucketOffsetEntry.getValue(); - LogTablet logTablet = getReplicaOrException(tb).getLogTablet(); + Replica replica = getReplicaOrException(tb); + LogTablet logTablet = replica.getLogTablet(); logTablet.updateLakeTableSnapshotId(lakeBucketOffset.getSnapshotId()); lakeBucketOffset @@ -1359,7 +1408,19 @@ public void notifyLakeTableOffset( lakeBucketOffset .getLogEndOffset() - .ifPresent(logTablet::updateLakeLogEndOffset); + .ifPresent( + lakeLogEndOffset -> { + logTablet.updateLakeLogEndOffset(lakeLogEndOffset); + if (replica.isHistoricalPartition() + && replica.isKvTable()) { + // Only an explicit log-end-offset notification can + // make historical cleanup eligible. + historicalPartitionManager.onLakeProgress( + replica, + lakeBucketOffset.getSnapshotId(), + lakeLogEndOffset); + } + }); lakeBucketOffset .getMaxTimestamp() @@ -1402,7 +1463,13 @@ private void makeLeaders( if (replica.isDataLakeEnabled()) { updateWithLakeTableSnapshot(replica); } + int previousLeaderEpoch = replica.getLeaderEpoch(); replica.makeLeader(data); + if (replica.isHistoricalPartition() + && replica.isKvTable() + && previousLeaderEpoch != replica.getLeaderEpoch()) { + historicalPartitionManager.onLeaderActivated(replica); + } // start the remote log tiering tasks for leaders remoteLogManager.startLogTiering(replica); @@ -1425,6 +1492,9 @@ private void updateWithLakeTableSnapshot(Replica replica) throws Exception { LakeTableSnapshot lakeTableSnapshot = optLakeTableSnapshot.get(); long snapshotId = optLakeTableSnapshot.get().getSnapshotId(); replica.getLogTablet().updateLakeTableSnapshotId(snapshotId); + lakeTableSnapshot + .getLogEndOffset(tb) + .ifPresent(replica.getLogTablet()::updateLakeLogEndOffset); if (replica.isHistoricalPartition()) { // The historical overlay will be rebuilt from this snapshot's lake offset. // Refresh a cached lookuper before it becomes the fallback for data omitted @@ -1432,9 +1502,6 @@ private void updateWithLakeTableSnapshot(Replica replica) throws Exception { historicalPartitionManager.requireLakeSnapshot( replica.getTableBucket().getTableId(), snapshotId); } - lakeTableSnapshot - .getLogEndOffset(tb) - .ifPresent(replica.getLogTablet()::updateLakeLogEndOffset); } } catch (Exception e) { if (replica.isHistoricalPartition()) { @@ -1476,6 +1543,9 @@ private void makeFollowers( replicasBecomeFollower.add(replica); scannerManager.closeScannersForBucket(tb); } + if (replica.isHistoricalPartition()) { + historicalPartitionManager.onReplicaStopped(tb); + } // stop the remote log tiering tasks for followers remoteLogManager.stopLogTiering(replica); result.put(tb, new NotifyLeaderAndIsrResultForBucket(tb)); @@ -2278,6 +2348,9 @@ private StopReplicaResultForBucket stopReplica( HostedReplica replica = getReplica(tb); if (replica instanceof OnlineReplica) { Replica replicaToDelete = ((OnlineReplica) replica).getReplica(); + if (replicaToDelete.isHistoricalPartition()) { + historicalPartitionManager.onReplicaStopped(tb); + } if (deleteLocal) { if (allReplicas.remove(tb) != null) { serverMetricGroup.removeTableBucketMetricGroup( diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java index 13ec002e0df..62a66d61137 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java @@ -19,7 +19,9 @@ import org.apache.fluss.annotation.Internal; import org.apache.fluss.annotation.VisibleForTesting; +import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; +import org.apache.fluss.exception.ConfigException; import org.apache.fluss.exception.HistoricalPartitionThrottledException; import org.apache.fluss.exception.InvalidPartitionException; import org.apache.fluss.lake.lakestorage.LakeTableLookuper; @@ -43,11 +45,17 @@ import org.apache.fluss.server.storage.LocalDiskManager; import org.apache.fluss.utils.ByteArraySlice; import org.apache.fluss.utils.ByteArrayWrapper; +import org.apache.fluss.utils.clock.Clock; +import org.apache.fluss.utils.clock.SystemClock; import org.apache.fluss.utils.concurrent.Scheduler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import javax.annotation.Nullable; import java.io.File; +import java.time.Duration; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -55,15 +63,29 @@ import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicBoolean; +import static org.apache.fluss.utils.Preconditions.checkArgument; import static org.apache.fluss.utils.Preconditions.checkNotNull; /** Coordinates lookup, write, and lifecycle operations for historical partitions. */ @Internal public final class HistoricalPartitionManager implements AutoCloseable { + private static final Logger LOG = LoggerFactory.getLogger(HistoricalPartitionManager.class); + private static final long MAX_HISTORICAL_KV_SIZE_BYTES = 5L * 1024 * 1024 * 1024; + private final HistoricalPartitionTaskExecutor taskExecutor; private final HistoricalLakeLookupManager lakeLookupManager; + private final Clock clock; + private volatile long cleanupIdleTimeMs; + private final long maxHistoricalKvSizeBytes; + // Per-physical-bucket state for coordinating historical write admission and overlay cleanup. + // The state is replaced when a new leader epoch is activated and removed when the local + // replica stops. + private final ConcurrentMap historicalWriteStates; /** Creates a historical partition manager from the tablet server dependencies. */ public HistoricalPartitionManager( @@ -72,8 +94,10 @@ public HistoricalPartitionManager( LocalDiskManager localDiskManager, File dataDir, long dataDirVolumeBytes, - Scheduler scheduler) { + Scheduler scheduler, + Clock clock) { this( + conf, new HistoricalPartitionTaskExecutor(conf), new HistoricalLakeLookupManager( conf, @@ -81,16 +105,45 @@ public HistoricalPartitionManager( localDiskManager, dataDir, dataDirVolumeBytes, - scheduler)); + scheduler), + clock, + MAX_HISTORICAL_KV_SIZE_BYTES); } @VisibleForTesting HistoricalPartitionManager( HistoricalPartitionTaskExecutor taskExecutor, HistoricalLakeLookupManager lakeLookupManager) { + this( + new Configuration(), + taskExecutor, + lakeLookupManager, + SystemClock.getInstance(), + MAX_HISTORICAL_KV_SIZE_BYTES); + } + + @VisibleForTesting + HistoricalPartitionManager( + Configuration conf, + HistoricalPartitionTaskExecutor taskExecutor, + HistoricalLakeLookupManager lakeLookupManager, + Clock clock, + long maxHistoricalKvSizeBytes) { + Duration cleanupIdleTime = + conf.get(ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME); + checkArgument( + !cleanupIdleTime.isNegative(), + "%s must not be negative.", + ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME.key()); + checkArgument( + maxHistoricalKvSizeBytes > 0L, "maxHistoricalKvSizeBytes must be greater than 0."); this.taskExecutor = checkNotNull(taskExecutor, "taskExecutor must not be null"); this.lakeLookupManager = checkNotNull(lakeLookupManager, "lakeLookupManager must not be null"); + this.clock = checkNotNull(clock, "clock must not be null"); + this.cleanupIdleTimeMs = cleanupIdleTime.toMillis(); + this.maxHistoricalKvSizeBytes = maxHistoricalKvSizeBytes; + this.historicalWriteStates = new ConcurrentHashMap<>(); } /** Starts the resources used by historical partition operations. */ @@ -98,6 +151,41 @@ public void startup(Scheduler scheduler) { lakeLookupManager.startup(scheduler); } + /** Starts tracking cleanup activity for a newly activated historical KV leader. */ + public void onLeaderActivated(Replica replica) { + historicalWriteStates.put( + replica.getTableBucket(), new HistoricalWriteState(clock.milliseconds())); + } + + /** Stops tracking cleanup activity for a replica that is no longer a local leader. */ + public void onReplicaStopped(TableBucket tableBucket) { + historicalWriteStates.remove(tableBucket); + } + + /** Records new lake progress and schedules any cleanup that it makes eligible. */ + public void onLakeProgress(Replica replica, long lakeSnapshotId, long lakeLogEndOffset) { + if (!replica.isLeader() || !replica.isKvTable()) { + return; + } + int expectedLeaderEpoch = replica.getLeaderEpoch(); + long localLogEndOffset = replica.getLocalLogEndOffset(); + if (lakeLogEndOffset != localLogEndOffset) { + return; + } + HistoricalWriteState state = historicalWriteStateFor(replica); + boolean maxSizeReached = state.maxSizeReached.get(); + // Lake progress is the cleanup trigger. The ordered cleanup task rechecks the latest write + // time when it actually runs, so a write accepted after this notification cancels an idle + // cleanup without being overtaken by it. + scheduleCleanup( + replica, + state, + maxSizeReached, + lakeSnapshotId, + expectedLeaderEpoch, + lakeLogEndOffset); + } + /** Looks up historical keys from the local overlay and then lake storage. */ public CompletableFuture lookup( Replica replica, @@ -119,7 +207,8 @@ public CompletableFuture lookup( + tableBucket + " (original partition " + lookupData.originalPartitionName() - + ").")))); + + ") because the historical request " + + "queue is full.")))); } catch (RuntimeException e) { return CompletableFuture.completedFuture( new LookupResultForBucket( @@ -141,6 +230,20 @@ public CompletableFuture putKv( checkNotNull( putData.originalPartitionName(), "originalPartitionName must not be null"); + HistoricalWriteState state = historicalWriteStateFor(replica); + if (state.maxSizeReached.get()) { + return CompletableFuture.completedFuture( + maxSizeThrottledResult( + putData, originalPartitionName, maxHistoricalKvSizeBytes)); + } + long liveSstSize = replica.logicalStorageKvSize(); + if (liveSstSize >= maxHistoricalKvSizeBytes) { + markMaxSizeReached(replica, state); + return CompletableFuture.completedFuture( + maxSizeThrottledResult( + putData, originalPartitionName, maxHistoricalKvSizeBytes)); + } + state.lastHistoricalWriteMs = clock.milliseconds(); return taskExecutor.submitOrdered( putData.tableBucket(), () -> { @@ -152,6 +255,7 @@ public CompletableFuture putKv( targetColumns, mergeMode, requiredAcks); + state.lastHistoricalWriteMs = clock.milliseconds(); return PutKvResultForBucket.historicalSuccess( putData.tableBucket(), appendInfo.lastOffset() + 1, @@ -163,17 +267,7 @@ public CompletableFuture putKv( originalPartitionName); } }, - () -> - PutKvResultForBucket.historicalFailure( - putData.tableBucket(), - ApiError.fromThrowable( - new HistoricalPartitionThrottledException( - "Historical write is throttled for " - + putData.tableBucket() - + " (original partition " - + originalPartitionName - + ").")), - originalPartitionName)); + () -> requestLimitThrottledResult(putData, originalPartitionName)); } catch (RuntimeException e) { return CompletableFuture.completedFuture( PutKvResultForBucket.historicalFailure( @@ -183,9 +277,33 @@ public CompletableFuture putKv( } } - /** Applies dynamic historical lookup configuration changes. */ + /** Validates dynamic historical partition configuration changes. */ + public void validate(Configuration newConf) throws ConfigException { + Duration newCleanupIdleTime = + newConf.get(ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME); + if (newCleanupIdleTime.isNegative()) { + throw new ConfigException( + String.format( + "Invalid configuration for %s, it must not be negative.", + ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME.key())); + } + } + + /** Applies dynamic historical partition configuration changes. */ public void reconfigure(Configuration newConf) { lakeLookupManager.reconfigure(newConf); + long newCleanupIdleTimeMs = + newConf.get(ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME) + .toMillis(); + if (newCleanupIdleTimeMs == cleanupIdleTimeMs) { + return; + } + long oldCleanupIdleTimeMs = cleanupIdleTimeMs; + cleanupIdleTimeMs = newCleanupIdleTimeMs; + LOG.info( + "Historical KV cleanup idle time reconfigured: {} ms -> {} ms.", + oldCleanupIdleTimeMs, + newCleanupIdleTimeMs); } /** Invalidates the cached lake lookuper for the given table. */ @@ -286,8 +404,117 @@ LogAppendInfo processPut( requiredAcks); } + private void markMaxSizeReached(Replica replica, HistoricalWriteState state) { + if (state.maxSizeReached.compareAndSet(false, true)) { + LOG.warn( + "Pausing historical writes for {} because its live SST size reached the " + + "maximum size {} bytes.", + replica.getTableBucket(), + maxHistoricalKvSizeBytes); + } + } + + private HistoricalWriteState historicalWriteStateFor(Replica replica) { + return historicalWriteStates.computeIfAbsent( + replica.getTableBucket(), + ignored -> new HistoricalWriteState(clock.milliseconds())); + } + + private void scheduleCleanup( + Replica replica, + HistoricalWriteState state, + boolean maxSizeReached, + long lakeSnapshotId, + int expectedLeaderEpoch, + long logEndOffset) { + CompletableFuture cleanupFuture; + cleanupFuture = + taskExecutor.submitOrderedMaintenance( + replica.getTableBucket(), + () -> + runCleanup( + replica, + state, + maxSizeReached, + lakeSnapshotId, + expectedLeaderEpoch, + logEndOffset)); + cleanupFuture.whenComplete( + (ignored, error) -> { + if (error != null) { + LOG.error( + "Historical KV cleanup failed for {}.", + replica.getTableBucket(), + error); + } + }); + } + + private void runCleanup( + Replica replica, + HistoricalWriteState state, + boolean maxSizeReached, + long lakeSnapshotId, + int expectedLeaderEpoch, + long logEndOffset) { + long now = clock.milliseconds(); + if (historicalWriteStates.get(replica.getTableBucket()) != state + || expectedLeaderEpoch != replica.getLeaderEpoch() + || (!maxSizeReached + && (cleanupIdleTimeMs <= 0L + || now < state.lastHistoricalWriteMs + || now - state.lastHistoricalWriteMs < cleanupIdleTimeMs))) { + return; + } + + if (replica.cleanupHistoricalKv( + expectedLeaderEpoch, + logEndOffset, + () -> requireLakeSnapshot(replica.getTableBucket().getTableId(), lakeSnapshotId))) { + state.maxSizeReached.set(false); + LOG.info( + "Cleaned {} historical KV overlay for {}.", + maxSizeReached ? "max-size-triggered" : "idle-triggered", + replica.getTableBucket()); + } + } + + private static PutKvResultForBucket requestLimitThrottledResult( + PutKvDataForBucket putData, String originalPartitionName) { + return PutKvResultForBucket.historicalFailure( + putData.tableBucket(), + ApiError.fromThrowable( + new HistoricalPartitionThrottledException( + "Historical write is throttled for " + + putData.tableBucket() + + " (original partition " + + originalPartitionName + + ") because the historical request queue is full.")), + originalPartitionName); + } + + private static PutKvResultForBucket maxSizeThrottledResult( + PutKvDataForBucket putData, String originalPartitionName, long maxHistoricalKvSize) { + return PutKvResultForBucket.historicalFailure( + putData.tableBucket(), + ApiError.fromThrowable( + new HistoricalPartitionThrottledException( + "Historical write is throttled for " + + putData.tableBucket() + + " (original partition " + + originalPartitionName + + ") because its historical KV overlay reached the live " + + "SST maximum size of " + + maxHistoricalKvSize + + " bytes. New writes are paused until lake tiering " + + "covers all previously accepted writes and the local " + + "overlay cleanup completes.")), + originalPartitionName); + } + @Override public void close() { + historicalWriteStates.clear(); taskExecutor.close(); lakeLookupManager.close(); } @@ -352,4 +579,19 @@ private LookupResultForBucket lookupInternal( tableBucket, originalPartitionName, ApiError.fromThrowable(e)); } } + + /** Per-bucket historical write activity and maximum-size state. */ + private static final class HistoricalWriteState { + // Latched when the live SST size reaches the maximum. It is cleared only after a cleanup + // covered by lake progress succeeds, so transient RocksDB size changes cannot resume + // writes prematurely. + private final AtomicBoolean maxSizeReached = new AtomicBoolean(); + + // Updated when a write is admitted and again when it completes successfully. + private volatile long lastHistoricalWriteMs; + + private HistoricalWriteState(long lastHistoricalWriteMs) { + this.lastHistoricalWriteMs = lastHistoricalWriteMs; + } + } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionTaskExecutor.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionTaskExecutor.java index 9bf66848fad..763f2cd1807 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionTaskExecutor.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionTaskExecutor.java @@ -141,24 +141,47 @@ public CompletableFuture submitOrdered( return CompletableFuture.completedFuture(throttledResult.get()); } - CompletableFuture future; - CompletableFuture tail; try { - synchronized (orderedTasksLock) { - CompletableFuture previousTail = orderedTaskTails.get(orderingKey); - if (previousTail == null) { - future = CompletableFuture.supplyAsync(task, executor); - } else { - future = previousTail.thenApplyAsync(ignored -> task.get(), executor); - } - // Convert success or failure into a normal completion used only for sequencing. - tail = future.handle((ignored, error) -> null); - orderedTaskTails.put(orderingKey, tail); - } + return enqueueOrdered(orderingKey, task, true); } catch (RuntimeException e) { requestPermits.release(); throw e; } + } + + /** + * Submits an internal maintenance task after all accepted tasks with the same ordering key. + * + *

Maintenance work is not a client request and therefore does not consume a request permit. + */ + public CompletableFuture submitOrderedMaintenance( + Object orderingKey, Runnable maintenanceTask) { + checkNotNull(orderingKey, "orderingKey must not be null."); + checkNotNull(maintenanceTask, "maintenanceTask must not be null."); + return enqueueOrdered( + orderingKey, + () -> { + maintenanceTask.run(); + return null; + }, + false); + } + + private CompletableFuture enqueueOrdered( + Object orderingKey, Supplier task, boolean releaseRequestPermit) { + CompletableFuture future; + CompletableFuture tail; + synchronized (orderedTasksLock) { + CompletableFuture previousTail = orderedTaskTails.get(orderingKey); + if (previousTail == null) { + future = CompletableFuture.supplyAsync(task, executor); + } else { + future = previousTail.thenApplyAsync(ignored -> task.get(), executor); + } + // Convert success or failure into a normal completion used only for sequencing. + tail = future.handle((ignored, error) -> null); + orderedTaskTails.put(orderingKey, tail); + } CompletableFuture currentTail = tail; tail.whenComplete( @@ -167,17 +190,24 @@ public CompletableFuture submitOrdered( orderedTaskTails.remove(orderingKey, currentTail); } }); - return trackAcceptedRequest(future); + return trackAcceptedRequest(future, releaseRequestPermit); } private CompletableFuture trackAcceptedRequest(CompletableFuture future) { + return trackAcceptedRequest(future, true); + } + + private CompletableFuture trackAcceptedRequest( + CompletableFuture future, boolean releaseRequestPermit) { pendingRequests.add(future); future.whenComplete( (ignored, error) -> { - // Release the permit exactly once when the accepted task reaches a terminal - // state, including exceptional completion and cancellation. pendingRequests.remove(future); - requestPermits.release(); + if (releaseRequestPermit) { + // Release the permit exactly once when the accepted request reaches a + // terminal state, including exceptional completion and cancellation. + requestPermits.release(); + } }); return future; } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java b/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java index bd3ef49b35a..c3468b7a941 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java @@ -36,6 +36,7 @@ import org.apache.fluss.rpc.entity.FetchLogResultForBucket; import org.apache.fluss.rpc.entity.LookupResultForBucket; import org.apache.fluss.rpc.entity.PrefixLookupResultForBucket; +import org.apache.fluss.rpc.entity.ProduceLogResultForBucket; import org.apache.fluss.rpc.entity.ResultForBucket; import org.apache.fluss.rpc.gateway.CoordinatorGateway; import org.apache.fluss.rpc.gateway.TabletServerGateway; @@ -92,6 +93,7 @@ import org.apache.fluss.server.entity.NotifyLakeTableOffsetData; import org.apache.fluss.server.entity.NotifyLeaderAndIsrData; import org.apache.fluss.server.entity.NotifyRemoteLogOffsetsData; +import org.apache.fluss.server.entity.ProduceLogDataForBucket; import org.apache.fluss.server.entity.PutKvDataForBucket; import org.apache.fluss.server.entity.StopReplicaData; import org.apache.fluss.server.entity.UserContext; @@ -125,6 +127,7 @@ import java.util.stream.Collectors; import static org.apache.fluss.rpc.util.CommonRpcMessageUtils.hasHistoricalLookup; +import static org.apache.fluss.rpc.util.CommonRpcMessageUtils.hasHistoricalProduce; import static org.apache.fluss.rpc.util.CommonRpcMessageUtils.hasHistoricalPut; import static org.apache.fluss.security.acl.OperationType.DESCRIBE; import static org.apache.fluss.security.acl.OperationType.READ; @@ -137,7 +140,6 @@ import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getNotifyLeaderAndIsrRequestData; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getNotifyRemoteLogOffsetsData; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getNotifySnapshotOffsetData; -import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getProduceLogData; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getStopReplicaData; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getTableFilterInfoMap; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getTableStatsRequestData; @@ -157,6 +159,7 @@ import static org.apache.fluss.server.utils.ServerRpcMessageUtils.toHistoricalLookupData; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.toLookupData; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.toPrefixLookupData; +import static org.apache.fluss.server.utils.ServerRpcMessageUtils.toProduceLogDataForBuckets; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.toPutKvDataForBuckets; /** An RPC Gateway service for tablet server. */ @@ -216,13 +219,29 @@ public void shutdown() {} public CompletableFuture produceLog(ProduceLogRequest request) { authorizeTable(WRITE, request.getTableId()); CompletableFuture response = new CompletableFuture<>(); - Map produceLogData = getProduceLogData(request); - replicaManager.appendRecordsToLog( - request.getTimeoutMs(), - request.getAcks(), - produceLogData, - new UserContext(currentSession().getPrincipal()), - bucketResponseMap -> response.complete(makeProduceLogResponse(bucketResponseMap))); + List produceLogData = toProduceLogDataForBuckets(request); + UserContext userContext = new UserContext(currentSession().getPrincipal()); + Consumer> responseCallback = + results -> response.complete(makeProduceLogResponse(results)); + if (hasHistoricalProduce(request)) { + replicaManager.appendHistoricalRecordsToLog( + request.getTimeoutMs(), + request.getAcks(), + produceLogData, + userContext, + responseCallback); + } else { + Map recordsByBucket = new HashMap<>(); + for (ProduceLogDataForBucket bucketData : produceLogData) { + recordsByBucket.put(bucketData.tableBucket(), bucketData.records()); + } + replicaManager.appendRecordsToLog( + request.getTimeoutMs(), + request.getAcks(), + recordsByBucket, + userContext, + responseCallback); + } return response; } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java index 02297ead2dd..58a35b392b0 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java @@ -187,6 +187,7 @@ import org.apache.fluss.server.entity.NotifyLeaderAndIsrData; import org.apache.fluss.server.entity.NotifyLeaderAndIsrResultForBucket; import org.apache.fluss.server.entity.NotifyRemoteLogOffsetsData; +import org.apache.fluss.server.entity.ProduceLogDataForBucket; import org.apache.fluss.server.entity.PutKvDataForBucket; import org.apache.fluss.server.entity.StopReplicaData; import org.apache.fluss.server.entity.StopReplicaResultForBucket; @@ -231,6 +232,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.apache.fluss.rpc.util.CommonRpcMessageUtils.hasHistoricalProduce; import static org.apache.fluss.rpc.util.CommonRpcMessageUtils.toByteBuffer; import static org.apache.fluss.rpc.util.CommonRpcMessageUtils.toPbAclInfo; import static org.apache.fluss.utils.Preconditions.checkNotNull; @@ -892,12 +894,20 @@ public static StopReplicaResponse makeStopReplicaResponse( return stopReplicaResponse; } - public static Map getProduceLogData( + /** Converts produce-log requests while preserving their historical partition context. */ + public static List toProduceLogDataForBuckets( ProduceLogRequest produceRequest) { long tableId = produceRequest.getTableId(); - Map produceEntryData = new HashMap<>(); + List produceLogData = + new ArrayList<>(produceRequest.getBucketsReqsCount()); + Map> originalPartitionsByBucket = new HashMap<>(); + boolean historicalWriteRequest = hasHistoricalProduce(produceRequest); for (PbProduceLogReqForBucket produceLogReqForBucket : produceRequest.getBucketsReqsList()) { + if (produceLogReqForBucket.hasOriginalPartitionName() != historicalWriteRequest) { + throw new IllegalArgumentException( + "Normal and historical writes cannot be mixed in the same request."); + } ByteBuffer recordBuffer = toByteBuffer(produceLogReqForBucket.getRecordsSlice()); MemoryLogRecords logRecords = MemoryLogRecords.pointToByteBuffer(recordBuffer); TableBucket tb = @@ -907,9 +917,23 @@ public static Map getProduceLogData( ? produceLogReqForBucket.getPartitionId() : null, produceLogReqForBucket.getBucketId()); - produceEntryData.put(tb, logRecords); + String originalPartitionName = + produceLogReqForBucket.hasOriginalPartitionName() + ? produceLogReqForBucket.getOriginalPartitionName() + : null; + Set originalPartitions = + originalPartitionsByBucket.computeIfAbsent(tb, ignored -> new HashSet<>()); + if (!originalPartitions.add(originalPartitionName)) { + throw new IllegalArgumentException( + "A ProduceLog request contains duplicate table bucket " + + tb + + " and original partition " + + originalPartitionName + + '.'); + } + produceLogData.add(new ProduceLogDataForBucket(tb, logRecords, originalPartitionName)); } - return produceEntryData; + return produceLogData; } public static ProduceLogResponse makeProduceLogResponse( @@ -923,6 +947,9 @@ public static ProduceLogResponse makeProduceLogResponse( if (tableBucket.getPartitionId() != null) { producedBucket.setPartitionId(tableBucket.getPartitionId()); } + if (bucketResult.getOriginalPartitionName() != null) { + producedBucket.setOriginalPartitionName(bucketResult.getOriginalPartitionName()); + } if (bucketResult.failed()) { producedBucket.setError( diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java index 8ba8204a54c..0bd9ab6f522 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java @@ -214,10 +214,6 @@ private static void checkHistoricalPartition( DataLakeFormat.PAIMON, dataLakeFormat.get())); } - if (!tableDescriptor.hasPrimaryKey()) { - unmetRequirements.add("the table must define a primary key"); - } - int partitionKeyCount = tableDescriptor.getPartitionKeys().size(); if (partitionKeyCount != 1) { unmetRequirements.add( diff --git a/fluss-server/src/test/java/org/apache/fluss/server/DynamicConfigChangeTest.java b/fluss-server/src/test/java/org/apache/fluss/server/DynamicConfigChangeTest.java index 5bc4a1fe8ce..adc6a68c63a 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/DynamicConfigChangeTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/DynamicConfigChangeTest.java @@ -191,6 +191,37 @@ void testAlterLakehouseConfigs() throws Exception { } } + @Test + void testAlterHistoricalKvCleanupIdleTime() throws Exception { + DynamicConfigManager dynamicConfigManager = createManager(new Configuration()); + AtomicReference cleanupIdleTime = new AtomicReference<>(); + dynamicConfigManager.register( + new ServerReconfigurable() { + @Override + public void validate(Configuration newConfig) throws ConfigException {} + + @Override + public void reconfigure(Configuration newConfig) { + cleanupIdleTime.set( + newConfig.get( + ConfigOptions + .SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME)); + } + }); + dynamicConfigManager.startup(); + + alterConfig( + dynamicConfigManager, + ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME.key(), + "5min"); + + assertThat(cleanupIdleTime.get()).isEqualTo(Duration.ofMinutes(5)); + assertThat(zookeeperClient.fetchEntityConfig()) + .containsEntry( + ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME.key(), + "5min"); + } + @Test void testOverrideConfigs() throws Exception { Configuration configuration = new Configuration(); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerTest.java index db4a88bc085..3364aa61b08 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerTest.java @@ -67,6 +67,7 @@ import org.apache.fluss.server.entity.FetchReqInfo; import org.apache.fluss.server.entity.NotifyLeaderAndIsrData; import org.apache.fluss.server.entity.NotifyLeaderAndIsrResultForBucket; +import org.apache.fluss.server.entity.ProduceLogDataForBucket; import org.apache.fluss.server.entity.StopReplicaData; import org.apache.fluss.server.entity.StopReplicaResultForBucket; import org.apache.fluss.server.kv.KvTablet; @@ -238,6 +239,32 @@ tb, genMemoryLogRecordsByObject(DATA1)), "Unknown table or bucket: TableBucket{tableId=10001, bucket=0}"))); } + @Test + void testProduceHistoricalLogBatchesToSameTableBucket() throws Exception { + replicaManager.getDiskUsageMonitor().update(0.10); + TableBucket tableBucket = new TableBucket(DATA1_TABLE_ID, 1); + makeLogTableAsLeader(tableBucket.getBucket()); + + CompletableFuture> future = new CompletableFuture<>(); + replicaManager.appendHistoricalRecordsToLog( + 20_000, + 1, + Arrays.asList( + new ProduceLogDataForBucket( + tableBucket, genMemoryLogRecordsByObject(DATA1), "dt=2025-01-01"), + new ProduceLogDataForBucket( + tableBucket, genMemoryLogRecordsByObject(DATA1), "dt=2025-01-02")), + null, + future::complete); + + assertThat(future.get()) + .containsExactlyInAnyOrder( + ProduceLogResultForBucket.historicalSuccess( + tableBucket, 0L, 10L, "dt=2025-01-01"), + ProduceLogResultForBucket.historicalSuccess( + tableBucket, 10L, 20L, "dt=2025-01-02")); + } + @Test void testFetchLog() throws Exception { SchemaGetter schemaGetter = diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java index b18f96b3a7a..eea8578ff2a 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java @@ -22,6 +22,7 @@ import org.apache.fluss.config.AutoPartitionTimeUnit; import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; +import org.apache.fluss.exception.ConfigException; import org.apache.fluss.exception.HistoricalPartitionThrottledException; import org.apache.fluss.exception.InvalidPartitionException; import org.apache.fluss.lake.lakestorage.LakeTableLookuper; @@ -83,6 +84,7 @@ import com.github.benmanes.caffeine.cache.Scheduler; import com.github.benmanes.caffeine.cache.Ticker; import org.junit.jupiter.api.Test; +import org.rocksdb.FlushOptions; import javax.annotation.Nullable; @@ -123,6 +125,10 @@ class HistoricalPartitionManagerTest extends ReplicaTestBase { private static final String ANOTHER_ORIGINAL_PARTITION = "20240108"; private static final String HISTORICAL_PARTITION = HISTORICAL_PARTITION_VALUE; private static final TableBucket TABLE_BUCKET = new TableBucket(TABLE_ID, PARTITION_ID, 0); + private static final RowType HISTORICAL_KEY_TYPE = + DataTypes.ROW( + new DataField("id", DataTypes.INT()), + new DataField("region", DataTypes.STRING())); @Test void testResolvesMultipleLakeMissesWithoutPrewriteRollback() throws Exception { @@ -728,6 +734,291 @@ void testRecoversHistoricalOverlayFromLakeCommitOffset() throws Exception { } } + @Test + void testCleansFullyTieredHistoricalOverlayAfterWriteIdleTime() throws Exception { + TableInfo tableInfo = registerHistoricalTableAndBecomeLeader(); + Replica replica = replicaManager.getReplicaOrException(TABLE_BUCKET); + KvTablet originalKvTablet = replica.getKvTablet(); + assertThat(originalKvTablet).isNotNull(); + + Configuration cleanupConf = lookupConfiguration(); + cleanupConf.set( + ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME, + Duration.ofMinutes(1)); + ManuallyTriggeredScheduledExecutorService executor = + new ManuallyTriggeredScheduledExecutorService(); + TestingHistoricalLakeLookupManager lakeLookupManager = + new TestingHistoricalLakeLookupManager(cleanupConf); + HistoricalPartitionManager historicalPartitionManager = + createCleanupManager( + cleanupConf, executor, lakeLookupManager, replica, Long.MAX_VALUE); + + byte[] primaryKey = new CompactedKeyEncoder(HISTORICAL_KEY_TYPE).encodeKey(row(1, "us")); + Object[] valueObjects = new Object[] {1, "us", ORIGINAL_PARTITION, "v1"}; + byte[] lakeValue = + ValueEncoder.encodeValue( + (short) tableInfo.getSchemaId(), + compactedRow(tableInfo.getRowType(), valueObjects)); + lakeLookupManager.putLakeValue(ORIGINAL_PARTITION, lakeValue); + + try { + CompletableFuture putFuture = + putHistoricalRecords( + historicalPartitionManager, + replica, + batch( + HISTORICAL_KEY_TYPE, + tableInfo.getRowType(), + Tuple2.of(new Object[] {1, "us"}, valueObjects))); + executor.triggerAll(); + assertThat(putFuture.get(10, TimeUnit.SECONDS).failed()).isFalse(); + flushAndWait(originalKvTablet, Long.MAX_VALUE); + + manualClock.advanceTime(Duration.ofMinutes(1)); + // The idle policy cannot clean until lake progress covers the local WAL. + assertThat(executor.numQueuedRunnables()).isZero(); + assertThat(replica.getKvTablet()).isSameAs(originalKvTablet); + + long tieredOffset = replica.getLocalLogEndOffset(); + historicalPartitionManager.onLakeProgress(replica, 9L, tieredOffset - 1); + assertThat(executor.numQueuedRunnables()).isZero(); + + replica.getLogTablet().updateLakeLogEndOffset(tieredOffset); + historicalPartitionManager.onLakeProgress(replica, 10L, tieredOffset); + assertThat(executor.numQueuedRunnables()).isOne(); + assertThat(lakeLookupManager.requiredLakeSnapshotIds).isEmpty(); + + // A write admitted after idle cleanup was scheduled must either cancel that cleanup + // or run against the newly created overlay. It must never be lost during the reset. + CompletableFuture laterPut = + putHistoricalRecords( + historicalPartitionManager, + replica, + batch( + HISTORICAL_KEY_TYPE, + tableInfo.getRowType(), + Tuple2.of(new Object[] {1, "us"}, valueObjects))); + executor.triggerAll(); + assertThat(laterPut.get(10, TimeUnit.SECONDS).failed()).isFalse(); + assertThat(replica.getKvTablet()).isSameAs(originalKvTablet); + assertThat(lakeLookupManager.requiredLakeSnapshotIds).isEmpty(); + + flushAndWait(originalKvTablet, Long.MAX_VALUE); + manualClock.advanceTime(Duration.ofMinutes(1)); + long latestTieredOffset = replica.getLocalLogEndOffset(); + replica.getLogTablet().updateLakeLogEndOffset(latestTieredOffset); + + Configuration longerIdleTimeConf = new Configuration(cleanupConf); + longerIdleTimeConf.set( + ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME, + Duration.ofMinutes(2)); + historicalPartitionManager.validate(longerIdleTimeConf); + historicalPartitionManager.reconfigure(longerIdleTimeConf); + historicalPartitionManager.onLakeProgress(replica, 11L, latestTieredOffset); + assertThat(lakeLookupManager.requiredLakeSnapshotIds).isEmpty(); + executor.triggerAll(); + assertThat(replica.getKvTablet()).isSameAs(originalKvTablet); + + Configuration shorterIdleTimeConf = new Configuration(cleanupConf); + historicalPartitionManager.validate(shorterIdleTimeConf); + historicalPartitionManager.reconfigure(shorterIdleTimeConf); + historicalPartitionManager.onLakeProgress(replica, 12L, latestTieredOffset); + executor.triggerAll(); + + KvTablet cleanedKvTablet = replica.getKvTablet(); + assertThat(cleanedKvTablet).isNotNull().isNotSameAs(originalKvTablet); + assertThat(lakeLookupManager.requiredLakeSnapshotIds).containsExactly(12L); + assertThat(cleanedKvTablet.getRocksDBKv().limitScan(10)).isEmpty(); + assertThat(cleanedKvTablet.lookupHistoricalLocal(ORIGINAL_PARTITION, primaryKey)) + .isEqualTo(KvStateLookupResult.notFound()); + + CompletableFuture lookupFuture = + historicalPartitionManager.lookup( + replica, + new LookupDataForBucket( + TABLE_BUCKET, + Collections.singletonList(primaryKey), + ORIGINAL_PARTITION), + (lookupTimeNanos, lookupFileDownloaded) -> {}); + executor.triggerAll(); + assertThat(lookupFuture.get(10, TimeUnit.SECONDS).lookupValues()) + .containsExactly(lakeValue); + } finally { + historicalPartitionManager.close(); + } + } + + @Test + void testRejectsNegativeCleanupIdleTimeDuringReconfiguration() { + Configuration cleanupConf = lookupConfiguration(); + ManuallyTriggeredScheduledExecutorService executor = + new ManuallyTriggeredScheduledExecutorService(); + HistoricalPartitionManager historicalPartitionManager = + new HistoricalPartitionManager( + cleanupConf, + new HistoricalPartitionTaskExecutor(cleanupConf, executor), + new TestingHistoricalLakeLookupManager(cleanupConf), + manualClock, + Long.MAX_VALUE); + Configuration invalidConf = new Configuration(cleanupConf); + invalidConf.set( + ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME, + Duration.ofMillis(-1)); + + try { + assertThatThrownBy(() -> historicalPartitionManager.validate(invalidConf)) + .isInstanceOf(ConfigException.class) + .hasMessageContaining( + ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME.key(), + "must not be negative"); + } finally { + historicalPartitionManager.close(); + } + } + + @Test + void testCleanupRequiresLakeAndLocalOffsetsToMatch() throws Exception { + TableInfo tableInfo = registerHistoricalTableAndBecomeLeader(); + Replica replica = replicaManager.getReplicaOrException(TABLE_BUCKET); + KvTablet originalKvTablet = replica.getKvTablet(); + assertThat(originalKvTablet).isNotNull(); + + Configuration cleanupConf = lookupConfiguration(); + cleanupConf.set( + ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME, + Duration.ofMinutes(1)); + ManuallyTriggeredScheduledExecutorService executor = + new ManuallyTriggeredScheduledExecutorService(); + TestingHistoricalLakeLookupManager lakeLookupManager = + new TestingHistoricalLakeLookupManager(cleanupConf); + HistoricalPartitionManager historicalPartitionManager = + createCleanupManager( + cleanupConf, executor, lakeLookupManager, replica, Long.MAX_VALUE); + + KvRecordBatch records = + batch( + HISTORICAL_KEY_TYPE, + tableInfo.getRowType(), + Tuple2.of( + new Object[] {1, "us"}, + new Object[] {1, "us", ORIGINAL_PARTITION, "v1"})); + + try { + CompletableFuture firstPut = + putHistoricalRecords(historicalPartitionManager, replica, records); + executor.triggerAll(); + assertThat(firstPut.get(10, TimeUnit.SECONDS).failed()).isFalse(); + + long firstTieredOffset = replica.getLocalLogEndOffset(); + replica.getLogTablet().updateLakeLogEndOffset(firstTieredOffset); + historicalPartitionManager.onLakeProgress(replica, 10L, firstTieredOffset); + executor.triggerAll(); + assertThat(replica.getKvTablet()).isSameAs(originalKvTablet); + assertThat(lakeLookupManager.requiredLakeSnapshotIds).isEmpty(); + + CompletableFuture secondPut = + putHistoricalRecords(historicalPartitionManager, replica, records); + manualClock.advanceTime(Duration.ofMinutes(1)); + historicalPartitionManager.onLakeProgress(replica, 10L, firstTieredOffset); + + // The second write runs before the cleanup that captured snapshot 10 / first offset. + executor.trigger(); + assertThat(secondPut.get(10, TimeUnit.SECONDS).failed()).isFalse(); + long secondTieredOffset = replica.getLocalLogEndOffset(); + assertThat(secondTieredOffset).isGreaterThan(firstTieredOffset); + + manualClock.advanceTime(Duration.ofMinutes(1)); + // A lake offset beyond the local end is inconsistent and must not schedule cleanup. + historicalPartitionManager.onLakeProgress(replica, 11L, secondTieredOffset + 1); + executor.trigger(); + + // Snapshot 10 does not cover the second write, so its queued cleanup must be skipped. + assertThat(replica.getKvTablet()).isSameAs(originalKvTablet); + assertThat(lakeLookupManager.requiredLakeSnapshotIds).isEmpty(); + assertThat(executor.numQueuedRunnables()).isZero(); + + replica.getLogTablet().updateLakeLogEndOffset(secondTieredOffset); + historicalPartitionManager.onLakeProgress(replica, 12L, secondTieredOffset); + executor.triggerAll(); + assertThat(replica.getKvTablet()).isNotNull().isNotSameAs(originalKvTablet); + assertThat(lakeLookupManager.requiredLakeSnapshotIds).containsExactly(12L); + } finally { + historicalPartitionManager.close(); + } + } + + @Test + void testMaxSizeBlocksWritesUntilOverlayIsTieredAndCleaned() throws Exception { + TableInfo tableInfo = registerHistoricalTableAndBecomeLeader(); + Replica replica = replicaManager.getReplicaOrException(TABLE_BUCKET); + KvTablet originalKvTablet = replica.getKvTablet(); + assertThat(originalKvTablet).isNotNull(); + + Configuration cleanupConf = lookupConfiguration(); + cleanupConf.set( + ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME, Duration.ZERO); + ManuallyTriggeredScheduledExecutorService executor = + new ManuallyTriggeredScheduledExecutorService(); + HistoricalPartitionManager historicalPartitionManager = + createCleanupManager( + cleanupConf, + executor, + new TestingHistoricalLakeLookupManager(cleanupConf), + replica, + 1L); + + KvRecordBatch firstBatch = + batch( + HISTORICAL_KEY_TYPE, + tableInfo.getRowType(), + Tuple2.of( + new Object[] {1, "us"}, + new Object[] {1, "us", ORIGINAL_PARTITION, "v1"})); + KvRecordBatch secondBatch = + batch( + HISTORICAL_KEY_TYPE, + tableInfo.getRowType(), + Tuple2.of( + new Object[] {2, "eu"}, + new Object[] {2, "eu", ORIGINAL_PARTITION, "v2"})); + + try { + CompletableFuture firstPut = + putHistoricalRecords(historicalPartitionManager, replica, firstBatch); + executor.triggerAll(); + assertThat(firstPut.get(10, TimeUnit.SECONDS).failed()).isFalse(); + flushAndWait(originalKvTablet, Long.MAX_VALUE); + try (FlushOptions flushOptions = new FlushOptions().setWaitForFlush(true)) { + originalKvTablet.getRocksDBKv().getDb().flush(flushOptions); + } + assertThat(originalKvTablet.liveSstFilesSize()).isPositive(); + + PutKvResultForBucket blockedWrite = + putHistoricalRecords(historicalPartitionManager, replica, secondBatch) + .get(10, TimeUnit.SECONDS); + assertThat(blockedWrite.getError().error()) + .isEqualTo(Errors.HISTORICAL_PARTITION_THROTTLED); + assertThat(blockedWrite.getError().message()) + .contains( + "reached the live SST maximum size of 1 bytes", + "lake tiering covers all previously accepted writes"); + assertThat(executor.numQueuedRunnables()).isZero(); + + long tieredOffset = replica.getLocalLogEndOffset(); + replica.getLogTablet().updateLakeLogEndOffset(tieredOffset); + historicalPartitionManager.onLakeProgress(replica, 11L, tieredOffset); + executor.triggerAll(); + assertThat(replica.getKvTablet()).isNotNull().isNotSameAs(originalKvTablet); + + CompletableFuture resumedWrite = + putHistoricalRecords(historicalPartitionManager, replica, secondBatch); + executor.triggerAll(); + assertThat(resumedWrite.get(10, TimeUnit.SECONDS).failed()).isFalse(); + } finally { + historicalPartitionManager.close(); + } + } + @Test void testHistoricalLookupThrottledWhenPermitsExhausted() throws Exception { registerHistoricalTableAndBecomeLeader(); @@ -936,6 +1227,33 @@ private static void await(CountDownLatch latch) { } } + private HistoricalPartitionManager createCleanupManager( + Configuration configuration, + ManuallyTriggeredScheduledExecutorService executor, + TestingHistoricalLakeLookupManager lakeLookupManager, + Replica replica, + long maxHistoricalKvSizeBytes) { + HistoricalPartitionManager manager = + new HistoricalPartitionManager( + configuration, + new HistoricalPartitionTaskExecutor(configuration, executor), + lakeLookupManager, + manualClock, + maxHistoricalKvSizeBytes); + manager.onLeaderActivated(replica); + return manager; + } + + private static CompletableFuture putHistoricalRecords( + HistoricalPartitionManager manager, Replica replica, KvRecordBatch records) { + return manager.putKv( + replica, + new PutKvDataForBucket(TABLE_BUCKET, records, ORIGINAL_PARTITION), + null, + MergeMode.DEFAULT, + 1); + } + @SafeVarargs private static KvRecordBatch batch( RowType keyType, RowType rowType, Tuple2... keyAndValues) @@ -971,6 +1289,7 @@ private final class TestingHistoricalLakeLookupManager extends HistoricalLakeLoo private final AtomicInteger lookupCount = new AtomicInteger(); private final AtomicInteger lookupBatchCount = new AtomicInteger(); private final Map lakeValuesByPartition = new HashMap<>(); + private final List requiredLakeSnapshotIds = new ArrayList<>(); private volatile @Nullable Runnable lookupHook; private TestingHistoricalLakeLookupManager(Configuration configuration) { @@ -992,6 +1311,12 @@ private void setLookupHook(Runnable lookupHook) { this.lookupHook = lookupHook; } + @Override + void requireLakeSnapshot(long tableId, long snapshotId) { + requiredLakeSnapshotIds.add(snapshotId); + super.requireLakeSnapshot(tableId, snapshotId); + } + @Override List lookup( LookupDataForBucket lookupData, diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionTaskExecutorTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionTaskExecutorTest.java index 98574e7d9fd..4293fc3d0df 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionTaskExecutorTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionTaskExecutorTest.java @@ -148,6 +148,37 @@ void testOrderedTaskContinuesAfterPreviousFailure() throws Exception { assertThat(taskExecutor.numInflightRequests()).isZero(); } + @Test + void testMaintenanceTaskKeepsOrderWithoutConsumingRequestPermit() throws Exception { + ManualExecutor executor = new ManualExecutor(); + HistoricalPartitionTaskExecutor taskExecutor = + new HistoricalPartitionTaskExecutor(configuration(1), executor); + List executionOrder = new ArrayList<>(); + + CompletableFuture write = + taskExecutor.submitOrdered( + "bucket", + () -> { + executionOrder.add("write"); + return "written"; + }, + () -> "throttled"); + CompletableFuture maintenance = + taskExecutor.submitOrderedMaintenance( + "bucket", () -> executionOrder.add("maintenance")); + + assertThat(taskExecutor.numInflightRequests()).isOne(); + assertThat(taskExecutor.submitOrdered("another-bucket", () -> "written", () -> "throttled")) + .isCompletedWithValue("throttled"); + + executor.runNext(); + executor.runNext(); + assertThat(write).isCompletedWithValue("written"); + assertThat(maintenance).isDone(); + assertThat(executionOrder).containsExactly("write", "maintenance"); + assertThat(taskExecutor.numInflightRequests()).isZero(); + } + @Test void testRejectNonPositiveRequestLimit() { assertThatThrownBy( diff --git a/fluss-server/src/test/java/org/apache/fluss/server/utils/HistoricalPartitionTableValidationTest.java b/fluss-server/src/test/java/org/apache/fluss/server/utils/HistoricalPartitionTableValidationTest.java index cb427c23ec8..7fa4a90a1d6 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/utils/HistoricalPartitionTableValidationTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/utils/HistoricalPartitionTableValidationTest.java @@ -26,6 +26,7 @@ import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThatCode; import static org.assertj.core.api.Assertions.assertThatThrownBy; class HistoricalPartitionTableValidationTest { @@ -54,7 +55,6 @@ void testReportsAllUnmetHistoricalPartitionRequirements() { + "'table.datalake.enabled' must be set to true; " + "'table.datalake.format' must be set to 'paimon' " + "(currently not set); " - + "the table must define a primary key; " + "the table must define exactly one partition key (found 0)."); // Case 2: Aggregate requirements before related validators can report only one failure. @@ -80,7 +80,32 @@ void testReportsAllUnmetHistoricalPartitionRequirements() { "'table.datalake.historical-partition.enabled' has unmet requirements: " + "'table.datalake.format' must be set to 'paimon' " + "(currently 'iceberg'); " - + "the table must define a primary key; " + "the table must define exactly one partition key (found 0)."); } + + @Test + void testAllowsHistoricalPartitionForLogTable() { + TableDescriptor logTableDescriptor = + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("dt", DataTypes.STRING()) + .build()) + .partitionedBy("dt") + .distributedBy(1, "id") + .property(ConfigOptions.TABLE_REPLICATION_FACTOR, 1) + .property(ConfigOptions.TABLE_AUTO_PARTITION_ENABLED, true) + .property(ConfigOptions.TABLE_AUTO_PARTITION_KEY, "dt") + .property(ConfigOptions.TABLE_DATALAKE_ENABLED, true) + .property(ConfigOptions.TABLE_DATALAKE_FORMAT, DataLakeFormat.PAIMON) + .property(ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_ENABLED, true) + .build(); + + assertThatCode( + () -> + TableDescriptorValidation.validateTableDescriptor( + logTableDescriptor, 100, DataLakeFormat.PAIMON)) + .doesNotThrowAnyException(); + } } diff --git a/fluss-server/src/test/java/org/apache/fluss/server/utils/ServerRpcMessageUtilsTest.java b/fluss-server/src/test/java/org/apache/fluss/server/utils/ServerRpcMessageUtilsTest.java index 9f318eded4c..c5ed0c48a6b 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/utils/ServerRpcMessageUtilsTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/utils/ServerRpcMessageUtilsTest.java @@ -21,12 +21,18 @@ import org.apache.fluss.record.KvRecordBatch; import org.apache.fluss.row.encode.KvValueLayout; import org.apache.fluss.rpc.entity.LookupResultForBucket; +import org.apache.fluss.rpc.entity.ProduceLogResultForBucket; import org.apache.fluss.rpc.entity.PutKvResultForBucket; import org.apache.fluss.rpc.messages.LookupResponse; +import org.apache.fluss.rpc.messages.PbProduceLogReqForBucket; +import org.apache.fluss.rpc.messages.PbProduceLogRespForBucket; import org.apache.fluss.rpc.messages.PbPutKvReqForBucket; import org.apache.fluss.rpc.messages.PbPutKvRespForBucket; +import org.apache.fluss.rpc.messages.ProduceLogRequest; +import org.apache.fluss.rpc.messages.ProduceLogResponse; import org.apache.fluss.rpc.messages.PutKvRequest; import org.apache.fluss.rpc.messages.PutKvResponse; +import org.apache.fluss.server.entity.ProduceLogDataForBucket; import org.apache.fluss.server.entity.PutKvDataForBucket; import org.junit.jupiter.api.Test; @@ -64,6 +70,52 @@ void testLookupResponseSupportsMixedValueLayouts() { .containsExactly(new byte[] {1, 0, 11, 12}, new byte[] {1, 0, 21, 22}); } + @Test + void testHistoricalProduceLogRequestAndResponsePreserveOriginalPartitions() { + long tableId = 1L; + long partitionId = 2L; + PbProduceLogReqForBucket firstBucketRequest = + new PbProduceLogReqForBucket() + .setPartitionId(partitionId) + .setBucketId(0) + .setRecords(new byte[0]) + .setOriginalPartitionName("dt=2025-01-01"); + ProduceLogRequest request = + new ProduceLogRequest().setTableId(tableId).setAcks(1).setTimeoutMs(10_000); + request.addAllBucketsReqs( + Arrays.asList( + firstBucketRequest, + new PbProduceLogReqForBucket() + .copyFrom(firstBucketRequest) + .setOriginalPartitionName("dt=2025-01-02"))); + + TableBucket tableBucket = new TableBucket(tableId, partitionId, 0); + List decoded = + ServerRpcMessageUtils.toProduceLogDataForBuckets(request); + assertThat(decoded) + .extracting(ProduceLogDataForBucket::tableBucket) + .containsOnly(tableBucket); + assertThat(decoded) + .extracting(ProduceLogDataForBucket::originalPartitionName) + .containsExactly("dt=2025-01-01", "dt=2025-01-02"); + + ProduceLogResponse response = + ServerRpcMessageUtils.makeProduceLogResponse( + Arrays.asList( + ProduceLogResultForBucket.historicalSuccess( + tableBucket, 0L, 1L, "dt=2025-01-01"), + ProduceLogResultForBucket.historicalSuccess( + tableBucket, 1L, 2L, "dt=2025-01-02"))); + assertThat(response.getBucketsRespsList()) + .extracting(PbProduceLogRespForBucket::getOriginalPartitionName) + .containsExactly("dt=2025-01-01", "dt=2025-01-02"); + + request.addAllBucketsReqs(Collections.singletonList(firstBucketRequest)); + assertThatThrownBy(() -> ServerRpcMessageUtils.toProduceLogDataForBuckets(request)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("duplicate table bucket"); + } + @Test void testHistoricalPutKvRequestAndResponsePreserveOriginalPartitions() throws Exception { long tableId = 1L; From e3adb4ad568a9cb8b287e6a668894e93b47e15da Mon Sep 17 00:00:00 2001 From: luoyuxia Date: Thu, 27 Aug 2026 19:28:52 +0800 Subject: [PATCH 2/4] [client][server][paimon] Refine historical partition writes Simplify historical write routing, request handling, and Paimon tiering integration while removing redundant tests. Co-Authored-By: Codex AI-Model: gpt-5 AI-Contributed/Feature: 446/446 AI-Contributed/UT: 576/576 --- .../fluss/client/write/RecordAccumulator.java | 124 +++++++---- .../org/apache/fluss/client/write/Sender.java | 105 +++++---- .../fluss/client/write/WriterClient.java | 45 ++-- .../client/write/RecordAccumulatorTest.java | 88 -------- .../apache/fluss/client/write/SenderTest.java | 203 +----------------- .../apache/fluss/config/ConfigOptions.java | 2 +- .../source/split/TieringSplitGenerator.java | 48 +++-- .../lake/paimon/tiering/RecordWriter.java | 1 + .../tiering/mergetree/MergeTreeWriter.java | 18 -- .../lookup/HistoricalPartitionITCase.java | 68 ------ .../paimon/tiering/PaimonTieringTest.java | 113 ++++------ .../rpc/entity/ProduceLogResultForBucket.java | 1 + .../netty/client/ServerConnectionTest.java | 14 +- .../entity/ProduceLogDataForBucket.java | 1 + .../apache/fluss/server/replica/Replica.java | 82 ++++--- .../fluss/server/replica/ReplicaManager.java | 19 +- .../fluss/server/DynamicConfigChangeTest.java | 32 +-- .../HistoricalPartitionTaskExecutorTest.java | 31 --- ...istoricalPartitionTableValidationTest.java | 27 --- 19 files changed, 285 insertions(+), 737 deletions(-) diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java b/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java index 7546e48b8e5..84ee7331e19 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java @@ -342,41 +342,67 @@ public void reEnqueue(ReadyWriteBatch readyWriteBatch) { } /** - * Tries to route writes for an original partition path to the given physical target. + * Routes writes for an original partition path to the given physical target. * *

The accumulator keeps queues keyed by {@code originalPath}, while metadata lookup, leader * discovery, and RPC sending use {@code targetPath}. The target may therefore be either the * original partition itself or the shared historical partition. * - *

The first queue creation fixes the target for that original path. A later call succeeds - * only when it selects the same target; this method never moves queued or inflight batches - * between physical partitions. + *

A normal target can be replaced by the historical target only while the original path has + * no incomplete batch. This method never moves queued or inflight batches between physical + * partitions. * - * @return true if the target was installed or already matches, false if a different target was - * fixed previously + * @throws FlussRuntimeException if a different target was fixed previously */ - boolean tryRouteWritesTo( + void routeWritesTo( PhysicalTablePath originalPath, PhysicalTablePath targetPath, long targetPartitionId) { BucketAndWriteBatches resolvedTarget = new BucketAndWriteBatches(targetPartitionId, true, targetPath); // Install the route atomically before append can create the first queue for this path. BucketAndWriteBatches existing = writeBatches.putIfAbsent(originalPath, resolvedTarget); if (existing == null) { - return true; + return; } - // An append may already have fixed this path to a target. Keep that target and only accept - // the metadata result when it describes the same physical partition. - if (!existing.targetPath.equals(targetPath)) { - return false; + synchronized (existing) { + if (existing.targetPath.equals(targetPath)) { + existing.partitionId = targetPartitionId; + return; + } + if (!existing.isHistoricalWriteTarget() && resolvedTarget.isHistoricalWriteTarget()) { + if (hasIncompleteBatchFor(originalPath)) { + throw new FlussRuntimeException( + String.format( + "Cannot route writes for %s to %s while this writer has " + + "incomplete writes to %s.", + originalPath, targetPath, existing.targetPath)); + } + existing.targetPath = targetPath; + existing.partitionId = targetPartitionId; + return; + } + + throw new FlussRuntimeException( + String.format( + "Cannot route writes for %s to %s because this writer already routed " + + "the partition to %s.", + originalPath, targetPath, existing.targetPath)); } - existing.partitionId = targetPartitionId; - return true; } - /** Returns whether a write target has already been chosen for this original path. */ - boolean hasWriteTarget(PhysicalTablePath originalPath) { - return writeBatches.containsKey(originalPath); + private boolean hasIncompleteBatchFor(PhysicalTablePath physicalTablePath) { + for (WriteBatch batch : incomplete.copyAll()) { + if (batch.physicalTablePath().equals(physicalTablePath)) { + return true; + } + } + return false; + } + + /** Returns whether this original path is already routed to the historical partition. */ + boolean hasHistoricalWriteTarget(PhysicalTablePath originalPath) { + BucketAndWriteBatches writeTarget = writeBatches.get(originalPath); + return writeTarget != null && writeTarget.isHistoricalWriteTarget(); } /** Returns whether the target belongs to a table with historical partition support enabled. */ @@ -671,41 +697,43 @@ private RecordAppendResult appendNewBatch( Deque deque, List segments) throws Exception { - RecordAppendResult appendResult = tryAppend(writeRecord, callback, deque); - if (appendResult != null) { - // Somebody else found us a batch, return the one we waited for! Hopefully this doesn't - // happen often... - return appendResult; - } - PhysicalTablePath physicalTablePath = writeRecord.getPhysicalTablePath(); - PreAllocatedPagedOutputView outputView = new PreAllocatedPagedOutputView(segments); - int schemaId = tableInfo.getSchemaId(); - WriteFormat writeFormat = writeRecord.getWriteFormat(); BucketAndWriteBatches bucketAndWriteBatches = checkNotNull( writeBatches.get(physicalTablePath), "Write batches for %s must exist.", physicalTablePath); - String originalPartitionName = - bucketAndWriteBatches.isHistoricalWriteTarget() - ? checkNotNull(physicalTablePath.getPartitionName()) - : null; - final WriteBatch batch = - createWriteBatch( - writeRecord, - bucketId, - tableInfo, - writeFormat, - physicalTablePath, - outputView, - schemaId, - originalPartitionName); + synchronized (bucketAndWriteBatches) { + RecordAppendResult appendResult = tryAppend(writeRecord, callback, deque); + if (appendResult != null) { + // Somebody else found us a batch, return the one we waited for! Hopefully this + // doesn't happen often... + return appendResult; + } - batch.tryAppend(writeRecord, callback); - deque.addLast(batch); - incomplete.add(batch); - return new RecordAppendResult(deque.size() > 1 || batch.isClosed(), true, false); + PreAllocatedPagedOutputView outputView = new PreAllocatedPagedOutputView(segments); + int schemaId = tableInfo.getSchemaId(); + WriteFormat writeFormat = writeRecord.getWriteFormat(); + String originalPartitionName = + bucketAndWriteBatches.isHistoricalWriteTarget() + ? checkNotNull(physicalTablePath.getPartitionName()) + : null; + final WriteBatch batch = + createWriteBatch( + writeRecord, + bucketId, + tableInfo, + writeFormat, + physicalTablePath, + outputView, + schemaId, + originalPartitionName); + + batch.tryAppend(writeRecord, callback); + deque.addLast(batch); + incomplete.add(batch); + return new RecordAppendResult(deque.size() > 1 || batch.isClosed(), true, false); + } } private WriteBatch createWriteBatch( @@ -1084,6 +1112,8 @@ private List getAllBucketsInCurrentNode(Integer currentNode, Clu Set physicalTablePaths = cluster.getBucketLocationsByPath().keySet(); for (PhysicalTablePath path : physicalTablePaths) { BucketAndWriteBatches bucketAndWriteBatches = writeBatches.get(path); + // A historical route uses the original path only as the accumulator queue key. Its + // actual bucket locations come from the historical target and are added below. if (bucketAndWriteBatches != null && bucketAndWriteBatches.isHistoricalWriteTarget()) { continue; } @@ -1111,6 +1141,8 @@ private List getAllBucketsInCurrentNode(Integer currentNode, Clu bucketAndWriteBatches.targetPath)) { if (bucketLocation.getLeader() != null && Objects.equals(currentNode, bucketLocation.getLeader())) { + // Keep the original path so drain can find its original-keyed queue. The + // TableBucket, leader, and replicas still describe the historical RPC target. buckets.add( new BucketLocation( originalPath, @@ -1260,7 +1292,7 @@ public void destroyResources() { private static class BucketAndWriteBatches { public final boolean isPartitionedTable; /** The physical partition used for metadata lookup, leader discovery, and write RPCs. */ - private final PhysicalTablePath targetPath; + private volatile PhysicalTablePath targetPath; public volatile @Nullable Long partitionId; // Write batches for each bucket in queue. diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java b/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java index 2c71c584717..29600e2521b 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java @@ -393,38 +393,24 @@ private void sendWriteRequest(int destination, short acks, List batches); } else { writeBatchByTable.forEach( - (tableId, writeBatches) -> { - boolean logBatches = isLogBatches(writeBatches); - for (List requestGroup : packRequestGroups(writeBatches)) { - if (logBatches) { - sendProduceLogRequestAndHandleResponse( - gateway, - makeProduceLogRequest( - tableId, acks, maxRequestTimeoutMs, requestGroup), - tableId, - requestGroup); - } else { - sendPutKvRequestAndHandleResponse( - gateway, - makePutKvRequest( - tableId, acks, maxRequestTimeoutMs, requestGroup), - tableId, - requestGroup); - } - } - }); + (tableId, writeBatches) -> + sendWriteRequestsForTable(gateway, tableId, acks, writeBatches)); } } /** - * Splits normal and historical batches into separate requests. + * Sends normal and historical batches in separate requests. * *

Normal and historical writes cannot share a request. Both write protocols correlate * historical responses by {@link TableBucket} and original partition name, so different * original partitions targeting the same historical table bucket can remain in one request. */ - private static List> packRequestGroups( + private void sendWriteRequestsForTable( + TabletServerGateway gateway, + long tableId, + short acks, List writeBatches) { + boolean logBatches = isLogBatches(writeBatches); List normalBatches = new ArrayList<>(); List historicalBatches = new ArrayList<>(); @@ -436,32 +422,50 @@ private static List> packRequestGroups( } } - List> requestGroups = new ArrayList<>(2); - if (!normalBatches.isEmpty()) { - requestGroups.add(normalBatches); + sendBatchesInRequest(gateway, tableId, acks, logBatches, normalBatches); + sendBatchesInRequest(gateway, tableId, acks, logBatches, historicalBatches); + } + + private void sendBatchesInRequest( + TabletServerGateway gateway, + long tableId, + short acks, + boolean logBatches, + List writeBatches) { + if (writeBatches.isEmpty()) { + return; } - if (!historicalBatches.isEmpty()) { - requestGroups.add(historicalBatches); + if (logBatches) { + sendProduceLogRequestAndHandleResponse( + gateway, + makeProduceLogRequest(tableId, acks, maxRequestTimeoutMs, writeBatches), + tableId, + writeBatches); + } else { + sendPutKvRequestAndHandleResponse( + gateway, + makePutKvRequest(tableId, acks, maxRequestTimeoutMs, writeBatches), + tableId, + writeBatches); } - return requestGroups; } - private static Map toBatchesByKey( + private static Map toWriteBatchesByKey( List writeBatches) { - Map recordsByKey = new HashMap<>(); + Map writeBatchesByKey = new HashMap<>(); for (ReadyWriteBatch readyWriteBatch : writeBatches) { WriteBatch writeBatch = readyWriteBatch.writeBatch(); WriteBatchKey key = new WriteBatchKey( readyWriteBatch.tableBucket(), writeBatch.getOriginalPartitionName()); - ReadyWriteBatch previous = recordsByKey.put(key, readyWriteBatch); + ReadyWriteBatch previous = writeBatchesByKey.put(key, readyWriteBatch); checkArgument( previous == null, "A write request contains duplicate table bucket %s and original partition %s.", readyWriteBatch.tableBucket(), writeBatch.getOriginalPartitionName()); } - return recordsByKey; + return writeBatchesByKey; } /** @@ -482,7 +486,7 @@ private void sendProduceLogRequestAndHandleResponse( ProduceLogRequest request, long tableId, List writeBatches) { - Map recordsByKey = toBatchesByKey(writeBatches); + Map writeBatchesByKey = toWriteBatchesByKey(writeBatches); long startTime = System.currentTimeMillis(); gateway.produceLog(request) .whenComplete( @@ -492,7 +496,8 @@ private void sendProduceLogRequestAndHandleResponse( if (e != null) { handleWriteRequestException(e, writeBatches); } else { - handleProduceLogResponse(produceLogResponse, tableId, recordsByKey); + handleProduceLogResponse( + produceLogResponse, tableId, writeBatchesByKey); } }); } @@ -502,7 +507,7 @@ private void sendPutKvRequestAndHandleResponse( PutKvRequest request, long tableId, List writeBatches) { - Map recordsByKey = toBatchesByKey(writeBatches); + Map writeBatchesByKey = toWriteBatchesByKey(writeBatches); long startTime = System.currentTimeMillis(); gateway.putKv(request) .whenComplete( @@ -512,7 +517,7 @@ private void sendPutKvRequestAndHandleResponse( if (e != null) { handleWriteRequestException(e, writeBatches); } else { - handlePutKvResponse(putKvResponse, tableId, recordsByKey); + handlePutKvResponse(putKvResponse, tableId, writeBatchesByKey); } }); } @@ -520,7 +525,7 @@ private void sendPutKvRequestAndHandleResponse( private void handleProduceLogResponse( ProduceLogResponse response, long tableId, - Map recordsByKey) { + Map writeBatchesByKey) { Set invalidMetadataTablesSet = new HashSet<>(); for (PbProduceLogRespForBucket logRespForBucket : response.getBucketsRespsList()) { TableBucket tb = @@ -531,7 +536,7 @@ private void handleProduceLogResponse( : null, logRespForBucket.getBucketId()); ReadyWriteBatch writeBatch = - recordsByKey.get( + writeBatchesByKey.get( new WriteBatchKey( tb, logRespForBucket.hasOriginalPartitionName() @@ -552,7 +557,7 @@ private void handleProduceLogResponse( private void handlePutKvResponse( PutKvResponse putKvResponse, long tableId, - Map recordsByKey) { + Map writeBatchesByKey) { Set invalidMetadataTablesSet = new HashSet<>(); for (PbPutKvRespForBucket respForBucket : putKvResponse.getBucketsRespsList()) { TableBucket tb = @@ -567,7 +572,7 @@ private void handlePutKvResponse( } ReadyWriteBatch writeBatch = - recordsByKey.get( + writeBatchesByKey.get( new WriteBatchKey( tb, respForBucket.hasOriginalPartitionName() @@ -704,6 +709,11 @@ private Set handleWriteBatchException( return invalidMetadataTables; } + /** + * Aborts pending writes when metadata confirms their target is missing. Such writes cannot make + * progress without a leader, and rerouting existing batches to the historical target is unsafe + * because their outcome and idempotent state may belong to the original target. + */ private void abortIfHistoricalWriteTargetMissing(Set unknownLeaderTables) throws Exception { for (PhysicalTablePath targetPath : unknownLeaderTables) { @@ -715,6 +725,18 @@ private void abortIfHistoricalWriteTargetMissing(Set unknownL } catch (Exception e) { Throwable t = ExceptionUtils.stripExecutionException(e); if (t instanceof PartitionNotExistException) { + // This target was considered usable when its batches were enqueued or first + // attempted, but is now confirmed missing. Transparently rerouting those + // batches to the historical partition is unsafe: an original-target attempt + // may have been accepted despite a lost response, and writer ID / batch + // sequence state cannot be reused across different physical TableBuckets. A + // safe failover must stop draining this path, wait for its in-flight requests, + // classify ambiguous outcomes, reset writer state, and preserve per-bucket + // ordering. This race requires partition retirement to overlap a writer that + // still holds the original route, so it is expected to be uncommon; fail + // closed for now. + // TODO: Implement safe in-flight historical failover if this path occurs + // frequently in practice. // Retrying a historical-enabled table without a leader would leave its // batches queued indefinitely. Fail only after checking the target itself so // ordinary writes in the bulk metadata request keep their existing behavior. @@ -806,6 +828,9 @@ void destroyResources() { private static final class WriteBatchKey { private final TableBucket tableBucket; + + // Distinguishes historical writes from different original partitions that share a target + // bucket. Normal writes have no original partition name. private final @Nullable String originalPartitionName; private WriteBatchKey(TableBucket tableBucket, @Nullable String originalPartitionName) { diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/WriterClient.java b/fluss-client/src/main/java/org/apache/fluss/client/write/WriterClient.java index cb46c627bed..e82bc57f8fb 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/WriterClient.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/WriterClient.java @@ -78,7 +78,6 @@ public class WriterClient { private static final Logger LOG = LoggerFactory.getLogger(WriterClient.class); public static final String SENDER_THREAD_PREFIX = "fluss-write-sender"; - private static final Duration MAX_DEFAULT_TIME_ZONE_DIFFERENCE = Duration.ofHours(26); /** * {@link ConfigOptions#CLIENT_WRITER_MAX_INFLIGHT_REQUESTS_PER_BUCKET} should be less than or * equal to this value when idempotence producer enabled to ensure message ordering. @@ -254,6 +253,18 @@ private void doSend(WriteRecord record, WriteCallback callback) { } } + /** + * Returns whether a partition is old enough that it may have expired under its retention + * policy. + * + *

This client-side precheck uses the time zone resolved from the table configuration. A + * {@code true} result does not confirm that the partition is missing; the caller must refresh + * metadata before routing the write to a historical partition. If the table does not explicitly + * configure a time zone and the Client and Coordinator use different defaults, they may + * classify partitions near the retention boundary differently. Late classification may fail a + * write to an already removed original partition, while early classification only causes an + * extra metadata refresh. + */ static boolean mayBeExpiredHistoricalPartition( PhysicalTablePath physicalTablePath, TableInfo tableInfo, Instant now) { String partitionName = physicalTablePath.getPartitionName(); @@ -264,25 +275,19 @@ static boolean mayBeExpiredHistoricalPartition( return false; } - // The table's default time zone is not persisted. Shift the expiration boundary by the - // largest IANA time-zone difference, then apply retention in the table's partition unit. - Instant latestPotentialServerTime = now.plus(MAX_DEFAULT_TIME_ZONE_DIFFERENCE); - if (!isPastAutoPartition(partitionName, strategy, latestPotentialServerTime)) { + if (!isPastAutoPartition(partitionName, strategy, now)) { return false; } - ZonedDateTime latestPotentialServerDateTime = - ZonedDateTime.ofInstant(latestPotentialServerTime, strategy.timeZone().toZoneId()); - String earliestPotentialRetainedPartition = + ZonedDateTime currentDateTime = + ZonedDateTime.ofInstant(now, strategy.timeZone().toZoneId()); + String earliestRetainedPartition = generateAutoPartitionTime( - latestPotentialServerDateTime, - -strategy.numToRetain(), - strategy.timeUnit(), - strategy); - return partitionName.compareTo(earliestPotentialRetainedPartition) < 0; + currentDateTime, -strategy.numToRetain(), strategy.timeUnit(), strategy); + return partitionName.compareTo(earliestRetainedPartition) < 0; } private synchronized void resolveHistoricalWriteTarget(PhysicalTablePath originalPath) { - if (accumulator.hasWriteTarget(originalPath)) { + if (accumulator.hasHistoricalWriteTarget(originalPath)) { return; } @@ -307,16 +312,8 @@ private synchronized void resolveHistoricalWriteTarget(PhysicalTablePath origina } } - if (!accumulator.tryRouteWritesTo( - originalPath, targetPath, metadataUpdater.getPartitionIdOrElseThrow(targetPath))) { - throw new FlussRuntimeException( - "Cannot route writes for " - + originalPath - + " to " - + targetPath - + " because the accumulator already contains writes for a different " - + "physical target."); - } + accumulator.routeWritesTo( + originalPath, targetPath, metadataUpdater.getPartitionIdOrElseThrow(targetPath)); } private void maybeAbortBatches(Throwable t) { diff --git a/fluss-client/src/test/java/org/apache/fluss/client/write/RecordAccumulatorTest.java b/fluss-client/src/test/java/org/apache/fluss/client/write/RecordAccumulatorTest.java index 21939d16888..acd1e4e2911 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/write/RecordAccumulatorTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/write/RecordAccumulatorTest.java @@ -43,7 +43,6 @@ import org.apache.fluss.row.BinaryRow; import org.apache.fluss.row.GenericRow; import org.apache.fluss.row.arrow.ArrowWriter; -import org.apache.fluss.row.encode.CompactedKeyEncoder; import org.apache.fluss.row.indexed.IndexedRow; import org.apache.fluss.rpc.GatewayClientProxy; import org.apache.fluss.rpc.RpcClient; @@ -72,21 +71,14 @@ import static org.apache.fluss.record.LogRecordBatch.CURRENT_LOG_MAGIC_VALUE; import static org.apache.fluss.record.LogRecordBatchFormat.recordBatchHeaderSize; import static org.apache.fluss.record.TestData.DATA1_PHYSICAL_TABLE_PATH; -import static org.apache.fluss.record.TestData.DATA1_PHYSICAL_TABLE_PATH_PK_PA_2024; import static org.apache.fluss.record.TestData.DATA1_ROW_TYPE; import static org.apache.fluss.record.TestData.DATA1_SCHEMA; -import static org.apache.fluss.record.TestData.DATA1_SCHEMA_PK; import static org.apache.fluss.record.TestData.DATA1_TABLE_ID; -import static org.apache.fluss.record.TestData.DATA1_TABLE_ID_PK; import static org.apache.fluss.record.TestData.DATA1_TABLE_INFO; -import static org.apache.fluss.record.TestData.DATA1_TABLE_INFO_PK; import static org.apache.fluss.record.TestData.DATA1_TABLE_PATH; -import static org.apache.fluss.record.TestData.DATA1_TABLE_PATH_PK; import static org.apache.fluss.record.TestData.DEFAULT_REMOTE_DATA_DIR; -import static org.apache.fluss.testutils.DataTestUtils.compactedRow; import static org.apache.fluss.testutils.DataTestUtils.indexedRow; import static org.apache.fluss.testutils.DataTestUtils.row; -import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -199,41 +191,6 @@ void testDrainBatches() throws Exception { verifyTableBucketInBatches(batches3, tb1, tb3); } - @Test - void testAppendAfterHistoricalTargetResolved() throws Exception { - long originalPartitionId = 11L; - long historicalPartitionId = 22L; - PhysicalTablePath originalPath = DATA1_PHYSICAL_TABLE_PATH_PK_PA_2024; - PhysicalTablePath anotherOriginalPath = PhysicalTablePath.of(DATA1_TABLE_PATH_PK, "2023"); - PhysicalTablePath historicalPath = - PhysicalTablePath.of(DATA1_TABLE_PATH_PK, HISTORICAL_PARTITION_VALUE); - TableBucket originalBucket = new TableBucket(DATA1_TABLE_ID_PK, originalPartitionId, 0); - TableBucket historicalBucket = new TableBucket(DATA1_TABLE_ID_PK, historicalPartitionId, 0); - cluster = - createPartitionedKvCluster( - originalPath, originalBucket, historicalPath, historicalBucket); - - RecordAccumulator accum = createTestRecordAccumulator(1024, 10L * 1024); - accum.tryRouteWritesTo(originalPath, historicalPath, historicalPartitionId); - accum.tryRouteWritesTo(anotherOriginalPath, historicalPath, historicalPartitionId); - accum.append(createKvRecord(originalPath), writeCallback, cluster, 0, false); - accum.append(createKvRecord(originalPath), writeCallback, cluster, 0, false); - accum.append(createKvRecord(anotherOriginalPath), writeCallback, cluster, 0, false); - - List drainedBatches = - accum.drain(cluster, Collections.singleton(node1.id()), Integer.MAX_VALUE) - .get(node1.id()); - - assertThat(drainedBatches).hasSize(2); - assertThat(drainedBatches) - .allSatisfy(batch -> assertThat(batch.tableBucket()).isEqualTo(historicalBucket)); - assertThat(drainedBatches) - .extracting(batch -> ((KvWriteBatch) batch.writeBatch()).getOriginalPartitionName()) - .containsExactlyInAnyOrder( - originalPath.getPartitionName(), anotherOriginalPath.getPartitionName()); - drainedBatches.forEach(batch -> accum.deallocate(batch.writeBatch())); - } - @Test void testDrainCompressedBatches() throws Exception { int batchSize = 10 * 1024; @@ -627,21 +584,6 @@ private WriteRecord createRecord(IndexedRow row, TableInfo tableInfo) { return WriteRecord.forIndexedAppend(tableInfo, DATA1_PHYSICAL_TABLE_PATH, row, null); } - private WriteRecord createKvRecord(PhysicalTablePath physicalTablePath) { - BinaryRow row = compactedRow(DATA1_ROW_TYPE, new Object[] {1, "a"}); - byte[] key = - new CompactedKeyEncoder(DATA1_ROW_TYPE, DATA1_SCHEMA_PK.getPrimaryKeyIndexes()) - .encodeKey(row); - return WriteRecord.forUpsert( - DATA1_TABLE_INFO_PK, - physicalTablePath, - row, - key, - key, - WriteFormat.COMPACTED_KV, - null); - } - private TableInfo withSchemaId(int schemaId) { return new TableInfo( DATA1_TABLE_INFO.getTablePath(), @@ -680,36 +622,6 @@ private Cluster updateCluster(List bucketLocations) { Collections.emptyMap()); } - private Cluster createPartitionedKvCluster( - PhysicalTablePath originalPath, - TableBucket originalBucket, - PhysicalTablePath historicalPath, - TableBucket historicalBucket) { - Map aliveTabletServersById = new HashMap<>(); - aliveTabletServersById.put(node1.id(), node1); - - Map> bucketsByPath = new HashMap<>(); - bucketsByPath.put( - originalPath, - Collections.singletonList( - new BucketLocation(originalPath, originalBucket, node1.id(), serverNodes))); - bucketsByPath.put( - historicalPath, - Collections.singletonList( - new BucketLocation( - historicalPath, historicalBucket, node1.id(), serverNodes))); - - Map partitionIdsByPath = new HashMap<>(); - partitionIdsByPath.put(originalPath, originalBucket.getPartitionId()); - partitionIdsByPath.put(historicalPath, historicalBucket.getPartitionId()); - return new Cluster( - aliveTabletServersById, - new ServerNode(0, "localhost", 89, ServerType.COORDINATOR), - bucketsByPath, - Collections.singletonMap(DATA1_TABLE_PATH_PK, DATA1_TABLE_ID_PK), - partitionIdsByPath); - } - private void delayedInterrupt(final Thread thread, final long delayMs) { Thread t = new Thread( diff --git a/fluss-client/src/test/java/org/apache/fluss/client/write/SenderTest.java b/fluss-client/src/test/java/org/apache/fluss/client/write/SenderTest.java index 3dc684a13a3..c3228c37a69 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/write/SenderTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/write/SenderTest.java @@ -42,7 +42,6 @@ import org.apache.fluss.row.BinaryRow; import org.apache.fluss.row.GenericRow; import org.apache.fluss.row.encode.CompactedKeyEncoder; -import org.apache.fluss.row.indexed.IndexedRow; import org.apache.fluss.rpc.entity.ProduceLogResultForBucket; import org.apache.fluss.rpc.entity.PutKvResultForBucket; import org.apache.fluss.rpc.messages.ApiMessage; @@ -92,7 +91,6 @@ import static org.apache.fluss.server.utils.ServerRpcMessageUtils.makePutKvResponse; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.toProduceLogDataForBuckets; import static org.apache.fluss.testutils.DataTestUtils.compactedRow; -import static org.apache.fluss.testutils.DataTestUtils.indexedRow; import static org.apache.fluss.testutils.DataTestUtils.row; import static org.apache.fluss.testutils.common.CommonTestUtils.retry; import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; @@ -128,44 +126,6 @@ public void teardown() throws Exception { sender.destroyResources(); } - @Test - void testSendsHistoricalPutWhenTargetResolvedBeforeAppend() throws Exception { - sender.destroyResources(); - String originalPartitionName = "20000101"; - TableInfo tableInfo = createHistoricalTableInfo(); - PhysicalTablePath originalPath = - PhysicalTablePath.of(tableInfo.getTablePath(), originalPartitionName); - PhysicalTablePath historicalPath = - PhysicalTablePath.of(tableInfo.getTablePath(), HISTORICAL_PARTITION_VALUE); - long historicalPartitionId = 22L; - TableBucket historicalBucket = - new TableBucket(tableInfo.getTableId(), historicalPartitionId, 0); - metadataUpdater = - new TestingMetadataUpdater( - Collections.singletonMap(tableInfo.getTablePath(), tableInfo)); - metadataUpdater.updateCluster( - partitionedCluster( - tableInfo, Collections.singletonMap(historicalPath, historicalBucket))); - sender = setupWithIdempotenceState(); - accumulator.tryRouteWritesTo(originalPath, historicalPath, historicalPartitionId); - - CompletableFuture future = - appendKvRecord(tableInfo, originalPath, 1, metadataUpdater.getCluster()); - - sender.runOnce(); - - assertThat(sender.numOfInFlightBatches(historicalBucket)).isOne(); - TestTabletServerGateway gateway = node1Gateway(); - PutKvRequest request = (PutKvRequest) gateway.getRequest(0); - assertThat(request.getBucketsReqAt(0).getPartitionId()).isEqualTo(historicalPartitionId); - assertThat(request.getBucketsReqAt(0).getOriginalPartitionName()) - .isEqualTo(originalPartitionName); - - gateway.response( - 0, createHistoricalPutKvResponse(historicalBucket, 1L, originalPartitionName)); - assertThat(future.get()).isNull(); - } - @Test void testPotentialExpirationUsesAutoPartitionTimeUnit() { TableInfo tableInfo = createHistoricalTableInfo(AutoPartitionTimeUnit.HOUR, 48); @@ -173,13 +133,13 @@ void testPotentialExpirationUsesAutoPartitionTimeUnit() { assertThat( WriterClient.mayBeExpiredHistoricalPartition( - PhysicalTablePath.of(tableInfo.getTablePath(), "2026082301"), + PhysicalTablePath.of(tableInfo.getTablePath(), "2026082123"), tableInfo, now)) .isTrue(); assertThat( WriterClient.mayBeExpiredHistoricalPartition( - PhysicalTablePath.of(tableInfo.getTablePath(), "2026082302"), + PhysicalTablePath.of(tableInfo.getTablePath(), "2026082200"), tableInfo, now)) .isFalse(); @@ -214,33 +174,7 @@ void testFailsWriteAfterMetadataConfirmsPartitionMissing() throws Exception { } @Test - void testMissingPartitionDoesNotAbortNormalWrites() throws Exception { - sender.destroyResources(); - TableInfo tableInfo = createNormalPartitionedTableInfo(); - PhysicalTablePath partitionPath = - PhysicalTablePath.of(tableInfo.getTablePath(), "20990101"); - TableBucket tableBucket = new TableBucket(tableInfo.getTableId(), 21L, 0); - metadataUpdater = missingPartitionMetadataUpdater(tableInfo); - metadataUpdater.updateCluster( - partitionedCluster( - tableInfo, Collections.singletonMap(partitionPath, tableBucket))); - sender = setupWithIdempotenceState(); - - CompletableFuture future = - appendKvRecord(tableInfo, partitionPath, 1, metadataUpdater.getCluster()); - sender.runOnce(); - - TestTabletServerGateway gateway = node1Gateway(); - gateway.response( - 0, createPutKvResponse(tableBucket, Errors.UNKNOWN_TABLE_OR_BUCKET_EXCEPTION)); - sender.runOnce(); - - assertThat(future).isNotDone(); - accumulator.abortAllBatches(new RuntimeException("Test cleanup.")); - } - - @Test - void testPackNormalAndHistoricalPutRequests() throws Exception { + void testNormalAndHistoricalPutRequests() throws Exception { sender.destroyResources(); TableInfo tableInfo = createHistoricalTableInfo(); PhysicalTablePath activePath = PhysicalTablePath.of(tableInfo.getTablePath(), "20990101"); @@ -264,9 +198,9 @@ void testPackNormalAndHistoricalPutRequests() throws Exception { CompletableFuture activeFuture = appendKvRecord(tableInfo, activePath, 1, metadataUpdater.getCluster()); - accumulator.tryRouteWritesTo( + accumulator.routeWritesTo( firstOriginalPath, historicalPath, historicalBucket.getPartitionId()); - accumulator.tryRouteWritesTo( + accumulator.routeWritesTo( secondOriginalPath, historicalPath, historicalBucket.getPartitionId()); CompletableFuture firstHistoricalFuture = appendKvRecord(tableInfo, firstOriginalPath, 2, metadataUpdater.getCluster()); @@ -311,80 +245,11 @@ void testPackNormalAndHistoricalPutRequests() throws Exception { historicalBucket, 1L, firstOriginalPath.getPartitionName())))); - assertThat(activeFuture).isDone(); - assertThat(firstHistoricalFuture).isDone(); - assertThat(secondHistoricalFuture).isDone(); assertThat(activeFuture.get()).isNull(); assertThat(firstHistoricalFuture.get()).isNull(); assertThat(secondHistoricalFuture.get()).isNull(); } - @Test - void testPackHistoricalProduceLogRequests() throws Exception { - sender.destroyResources(); - TableInfo tableInfo = createHistoricalLogTableInfo(); - PhysicalTablePath firstOriginalPath = - PhysicalTablePath.of(tableInfo.getTablePath(), "20000101"); - PhysicalTablePath secondOriginalPath = - PhysicalTablePath.of(tableInfo.getTablePath(), "20000102"); - PhysicalTablePath historicalPath = - PhysicalTablePath.of(tableInfo.getTablePath(), HISTORICAL_PARTITION_VALUE); - TableBucket historicalBucket = new TableBucket(tableInfo.getTableId(), 22L, 0); - - metadataUpdater = - new TestingMetadataUpdater( - Collections.singletonMap(tableInfo.getTablePath(), tableInfo)); - metadataUpdater.updateCluster( - partitionedCluster( - tableInfo, Collections.singletonMap(historicalPath, historicalBucket))); - sender = setupWithIdempotenceState(); - - accumulator.tryRouteWritesTo( - firstOriginalPath, historicalPath, historicalBucket.getPartitionId()); - accumulator.tryRouteWritesTo( - secondOriginalPath, historicalPath, historicalBucket.getPartitionId()); - CompletableFuture firstFuture = - appendLogRecord(tableInfo, firstOriginalPath, 1, metadataUpdater.getCluster()); - CompletableFuture secondFuture = - appendLogRecord(tableInfo, secondOriginalPath, 2, metadataUpdater.getCluster()); - - sender.runOnce(); - - TestTabletServerGateway gateway = node1Gateway(); - assertThat(gateway.pendingRequestSize()).isOne(); - ProduceLogRequest request = (ProduceLogRequest) gateway.getRequest(0); - assertThat(request.getBucketsReqsCount()).isEqualTo(2); - Set originalPartitionNames = new HashSet<>(); - for (int i = 0; i < request.getBucketsReqsCount(); i++) { - assertThat(request.getBucketsReqAt(i).getPartitionId()) - .isEqualTo(historicalBucket.getPartitionId()); - originalPartitionNames.add(request.getBucketsReqAt(i).getOriginalPartitionName()); - } - assertThat(originalPartitionNames) - .containsExactlyInAnyOrder( - firstOriginalPath.getPartitionName(), - secondOriginalPath.getPartitionName()); - - gateway.response( - 0, - makeProduceLogResponse( - Arrays.asList( - ProduceLogResultForBucket.historicalSuccess( - historicalBucket, - 1L, - 2L, - secondOriginalPath.getPartitionName()), - ProduceLogResultForBucket.historicalSuccess( - historicalBucket, - 0L, - 1L, - firstOriginalPath.getPartitionName())))); - assertThat(firstFuture).isDone(); - assertThat(secondFuture).isDone(); - assertThat(firstFuture.get()).isNull(); - assertThat(secondFuture.get()).isNull(); - } - @Test void testSimple() throws Exception { long offset = 0; @@ -1401,15 +1266,6 @@ private static TableInfo createHistoricalTableInfo() { private static TableInfo createHistoricalTableInfo( AutoPartitionTimeUnit timeUnit, int numToRetain) { - return createPartitionedKvTableInfo(timeUnit, numToRetain, true); - } - - private static TableInfo createNormalPartitionedTableInfo() { - return createPartitionedKvTableInfo(AutoPartitionTimeUnit.DAY, 7, false); - } - - private static TableInfo createPartitionedKvTableInfo( - AutoPartitionTimeUnit timeUnit, int numToRetain, boolean historicalPartitionEnabled) { Schema schema = Schema.newBuilder() .column("id", DataTypes.INT()) @@ -1427,9 +1283,7 @@ private static TableInfo createPartitionedKvTableInfo( .property(ConfigOptions.TABLE_AUTO_PARTITION_TIMEZONE, "UTC") .property(ConfigOptions.TABLE_DATALAKE_ENABLED, true) .property(ConfigOptions.TABLE_DATALAKE_FORMAT, DataLakeFormat.PAIMON) - .property( - ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_ENABLED, - historicalPartitionEnabled) + .property(ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_ENABLED, true) .build(); return TableInfo.of( DATA1_TABLE_PATH_PK, @@ -1456,26 +1310,6 @@ public boolean checkAndUpdatePartitionMetadata(PhysicalTablePath physicalTablePa }; } - private static TableInfo createHistoricalLogTableInfo() { - Schema schema = - Schema.newBuilder() - .column("id", DataTypes.INT()) - .column("dt", DataTypes.STRING()) - .build(); - TableDescriptor descriptor = - TableDescriptor.builder() - .schema(schema) - .partitionedBy("dt") - .distributedBy(1, "id") - .property(ConfigOptions.TABLE_AUTO_PARTITION_ENABLED, true) - .property(ConfigOptions.TABLE_DATALAKE_ENABLED, true) - .property(ConfigOptions.TABLE_DATALAKE_FORMAT, DataLakeFormat.PAIMON) - .property(ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_ENABLED, true) - .build(); - return TableInfo.of( - DATA1_TABLE_PATH, DATA1_TABLE_ID, 1, descriptor, DEFAULT_REMOTE_DATA_DIR, 1L, 1L); - } - private static Cluster partitionedCluster( TableInfo tableInfo, Map tableBucketsByPath) { int[] replicas = new int[] {TestingMetadataUpdater.NODE1.id()}; @@ -1531,23 +1365,6 @@ private CompletableFuture appendKvRecord( return future; } - private CompletableFuture appendLogRecord( - TableInfo tableInfo, PhysicalTablePath physicalTablePath, int id, Cluster cluster) - throws Exception { - IndexedRow row = - indexedRow( - tableInfo.getRowType(), - new Object[] {id, physicalTablePath.getPartitionName()}); - CompletableFuture future = new CompletableFuture<>(); - accumulator.append( - WriteRecord.forIndexedAppend(tableInfo, physicalTablePath, row, null), - (tableBucket, logEndOffset, error) -> future.complete(error), - cluster, - 0, - false); - return future; - } - private void appendToAccumulator(TableBucket tb, GenericRow row, WriteCallback writeCallback) throws Exception { appendToAccumulator(DATA1_TABLE_INFO, tb, row, writeCallback); @@ -1690,14 +1507,6 @@ private PutKvResponse createPutKvResponse(TableBucket tb, long endOffset) { Collections.singletonList(new PutKvResultForBucket(tb, endOffset))); } - private PutKvResponse createHistoricalPutKvResponse( - TableBucket tb, long endOffset, String originalPartitionName) { - return makePutKvResponse( - Collections.singletonList( - PutKvResultForBucket.historicalSuccess( - tb, endOffset, originalPartitionName))); - } - private PutKvResponse createPutKvResponse(TableBucket tb, long endOffset, float pressure) { return makePutKvResponse( Collections.singletonList(new PutKvResultForBucket(tb, endOffset, pressure))); diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java index 6685381cb0e..fe2d538cfaa 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java @@ -444,7 +444,7 @@ public class ConfigOptions { .durationType() .defaultValue(Duration.ofMinutes(30)) .withDescription( - "The historical KV write idle time after which a fully tiered local overlay can be cleaned. " + "The idle time after which fully tiered historical KV write state in the local overlay can be cleaned. " + "Set to 0 to disable idle cleanup."); public static final ConfigOption SERVER_DATA_DISK_WRITE_LIMIT_RATIO = diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/split/TieringSplitGenerator.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/split/TieringSplitGenerator.java index fc2cbd9a03d..a6282cfeabd 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/split/TieringSplitGenerator.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/split/TieringSplitGenerator.java @@ -141,24 +141,31 @@ private List generatePartitionTableSplit( .boxed() .collect(Collectors.toList())); KvSnapshots latestKvSnapshots = null; - if (tableInfo.hasPrimaryKey() && !historicalPartition) { - // get the table partition latest kv snapshot info - try { + if (tableInfo.hasPrimaryKey()) { + if (historicalPartition) { + // Historical KV replicas use the lake snapshot as their durable base and tier + // only the retained WAL, so they have no local KV snapshots to tier. latestKvSnapshots = - flussAdmin - .getLatestKvSnapshots(tableInfo.getTablePath(), partitionName) - .get(); - } catch (Exception e) { - throw new FlinkRuntimeException( - String.format( - "Failed to get table snapshot for table %s and partition %s", - tableInfo.getTablePath(), partitionName), - ExceptionUtils.stripCompletionException(e)); + new KvSnapshots( + tableInfo.getTableId(), + partitionId, + Collections.emptyMap(), + Collections.emptyMap()); + } else { + try { + latestKvSnapshots = + flussAdmin + .getLatestKvSnapshots(tableInfo.getTablePath(), partitionName) + .get(); + } catch (Exception e) { + throw new FlinkRuntimeException( + String.format( + "Failed to get table snapshot for table %s and partition %s", + tableInfo.getTablePath(), partitionName), + ExceptionUtils.stripCompletionException(e)); + } } } - // Historical KV replicas do not create regular KV snapshots. Their lake snapshot is - // the durable base, so tier them from the retained WAL like log tables. - splits.addAll( generateTableSplit( tableInfo, @@ -166,8 +173,7 @@ private List generatePartitionTableSplit( partitionName, lakeSnapshotInfo, latestKvSnapshots, - latestBucketsOffset, - historicalPartition)); + latestBucketsOffset)); } return splits; } @@ -202,8 +208,7 @@ private List generateNonPartitionedTableSplit( null, lakeSnapshotInfo, latestKvSnapshots, - latestBucketsOffset, - false); + latestBucketsOffset); } private List generateTableSplit( @@ -212,11 +217,10 @@ private List generateTableSplit( @Nullable String partitionName, @Nullable LakeSnapshot lakeSnapshotInfo, @Nullable KvSnapshots latestKvSnapshots, - Map latestBucketsOffset, - boolean historicalPartition) { + Map latestBucketsOffset) { List splits = new ArrayList<>(); - if (tableInfo.hasPrimaryKey() && !historicalPartition) { + if (tableInfo.hasPrimaryKey()) { // it's primary key table checkState(latestKvSnapshots != null); for (int bucket = 0; bucket < tableInfo.getNumBuckets(); bucket++) { diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/RecordWriter.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/RecordWriter.java index 6e762a96141..25d43d8400f 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/RecordWriter.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/RecordWriter.java @@ -42,6 +42,7 @@ public abstract class RecordWriter implements AutoCloseable { protected final int bucket; protected final List partitionKeys; protected final boolean historicalPartition; + // Null for historical writers, which derive the original partition from each record. protected final @Nullable BinaryRow fixedPartition; protected final FlussRecordAsPaimonRow flussRecordAsPaimonRow; diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/mergetree/MergeTreeWriter.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/mergetree/MergeTreeWriter.java index 7d48c850e9a..4dbc8100cb5 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/mergetree/MergeTreeWriter.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/mergetree/MergeTreeWriter.java @@ -45,24 +45,6 @@ public class MergeTreeWriter extends RecordWriter { private final IOManager ioManager; - public MergeTreeWriter( - FileStoreTable fileStoreTable, - TableBucket tableBucket, - @Nullable String partition, - List partitionKeys, - RowType flussRowType, - boolean paimonIncludingSystemColumns) { - this( - fileStoreTable, - tableBucket, - partition, - partitionKeys, - flussRowType, - (String[]) null, - paimonIncludingSystemColumns, - false); - } - public MergeTreeWriter( FileStoreTable fileStoreTable, TableBucket tableBucket, diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/HistoricalPartitionITCase.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/HistoricalPartitionITCase.java index a135d520162..0da14e9f710 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/HistoricalPartitionITCase.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/HistoricalPartitionITCase.java @@ -118,48 +118,6 @@ void testWriteAndTierHistoricalKvToPaimon() throws Exception { } } - @Test - void testWriteAndTierHistoricalLogToPaimon() throws Exception { - TablePath tablePath = TablePath.of(DEFAULT_DB, "historical_log_write_tiering"); - Schema schema = partitionedLogSchema(); - long tableId = createTable(tablePath, partitionedLogDescriptor(schema)); - - try { - long historicalPartitionId = waitUntilHistoricalPartitionReady(tablePath, tableId); - - List expectedRows = - Arrays.asList( - row(1, EXPIRED_PARTITION_NAME, "Alice"), - row(2, SECOND_EXPIRED_PARTITION_NAME, "Bob")); - writeRows(tablePath, expectedRows, true); - assertThat(admin.listPartitionInfos(tablePath).get()) - .noneMatch( - partitionInfo -> - EXPIRED_PARTITION_NAME.equals(partitionInfo.getPartitionName()) - || SECOND_EXPIRED_PARTITION_NAME.equals( - partitionInfo.getPartitionName())); - - TableBucket historicalBucket = new TableBucket(tableId, historicalPartitionId, 0); - assertThat(getLeaderReplica(historicalBucket).getLocalLogEndOffset()).isEqualTo(2); - - JobClient jobClient = buildTieringJob(execEnv); - try { - assertReplicaStatus(historicalBucket, 2); - checkFlussOffsetsInSnapshot( - tablePath, Collections.singletonMap(historicalBucket, 2L)); - - assertThat(readPaimonRows(tablePath)) - .containsExactlyInAnyOrder( - "1|" + EXPIRED_PARTITION_NAME + "|Alice", - "2|" + SECOND_EXPIRED_PARTITION_NAME + "|Bob"); - } finally { - jobClient.cancel().get(); - } - } finally { - dropTable(tablePath); - } - } - @ParameterizedTest(name = "defaultBucketKey={0}") @ValueSource(booleans = {true, false}) void testLookupExpiredPartitionFromPaimon(boolean defaultBucketKey) throws Exception { @@ -387,14 +345,6 @@ private static Schema partitionedPkSchema(boolean defaultBucketKey) { .build(); } - private static Schema partitionedLogSchema() { - return Schema.newBuilder() - .column("id", DataTypes.INT()) - .column("dt", DataTypes.STRING()) - .column("name", DataTypes.STRING()) - .build(); - } - private static Schema evolvedPartitionedPkSchema(boolean defaultBucketKey) { if (defaultBucketKey) { return Schema.newBuilder() @@ -447,24 +397,6 @@ private static TableDescriptor partitionedPkDescriptor( return builder.build(); } - private static TableDescriptor partitionedLogDescriptor(Schema schema) { - return TableDescriptor.builder() - .schema(schema) - .distributedBy(1, "id") - .partitionedBy("dt") - .property(ConfigOptions.TABLE_AUTO_PARTITION_ENABLED, true) - .property(ConfigOptions.TABLE_AUTO_PARTITION_KEY, "dt") - .property(ConfigOptions.TABLE_AUTO_PARTITION_TIME_UNIT, AutoPartitionTimeUnit.DAY) - .property( - ConfigOptions.TABLE_AUTO_PARTITION_NUM_RETENTION, - EXPIRED_PARTITION_RETENTION) - .property(ConfigOptions.TABLE_AUTO_PARTITION_TIMEZONE, "UTC") - .property(ConfigOptions.TABLE_DATALAKE_ENABLED, true) - .property(ConfigOptions.TABLE_DATALAKE_FRESHNESS, Duration.ofMillis(500)) - .property(ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_ENABLED, true) - .build(); - } - private static InternalRow dataRow( boolean defaultBucketKey, int id, String subId, String name) { return dataRow(defaultBucketKey, id, subId, name, EXPIRED_PARTITION_NAME); diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringTest.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringTest.java index e689778cf12..b960caacdf5 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringTest.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringTest.java @@ -17,7 +17,6 @@ package org.apache.fluss.lake.paimon.tiering; -import org.apache.fluss.config.AutoPartitionTimeUnit; import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; import org.apache.fluss.lake.batch.ArrowRecordBatch; @@ -67,13 +66,11 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import org.junit.jupiter.params.provider.ValueSource; import javax.annotation.Nullable; import java.io.File; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -88,6 +85,7 @@ import static org.apache.fluss.metadata.TableDescriptor.BUCKET_COLUMN_NAME; import static org.apache.fluss.metadata.TableDescriptor.OFFSET_COLUMN_NAME; import static org.apache.fluss.metadata.TableDescriptor.TIMESTAMP_COLUMN_NAME; +import static org.apache.fluss.record.ChangeType.APPEND_ONLY; import static org.apache.fluss.record.ChangeType.DELETE; import static org.apache.fluss.record.ChangeType.INSERT; import static org.apache.fluss.record.ChangeType.UPDATE_AFTER; @@ -224,20 +222,15 @@ void testTieringWriteTable(boolean isPrimaryKeyTable, boolean isPartitioned) thr } } - @ParameterizedTest - @ValueSource(booleans = {false, true}) - void testHistoricalPartitionTiering(boolean isPrimaryKeyTable) throws Exception { - TablePath tablePath = - TablePath.of( - "paimon", "test_historical_" + (isPrimaryKeyTable ? "primary_key" : "log")); - TableInfo tableInfo = createHistoricalTable(tablePath, isPrimaryKeyTable); + @Test + void testHistoricalPrimaryKeyTiering() throws Exception { + TablePath tablePath = TablePath.of("paimon", "test_historical_primary_key"); + TableInfo tableInfo = createHistoricalTable(tablePath, true); long timestamp = 1_000L; List records = Arrays.asList( - historicalRecord( - 0L, timestamp, 1, "partition-1", "20240101", isPrimaryKeyTable), - historicalRecord( - 1L, timestamp, 1, "partition-2", "20240102", isPrimaryKeyTable)); + historicalRecord(0L, timestamp, 1, "20240101", INSERT), + historicalRecord(1L, timestamp, 1, "20240102", INSERT)); PaimonWriteResult writeResult; try (LakeWriter lakeWriter = @@ -248,16 +241,22 @@ void testHistoricalPartitionTiering(boolean isPrimaryKeyTable) throws Exception writeResult = lakeWriter.complete(); } - assertThat(writeResult.commitMessages()).hasSize(2); SimpleVersionedSerializer serializer = paimonLakeTieringFactory.getWriteResultSerializer(); - assertThat(serializer.getVersion()).isEqualTo(1); - byte[] serialized = serializer.serialize(writeResult); - writeResult = serializer.deserialize(serializer.getVersion(), serialized); - assertThat(writeResult.commitMessages()).hasSize(2); + writeResult = + serializer.deserialize( + serializer.getVersion(), serializer.serialize(writeResult)); + assertHistoricalPartitions(writeResult); - commitWriteResults(tablePath, tableInfo, Collections.singletonList(writeResult)); - verifyHistoricalRecords(tablePath, isPrimaryKeyTable, records); + try (LakeCommitter committer = + createLakeCommitter(tablePath, tableInfo, new Configuration())) { + committer.commit( + committer.toCommittable(Collections.singletonList(writeResult)), + Collections.emptyMap()); + } + assertThat(paimonCatalog.listPartitions(toPaimon(tablePath))) + .extracting(partition -> partition.spec().get("c3")) + .containsExactlyInAnyOrder("20240101", "20240102"); } @Test @@ -268,10 +267,8 @@ void testHistoricalArrowBatchTiering() throws Exception { long timestamp = 1_000L; List records = Arrays.asList( - historicalRecord( - baseOffset, timestamp, 1, "partition-1", "20240101", false), - historicalRecord( - baseOffset + 1, timestamp, 2, "partition-2", "20240102", false)); + historicalRecord(baseOffset, timestamp, 1, "20240101", APPEND_ONLY), + historicalRecord(baseOffset + 1, timestamp, 2, "20240102", APPEND_ONLY)); PaimonWriteResult writeResult; try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE); @@ -289,9 +286,7 @@ void testHistoricalArrowBatchTiering() throws Exception { writeResult = lakeWriter.complete(); } - assertThat(writeResult.commitMessages()).hasSize(2); - commitWriteResults(tablePath, tableInfo, Collections.singletonList(writeResult)); - verifyHistoricalRecords(tablePath, false, records); + assertHistoricalPartitions(writeResult); } @Test @@ -675,21 +670,10 @@ private void verifyLogTableRecordsThreePartition( actualRecords.close(); } - private void verifyHistoricalRecords( - TablePath tablePath, boolean isPrimaryKeyTable, List records) - throws Exception { - List partitions = Arrays.asList("20240101", "20240102"); - assertThat(paimonCatalog.listPartitions(toPaimon(tablePath))) - .extracting(partition -> partition.spec().get("c3")) - .containsExactlyInAnyOrderElementsOf(partitions); - for (int i = 0; i < partitions.size(); i++) { - String partition = partitions.get(i); - verifyTableRecords( - getPaimonRows(tablePath, partition, isPrimaryKeyTable, 0), - Collections.singletonList(records.get(i)), - 0, - partition); - } + private void assertHistoricalPartitions(PaimonWriteResult writeResult) { + assertThat(writeResult.commitMessages()) + .extracting(message -> message.partition().getString(0).toString()) + .containsExactlyInAnyOrder("20240101", "20240102"); } private void verifyTableRecords( @@ -837,18 +821,13 @@ private GenericRecord toRecord(long offset, GenericRow row, ChangeType changeTyp } private LogRecord historicalRecord( - long offset, - long timestamp, - int key, - String value, - String partition, - boolean isPrimaryKeyTable) { - GenericRow row = new GenericRow(3); - row.setField(0, key); - row.setField(1, BinaryString.fromString(value)); - row.setField(2, BinaryString.fromString(partition)); - return new GenericRecord( - offset, timestamp, isPrimaryKeyTable ? INSERT : ChangeType.APPEND_ONLY, row); + long offset, long timestamp, int key, String partition, ChangeType changeType) { + GenericRow row = + GenericRow.of( + key, + BinaryString.fromString("value"), + BinaryString.fromString(partition)); + return new GenericRecord(offset, timestamp, changeType, row); } private void writeArrowRows(VectorSchemaRoot root, List records) { @@ -859,9 +838,8 @@ private void writeArrowRows(VectorSchemaRoot root, List records) { for (int i = 0; i < records.size(); i++) { org.apache.fluss.row.InternalRow row = records.get(i).getRow(); keyVector.setSafe(i, row.getInt(0)); - valueVector.setSafe(i, row.getString(1).toString().getBytes(StandardCharsets.UTF_8)); - partitionVector.setSafe( - i, row.getString(2).toString().getBytes(StandardCharsets.UTF_8)); + valueVector.setSafe(i, row.getString(1).toBytes()); + partitionVector.setSafe(i, row.getString(2).toBytes()); } root.setRowCount(records.size()); } @@ -1063,29 +1041,10 @@ private TableInfo createHistoricalTable(TablePath tablePath, boolean isPrimaryKe .partitionedBy("c3") .distributedBy(1) .property(ConfigOptions.TABLE_DATALAKE_ENABLED, true) - .property(ConfigOptions.TABLE_AUTO_PARTITION_ENABLED, true) - .property(ConfigOptions.TABLE_AUTO_PARTITION_KEY, "c3") - .property( - ConfigOptions.TABLE_AUTO_PARTITION_TIME_UNIT, - AutoPartitionTimeUnit.DAY) .build(); return TableInfo.of(tablePath, 0, 1, descriptor, DEFAULT_REMOTE_DATA_DIR, 1L, 1L); } - private void commitWriteResults( - TablePath tablePath, TableInfo tableInfo, List writeResults) - throws Exception { - try (LakeCommitter committer = - createLakeCommitter(tablePath, tableInfo, new Configuration())) { - PaimonCommittable committable = committer.toCommittable(writeResults); - assertThat( - committer - .commit(committable, Collections.emptyMap()) - .getCommittedSnapshotId()) - .isOne(); - } - } - private void createMultiPartitionTable(TablePath tablePath) throws Exception { Schema.Builder builder = Schema.newBuilder() diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/ProduceLogResultForBucket.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/ProduceLogResultForBucket.java index 98812e97447..434e4c68f43 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/ProduceLogResultForBucket.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/ProduceLogResultForBucket.java @@ -31,6 +31,7 @@ @Internal public class ProduceLogResultForBucket extends WriteResultForBucket { private final long baseOffset; + // Identifies the original partition for a historical write; null for a normal write. private final @Nullable String originalPartitionName; public ProduceLogResultForBucket(TableBucket tableBucket, long baseOffset, long endOffset) { diff --git a/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/client/ServerConnectionTest.java b/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/client/ServerConnectionTest.java index 541c6f50ee2..3b4e0596f3e 100644 --- a/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/client/ServerConnectionTest.java +++ b/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/client/ServerConnectionTest.java @@ -70,7 +70,6 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; import static org.apache.fluss.metrics.MetricNames.CLIENT_BYTES_IN_RATE_AVG; import static org.apache.fluss.metrics.MetricNames.CLIENT_BYTES_IN_RATE_TOTAL; @@ -251,8 +250,7 @@ public ChannelFuture connect(String host, int port) { @Test void testRejectHistoricalWritesForOldServer() throws Exception { nettyServer.close(); - OldWriteGatewayService oldGatewayService = new OldWriteGatewayService(); - buildNettyServer(oldGatewayService); + buildNettyServer(new OldWriteGatewayService()); ServerConnection connection = new ServerConnection( @@ -264,7 +262,6 @@ void testRejectHistoricalWritesForOldServer() throws Exception { try { assertThat(connection.send(ApiKeys.PUT_KV, putKvRequest(null)).get()) .isInstanceOf(PutKvResponse.class); - assertThat(oldGatewayService.putKvRequests).hasValue(1); assertThatThrownBy( () -> @@ -275,11 +272,9 @@ void testRejectHistoricalWritesForOldServer() throws Exception { .isInstanceOf(UnsupportedVersionException.class) .hasMessageContaining("require PUT_KV version 3 or newer") .hasMessageContaining("negotiated version 2"); - assertThat(oldGatewayService.putKvRequests).hasValue(1); assertThat(connection.send(ApiKeys.PRODUCE_LOG, produceLogRequest(null)).get()) .isInstanceOf(ProduceLogResponse.class); - assertThat(oldGatewayService.produceLogRequests).hasValue(1); assertThatThrownBy( () -> @@ -292,7 +287,6 @@ void testRejectHistoricalWritesForOldServer() throws Exception { .isInstanceOf(UnsupportedVersionException.class) .hasMessageContaining("require PRODUCE_LOG version 1 or newer") .hasMessageContaining("negotiated version 0"); - assertThat(oldGatewayService.produceLogRequests).hasValue(1); } finally { connection.close().get(); } @@ -346,10 +340,6 @@ private void buildNettyServer(TestingGatewayService gatewayService) throws Excep } private static class OldWriteGatewayService extends TestingTabletGatewayService { - - private final AtomicInteger putKvRequests = new AtomicInteger(); - private final AtomicInteger produceLogRequests = new AtomicInteger(); - @Override public CompletableFuture apiVersions(ApiVersionsRequest request) { return super.apiVersions(request) @@ -368,13 +358,11 @@ public CompletableFuture apiVersions(ApiVersionsRequest req @Override public CompletableFuture putKv(PutKvRequest request) { - putKvRequests.incrementAndGet(); return CompletableFuture.completedFuture(new PutKvResponse()); } @Override public CompletableFuture produceLog(ProduceLogRequest request) { - produceLogRequests.incrementAndGet(); return CompletableFuture.completedFuture(new ProduceLogResponse()); } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/entity/ProduceLogDataForBucket.java b/fluss-server/src/main/java/org/apache/fluss/server/entity/ProduceLogDataForBucket.java index 50897fea1dc..96232246152 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/entity/ProduceLogDataForBucket.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/entity/ProduceLogDataForBucket.java @@ -26,6 +26,7 @@ public class ProduceLogDataForBucket { private final TableBucket tableBucket; private final MemoryLogRecords records; + // Identifies the original partition for a historical write; null for a normal write. private final @Nullable String originalPartitionName; public ProduceLogDataForBucket( diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java index 0f578468a23..c099d7a4acb 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java @@ -382,35 +382,28 @@ public long getLakeLogEndOffset() { return logTablet.getLakeLogEndOffset(); } - /** Returns whether the lake and local log end offsets match for historical KV cleanup. */ - public boolean isHistoricalKvCleanupReady() { - return inReadLock( - leaderIsrUpdateLock, - () -> { - long localLogEndOffset = logTablet.localLogEndOffset(); - return isHistoricalKvCleanupReady(localLogEndOffset); - }); - } - /** * Drops and recreates a fully tiered historical KV overlay if leadership and offsets still * match. * * @param expectedLeaderEpoch leader epoch captured when cleanup was scheduled * @param logEndOffset matching lake and local log end offset that triggered cleanup - * @param prepareLakeLookup marks the covering lake snapshot before the overlay is dropped + * @param beforeCleanup action to run before the overlay is dropped * @return whether the overlay was cleaned */ public boolean cleanupHistoricalKv( - int expectedLeaderEpoch, long logEndOffset, Runnable prepareLakeLookup) { - checkNotNull(prepareLakeLookup, "prepareLakeLookup must not be null"); + int expectedLeaderEpoch, long logEndOffset, Runnable beforeCleanup) { + checkNotNull(beforeCleanup, "beforeCleanup must not be null"); return inWriteLock( leaderIsrUpdateLock, () -> { long localLogEndOffset = logTablet.localLogEndOffset(); + // Keep the scheduled snapshot and offset paired: newer lake progress may make + // the current lake and local offsets match while this task still references an + // older snapshot. if (leaderEpoch != expectedLeaderEpoch || localLogEndOffset != logEndOffset - || !isHistoricalKvCleanupReady(localLogEndOffset)) { + || !isEligibleForHistoricalKvCleanup(localLogEndOffset)) { return false; } @@ -420,17 +413,14 @@ public boolean cleanupHistoricalKv( tableBucket, localLogEndOffset, logTablet.getLakeLogEndOffset()); - try { - // A lookup started after the rebuilt empty overlay is published must open - // a lake view that covers the state removed by this cleanup. - prepareLakeLookup.run(); - dropKv(); - createHistoricalKvAfterCleanup(); - return true; - } catch (RuntimeException e) { - fatalErrorHandler.onFatalError(e); - throw e; - } + // A lookup started after the rebuilt empty overlay is published must open a + // lake view that covers the state removed by this cleanup. + beforeCleanup.run(); + dropKv(); + // TODO: Retry rebuilding this historical bucket instead of waiting for + // failover or restart. + createKv(); + return true; }); } @@ -779,7 +769,9 @@ private void logTableConfigChanges(TableInfo oldTableInfo, TableInfo newTableInf } } - private boolean isHistoricalKvCleanupReady(long localLogEndOffset) { + private boolean isEligibleForHistoricalKvCleanup(long localLogEndOffset) { + // The overlay must have a known lake base, contain writes after that base, and have all + // those writes covered by lake before it can be discarded. return isLeader() && isHistoricalPartition() && isKvTable() @@ -804,18 +796,34 @@ private void createKv() { // init kv tablet and get the snapshot it uses to init if have any Optional snapshotUsed = Optional.empty(); + Exception lastError = null; for (int i = 1; i <= INIT_KV_TABLET_MAX_RETRY_TIMES; i++) { try { snapshotUsed = initKvTablet(); + lastError = null; break; } catch (Exception e) { + lastError = e; LOG.warn( - "Fail to init kv tablet for bucket {}, retrying for {} times", + "Failed to init kv tablet for bucket {} on attempt {}/{}.", tableBucket, i, + INIT_KV_TABLET_MAX_RETRY_TIMES, e); } } + if (lastError != null) { + try { + dropKv(); + } catch (Exception cleanupError) { + lastError.addSuppressed(cleanupError); + } + throw new KvStorageException( + String.format( + "Failed to create KV tablet for bucket %s after %s attempts.", + tableBucket, INIT_KV_TABLET_MAX_RETRY_TIMES), + lastError); + } // A historical KV tablet is a disposable overlay over the lake snapshot. It is recovered // by replaying WAL from the lake log end offset and does not create its own KV snapshots. if (!isHistoricalPartition()) { @@ -841,25 +849,6 @@ private void dropKv() { historicalKvBaseOffset = -1L; } - private void createHistoricalKvAfterCleanup() { - checkState(isHistoricalPartition(), "Only a historical KV overlay can be cleaned."); - try { - closeableRegistryForKv = new CloseableRegistry(); - closeableRegistry.registerCloseable(closeableRegistryForKv); - initKvTablet(); - } catch (Exception e) { - try { - dropKv(); - } catch (Exception cleanupError) { - e.addSuppressed(cleanupError); - } - throw new KvStorageException( - String.format( - "Failed to recreate historical KV overlay for bucket %s.", tableBucket), - e); - } - } - private void mayFlushKv(long newHighWatermark) { KvTablet kvTablet = this.kvTablet; if (kvTablet != null) { @@ -2637,4 +2626,5 @@ public SchemaGetter getSchemaGetter() { public PeriodicSnapshotManager getKvSnapshotManager() { return kvSnapshotManager; } + } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java index 67fc81a3350..663ff0b336e 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java @@ -127,6 +127,7 @@ import org.apache.fluss.utils.FileUtils; import org.apache.fluss.utils.FlussPaths; import org.apache.fluss.utils.clock.Clock; +import org.apache.fluss.utils.concurrent.FutureUtils; import org.apache.fluss.utils.concurrent.Scheduler; import org.slf4j.Logger; @@ -698,14 +699,11 @@ public void appendHistoricalRecordsToLog( Collection entriesPerBucket, @Nullable UserContext userContext, Consumer> responseCallback) { - if (entriesPerBucket.isEmpty()) { - responseCallback.accept(Collections.emptyList()); - return; - } - - List results = Collections.synchronizedList(new ArrayList<>()); - AtomicInteger remaining = new AtomicInteger(entriesPerBucket.size()); + List> resultFutures = + new ArrayList<>(entriesPerBucket.size()); for (ProduceLogDataForBucket bucketData : entriesPerBucket) { + CompletableFuture resultFuture = new CompletableFuture<>(); + resultFutures.add(resultFuture); String originalPartitionName = checkNotNull( bucketData.originalPartitionName(), @@ -728,12 +726,11 @@ public void appendHistoricalRecordsToLog( result.getBaseOffset(), result.getWriteLogEndOffset(), originalPartitionName); - results.add(historicalResult); - if (remaining.decrementAndGet() == 0) { - responseCallback.accept(new ArrayList<>(results)); - } + resultFuture.complete(historicalResult); }); } + FutureUtils.combineAll(resultFutures) + .thenAccept(results -> responseCallback.accept(new ArrayList<>(results))); } /** diff --git a/fluss-server/src/test/java/org/apache/fluss/server/DynamicConfigChangeTest.java b/fluss-server/src/test/java/org/apache/fluss/server/DynamicConfigChangeTest.java index adc6a68c63a..7f50bfc1203 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/DynamicConfigChangeTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/DynamicConfigChangeTest.java @@ -192,34 +192,10 @@ void testAlterLakehouseConfigs() throws Exception { } @Test - void testAlterHistoricalKvCleanupIdleTime() throws Exception { - DynamicConfigManager dynamicConfigManager = createManager(new Configuration()); - AtomicReference cleanupIdleTime = new AtomicReference<>(); - dynamicConfigManager.register( - new ServerReconfigurable() { - @Override - public void validate(Configuration newConfig) throws ConfigException {} - - @Override - public void reconfigure(Configuration newConfig) { - cleanupIdleTime.set( - newConfig.get( - ConfigOptions - .SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME)); - } - }); - dynamicConfigManager.startup(); - - alterConfig( - dynamicConfigManager, - ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME.key(), - "5min"); - - assertThat(cleanupIdleTime.get()).isEqualTo(Duration.ofMinutes(5)); - assertThat(zookeeperClient.fetchEntityConfig()) - .containsEntry( - ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME.key(), - "5min"); + void testAllowsHistoricalKvCleanupIdleTimeToChangeDynamically() { + assertThat(new DynamicServerConfig(new Configuration()).isAllowedConfig( + ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME.key())) + .isTrue(); } @Test diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionTaskExecutorTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionTaskExecutorTest.java index 4293fc3d0df..98574e7d9fd 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionTaskExecutorTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionTaskExecutorTest.java @@ -148,37 +148,6 @@ void testOrderedTaskContinuesAfterPreviousFailure() throws Exception { assertThat(taskExecutor.numInflightRequests()).isZero(); } - @Test - void testMaintenanceTaskKeepsOrderWithoutConsumingRequestPermit() throws Exception { - ManualExecutor executor = new ManualExecutor(); - HistoricalPartitionTaskExecutor taskExecutor = - new HistoricalPartitionTaskExecutor(configuration(1), executor); - List executionOrder = new ArrayList<>(); - - CompletableFuture write = - taskExecutor.submitOrdered( - "bucket", - () -> { - executionOrder.add("write"); - return "written"; - }, - () -> "throttled"); - CompletableFuture maintenance = - taskExecutor.submitOrderedMaintenance( - "bucket", () -> executionOrder.add("maintenance")); - - assertThat(taskExecutor.numInflightRequests()).isOne(); - assertThat(taskExecutor.submitOrdered("another-bucket", () -> "written", () -> "throttled")) - .isCompletedWithValue("throttled"); - - executor.runNext(); - executor.runNext(); - assertThat(write).isCompletedWithValue("written"); - assertThat(maintenance).isDone(); - assertThat(executionOrder).containsExactly("write", "maintenance"); - assertThat(taskExecutor.numInflightRequests()).isZero(); - } - @Test void testRejectNonPositiveRequestLimit() { assertThatThrownBy( diff --git a/fluss-server/src/test/java/org/apache/fluss/server/utils/HistoricalPartitionTableValidationTest.java b/fluss-server/src/test/java/org/apache/fluss/server/utils/HistoricalPartitionTableValidationTest.java index 7fa4a90a1d6..9b6aa6913ea 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/utils/HistoricalPartitionTableValidationTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/utils/HistoricalPartitionTableValidationTest.java @@ -26,7 +26,6 @@ import org.junit.jupiter.api.Test; -import static org.assertj.core.api.Assertions.assertThatCode; import static org.assertj.core.api.Assertions.assertThatThrownBy; class HistoricalPartitionTableValidationTest { @@ -82,30 +81,4 @@ void testReportsAllUnmetHistoricalPartitionRequirements() { + "(currently 'iceberg'); " + "the table must define exactly one partition key (found 0)."); } - - @Test - void testAllowsHistoricalPartitionForLogTable() { - TableDescriptor logTableDescriptor = - TableDescriptor.builder() - .schema( - Schema.newBuilder() - .column("id", DataTypes.INT()) - .column("dt", DataTypes.STRING()) - .build()) - .partitionedBy("dt") - .distributedBy(1, "id") - .property(ConfigOptions.TABLE_REPLICATION_FACTOR, 1) - .property(ConfigOptions.TABLE_AUTO_PARTITION_ENABLED, true) - .property(ConfigOptions.TABLE_AUTO_PARTITION_KEY, "dt") - .property(ConfigOptions.TABLE_DATALAKE_ENABLED, true) - .property(ConfigOptions.TABLE_DATALAKE_FORMAT, DataLakeFormat.PAIMON) - .property(ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_ENABLED, true) - .build(); - - assertThatCode( - () -> - TableDescriptorValidation.validateTableDescriptor( - logTableDescriptor, 100, DataLakeFormat.PAIMON)) - .doesNotThrowAnyException(); - } } From 4b908f69fa14414760206e87fc5a364700fce53e Mon Sep 17 00:00:00 2001 From: luoyuxia Date: Thu, 27 Aug 2026 20:56:58 +0800 Subject: [PATCH 3/4] [server] Refine historical KV cleanup lifecycle Bind historical write state to the active KV overlay and defer idle cleanup until its deadline after lake progress catches up. Co-Authored-By: Codex AI-Model: gpt-5 AI-Contributed/Feature: 234/234 AI-Contributed/UT: 129/129 --- .../source/split/TieringSplitGenerator.java | 10 +- .../paimon/tiering/PaimonTieringTest.java | 7 +- .../apache/fluss/server/replica/Replica.java | 48 ++++- .../fluss/server/replica/ReplicaManager.java | 12 -- .../HistoricalPartitionManager.java | 164 ++++++++++-------- .../fluss/server/DynamicConfigChangeTest.java | 8 +- .../HistoricalPartitionManagerTest.java | 114 ++++++------ 7 files changed, 202 insertions(+), 161 deletions(-) diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/split/TieringSplitGenerator.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/split/TieringSplitGenerator.java index a6282cfeabd..5dd0bad2374 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/split/TieringSplitGenerator.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/split/TieringSplitGenerator.java @@ -155,7 +155,8 @@ private List generatePartitionTableSplit( try { latestKvSnapshots = flussAdmin - .getLatestKvSnapshots(tableInfo.getTablePath(), partitionName) + .getLatestKvSnapshots( + tableInfo.getTablePath(), partitionName) .get(); } catch (Exception e) { throw new FlinkRuntimeException( @@ -203,12 +204,7 @@ private List generateNonPartitionedTableSplit( } return generateTableSplit( - tableInfo, - null, - null, - lakeSnapshotInfo, - latestKvSnapshots, - latestBucketsOffset); + tableInfo, null, null, lakeSnapshotInfo, latestKvSnapshots, latestBucketsOffset); } private List generateTableSplit( diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringTest.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringTest.java index b960caacdf5..19c59600f92 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringTest.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringTest.java @@ -244,8 +244,7 @@ void testHistoricalPrimaryKeyTiering() throws Exception { SimpleVersionedSerializer serializer = paimonLakeTieringFactory.getWriteResultSerializer(); writeResult = - serializer.deserialize( - serializer.getVersion(), serializer.serialize(writeResult)); + serializer.deserialize(serializer.getVersion(), serializer.serialize(writeResult)); assertHistoricalPartitions(writeResult); try (LakeCommitter committer = @@ -824,9 +823,7 @@ private LogRecord historicalRecord( long offset, long timestamp, int key, String partition, ChangeType changeType) { GenericRow row = GenericRow.of( - key, - BinaryString.fromString("value"), - BinaryString.fromString(partition)); + key, BinaryString.fromString("value"), BinaryString.fromString(partition)); return new GenericRecord(offset, timestamp, changeType, row); } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java index c099d7a4acb..ef312728215 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java @@ -224,6 +224,9 @@ public final class Replica { private @Nullable PeriodicSnapshotManager kvSnapshotManager; // The lake log end offset used as the durable base of the current historical KV overlay. private volatile long historicalKvBaseOffset = -1L; + // Replaced together with the historical KV overlay so stale cleanup tasks can be fenced by + // identity without external replica lifecycle callbacks. + private volatile @Nullable HistoricalWriteState historicalWriteState; /** * Server-wide {@link ScannerManager}. Active sessions for this bucket are closed in {@link @@ -382,6 +385,11 @@ public long getLakeLogEndOffset() { return logTablet.getLakeLogEndOffset(); } + /** Returns the state owned by the active historical KV overlay, or null if none is active. */ + public @Nullable HistoricalWriteState getHistoricalWriteState() { + return historicalWriteState; + } + /** * Drops and recreates a fully tiered historical KV overlay if leadership and offsets still * match. @@ -824,9 +832,9 @@ private void createKv() { tableBucket, INIT_KV_TABLET_MAX_RETRY_TIMES), lastError); } - // A historical KV tablet is a disposable overlay over the lake snapshot. It is recovered - // by replaying WAL from the lake log end offset and does not create its own KV snapshots. - if (!isHistoricalPartition()) { + if (isHistoricalPartition()) { + historicalWriteState = new HistoricalWriteState(clock.milliseconds()); + } else { startPeriodicKvSnapshot(snapshotUsed.orElse(null)); } } @@ -846,6 +854,7 @@ private void dropKv() { kvManager.dropKv(tableBucket); kvTablet = null; } + historicalWriteState = null; historicalKvBaseOffset = -1L; } @@ -2627,4 +2636,37 @@ public PeriodicSnapshotManager getKvSnapshotManager() { return kvSnapshotManager; } + /** Write activity and maximum-size state owned by one historical KV overlay. */ + @ThreadSafe + public static final class HistoricalWriteState { + // Latched when the live SST size reaches the maximum. Replacing the overlay replaces this + // state, so transient RocksDB size changes cannot resume writes prematurely. + private final AtomicBoolean maxSizeReached = new AtomicBoolean(); + + private volatile long lastWriteMs; + + private HistoricalWriteState(long lastWriteMs) { + this.lastWriteMs = lastWriteMs; + } + + /** Returns whether historical writes are paused by the maximum-size limit. */ + public boolean maxSizeReached() { + return maxSizeReached.get(); + } + + /** Latches the maximum-size limit and returns whether this call changed the state. */ + public boolean markMaxSizeReached() { + return maxSizeReached.compareAndSet(false, true); + } + + /** Records the latest historical write activity time. */ + public void recordWrite(long timestampMs) { + lastWriteMs = timestampMs; + } + + /** Returns the latest historical write activity time. */ + public long lastWriteMs() { + return lastWriteMs; + } + } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java index 663ff0b336e..bea89b18ffb 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java @@ -1460,13 +1460,7 @@ private void makeLeaders( if (replica.isDataLakeEnabled()) { updateWithLakeTableSnapshot(replica); } - int previousLeaderEpoch = replica.getLeaderEpoch(); replica.makeLeader(data); - if (replica.isHistoricalPartition() - && replica.isKvTable() - && previousLeaderEpoch != replica.getLeaderEpoch()) { - historicalPartitionManager.onLeaderActivated(replica); - } // start the remote log tiering tasks for leaders remoteLogManager.startLogTiering(replica); @@ -1540,9 +1534,6 @@ private void makeFollowers( replicasBecomeFollower.add(replica); scannerManager.closeScannersForBucket(tb); } - if (replica.isHistoricalPartition()) { - historicalPartitionManager.onReplicaStopped(tb); - } // stop the remote log tiering tasks for followers remoteLogManager.stopLogTiering(replica); result.put(tb, new NotifyLeaderAndIsrResultForBucket(tb)); @@ -2345,9 +2336,6 @@ private StopReplicaResultForBucket stopReplica( HostedReplica replica = getReplica(tb); if (replica instanceof OnlineReplica) { Replica replicaToDelete = ((OnlineReplica) replica).getReplica(); - if (replicaToDelete.isHistoricalPartition()) { - historicalPartitionManager.onReplicaStopped(tb); - } if (deleteLocal) { if (allReplicas.remove(tb) != null) { serverMetricGroup.removeTableBucketMetricGroup( diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java index 62a66d61137..7bd230e3010 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java @@ -42,11 +42,11 @@ import org.apache.fluss.server.kv.historical.HistoricalValueLookup; import org.apache.fluss.server.log.LogAppendInfo; import org.apache.fluss.server.replica.Replica; +import org.apache.fluss.server.replica.Replica.HistoricalWriteState; import org.apache.fluss.server.storage.LocalDiskManager; import org.apache.fluss.utils.ByteArraySlice; import org.apache.fluss.utils.ByteArrayWrapper; import org.apache.fluss.utils.clock.Clock; -import org.apache.fluss.utils.clock.SystemClock; import org.apache.fluss.utils.concurrent.Scheduler; import org.slf4j.Logger; @@ -63,9 +63,6 @@ import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicBoolean; import static org.apache.fluss.utils.Preconditions.checkArgument; import static org.apache.fluss.utils.Preconditions.checkNotNull; @@ -80,12 +77,11 @@ public final class HistoricalPartitionManager implements AutoCloseable { private final HistoricalPartitionTaskExecutor taskExecutor; private final HistoricalLakeLookupManager lakeLookupManager; private final Clock clock; - private volatile long cleanupIdleTimeMs; + private final @Nullable Scheduler cleanupScheduler; private final long maxHistoricalKvSizeBytes; - // Per-physical-bucket state for coordinating historical write admission and overlay cleanup. - // The state is replaced when a new leader epoch is activated and removed when the local - // replica stops. - private final ConcurrentMap historicalWriteStates; + + private volatile long cleanupIdleTimeMs; + private volatile boolean closed; /** Creates a historical partition manager from the tablet server dependencies. */ public HistoricalPartitionManager( @@ -107,19 +103,8 @@ public HistoricalPartitionManager( dataDirVolumeBytes, scheduler), clock, - MAX_HISTORICAL_KV_SIZE_BYTES); - } - - @VisibleForTesting - HistoricalPartitionManager( - HistoricalPartitionTaskExecutor taskExecutor, - HistoricalLakeLookupManager lakeLookupManager) { - this( - new Configuration(), - taskExecutor, - lakeLookupManager, - SystemClock.getInstance(), - MAX_HISTORICAL_KV_SIZE_BYTES); + MAX_HISTORICAL_KV_SIZE_BYTES, + scheduler); } @VisibleForTesting @@ -128,7 +113,8 @@ public HistoricalPartitionManager( HistoricalPartitionTaskExecutor taskExecutor, HistoricalLakeLookupManager lakeLookupManager, Clock clock, - long maxHistoricalKvSizeBytes) { + long maxHistoricalKvSizeBytes, + @Nullable Scheduler cleanupScheduler) { Duration cleanupIdleTime = conf.get(ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME); checkArgument( @@ -141,9 +127,9 @@ public HistoricalPartitionManager( this.lakeLookupManager = checkNotNull(lakeLookupManager, "lakeLookupManager must not be null"); this.clock = checkNotNull(clock, "clock must not be null"); + this.cleanupScheduler = cleanupScheduler; this.cleanupIdleTimeMs = cleanupIdleTime.toMillis(); this.maxHistoricalKvSizeBytes = maxHistoricalKvSizeBytes; - this.historicalWriteStates = new ConcurrentHashMap<>(); } /** Starts the resources used by historical partition operations. */ @@ -151,17 +137,6 @@ public void startup(Scheduler scheduler) { lakeLookupManager.startup(scheduler); } - /** Starts tracking cleanup activity for a newly activated historical KV leader. */ - public void onLeaderActivated(Replica replica) { - historicalWriteStates.put( - replica.getTableBucket(), new HistoricalWriteState(clock.milliseconds())); - } - - /** Stops tracking cleanup activity for a replica that is no longer a local leader. */ - public void onReplicaStopped(TableBucket tableBucket) { - historicalWriteStates.remove(tableBucket); - } - /** Records new lake progress and schedules any cleanup that it makes eligible. */ public void onLakeProgress(Replica replica, long lakeSnapshotId, long lakeLogEndOffset) { if (!replica.isLeader() || !replica.isKvTable()) { @@ -172,11 +147,13 @@ public void onLakeProgress(Replica replica, long lakeSnapshotId, long lakeLogEnd if (lakeLogEndOffset != localLogEndOffset) { return; } - HistoricalWriteState state = historicalWriteStateFor(replica); - boolean maxSizeReached = state.maxSizeReached.get(); - // Lake progress is the cleanup trigger. The ordered cleanup task rechecks the latest write - // time when it actually runs, so a write accepted after this notification cancels an idle - // cleanup without being overtaken by it. + HistoricalWriteState state = replica.getHistoricalWriteState(); + if (state == null) { + return; + } + boolean maxSizeReached = state.maxSizeReached(); + // Lake progress establishes the cleanup candidate. Idle cleanup waits until the last write + // reaches its deadline, then enters the ordered queue behind already accepted writes. scheduleCleanup( replica, state, @@ -230,8 +207,11 @@ public CompletableFuture putKv( checkNotNull( putData.originalPartitionName(), "originalPartitionName must not be null"); - HistoricalWriteState state = historicalWriteStateFor(replica); - if (state.maxSizeReached.get()) { + HistoricalWriteState state = + checkNotNull( + replica.getHistoricalWriteState(), + "No active historical KV overlay for " + replica.getTableBucket()); + if (state.maxSizeReached()) { return CompletableFuture.completedFuture( maxSizeThrottledResult( putData, originalPartitionName, maxHistoricalKvSizeBytes)); @@ -243,7 +223,7 @@ public CompletableFuture putKv( maxSizeThrottledResult( putData, originalPartitionName, maxHistoricalKvSizeBytes)); } - state.lastHistoricalWriteMs = clock.milliseconds(); + state.recordWrite(clock.milliseconds()); return taskExecutor.submitOrdered( putData.tableBucket(), () -> { @@ -255,7 +235,6 @@ public CompletableFuture putKv( targetColumns, mergeMode, requiredAcks); - state.lastHistoricalWriteMs = clock.milliseconds(); return PutKvResultForBucket.historicalSuccess( putData.tableBucket(), appendInfo.lastOffset() + 1, @@ -405,7 +384,7 @@ LogAppendInfo processPut( } private void markMaxSizeReached(Replica replica, HistoricalWriteState state) { - if (state.maxSizeReached.compareAndSet(false, true)) { + if (state.markMaxSizeReached()) { LOG.warn( "Pausing historical writes for {} because its live SST size reached the " + "maximum size {} bytes.", @@ -414,12 +393,6 @@ private void markMaxSizeReached(Replica replica, HistoricalWriteState state) { } } - private HistoricalWriteState historicalWriteStateFor(Replica replica) { - return historicalWriteStates.computeIfAbsent( - replica.getTableBucket(), - ignored -> new HistoricalWriteState(clock.milliseconds())); - } - private void scheduleCleanup( Replica replica, HistoricalWriteState state, @@ -427,6 +400,19 @@ private void scheduleCleanup( long lakeSnapshotId, int expectedLeaderEpoch, long logEndOffset) { + if (closed + || replica.getHistoricalWriteState() != state + || expectedLeaderEpoch != replica.getLeaderEpoch() + || replica.getLocalLogEndOffset() != logEndOffset) { + return; + } + + if (!maxSizeReached + && deferIdleCleanupIfNeeded( + replica, state, lakeSnapshotId, expectedLeaderEpoch, logEndOffset)) { + return; + } + CompletableFuture cleanupFuture; cleanupFuture = taskExecutor.submitOrderedMaintenance( @@ -457,13 +443,13 @@ private void runCleanup( long lakeSnapshotId, int expectedLeaderEpoch, long logEndOffset) { - long now = clock.milliseconds(); - if (historicalWriteStates.get(replica.getTableBucket()) != state - || expectedLeaderEpoch != replica.getLeaderEpoch() - || (!maxSizeReached - && (cleanupIdleTimeMs <= 0L - || now < state.lastHistoricalWriteMs - || now - state.lastHistoricalWriteMs < cleanupIdleTimeMs))) { + if (replica.getHistoricalWriteState() != state + || expectedLeaderEpoch != replica.getLeaderEpoch()) { + return; + } + if (!maxSizeReached + && deferIdleCleanupIfNeeded( + replica, state, lakeSnapshotId, expectedLeaderEpoch, logEndOffset)) { return; } @@ -471,7 +457,6 @@ private void runCleanup( expectedLeaderEpoch, logEndOffset, () -> requireLakeSnapshot(replica.getTableBucket().getTableId(), lakeSnapshotId))) { - state.maxSizeReached.set(false); LOG.info( "Cleaned {} historical KV overlay for {}.", maxSizeReached ? "max-size-triggered" : "idle-triggered", @@ -479,6 +464,50 @@ private void runCleanup( } } + /** + * Returns whether idle cleanup must stop now. If the idle window has not elapsed, schedules the + * next check at its deadline. + */ + private boolean deferIdleCleanupIfNeeded( + Replica replica, + HistoricalWriteState state, + long lakeSnapshotId, + int expectedLeaderEpoch, + long logEndOffset) { + long idleTimeMs = cleanupIdleTimeMs; + if (idleTimeMs <= 0L) { + return true; + } + long delayMs = remainingIdleCleanupDelayMs(state, idleTimeMs); + if (delayMs <= 0L) { + return false; + } + checkNotNull(cleanupScheduler, "cleanupScheduler must not be null") + .scheduleOnce( + "historical-kv-idle-cleanup-" + replica.getTableBucket(), + () -> + scheduleCleanup( + replica, + state, + false, + lakeSnapshotId, + expectedLeaderEpoch, + logEndOffset), + delayMs); + return true; + } + + /** Returns the remaining delay before idle cleanup is eligible, or {@code 0} if it is due. */ + private long remainingIdleCleanupDelayMs(HistoricalWriteState state, long idleTimeMs) { + long now = clock.milliseconds(); + long lastWriteMs = state.lastWriteMs(); + if (now < lastWriteMs) { + // A backward clock jump restarts the idle window instead of cleaning prematurely. + return idleTimeMs; + } + return Math.max(0L, idleTimeMs - (now - lastWriteMs)); + } + private static PutKvResultForBucket requestLimitThrottledResult( PutKvDataForBucket putData, String originalPartitionName) { return PutKvResultForBucket.historicalFailure( @@ -514,7 +543,7 @@ private static PutKvResultForBucket maxSizeThrottledResult( @Override public void close() { - historicalWriteStates.clear(); + closed = true; taskExecutor.close(); lakeLookupManager.close(); } @@ -579,19 +608,4 @@ private LookupResultForBucket lookupInternal( tableBucket, originalPartitionName, ApiError.fromThrowable(e)); } } - - /** Per-bucket historical write activity and maximum-size state. */ - private static final class HistoricalWriteState { - // Latched when the live SST size reaches the maximum. It is cleared only after a cleanup - // covered by lake progress succeeds, so transient RocksDB size changes cannot resume - // writes prematurely. - private final AtomicBoolean maxSizeReached = new AtomicBoolean(); - - // Updated when a write is admitted and again when it completes successfully. - private volatile long lastHistoricalWriteMs; - - private HistoricalWriteState(long lastHistoricalWriteMs) { - this.lastHistoricalWriteMs = lastHistoricalWriteMs; - } - } } diff --git a/fluss-server/src/test/java/org/apache/fluss/server/DynamicConfigChangeTest.java b/fluss-server/src/test/java/org/apache/fluss/server/DynamicConfigChangeTest.java index 7f50bfc1203..6beccb61285 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/DynamicConfigChangeTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/DynamicConfigChangeTest.java @@ -193,8 +193,12 @@ void testAlterLakehouseConfigs() throws Exception { @Test void testAllowsHistoricalKvCleanupIdleTimeToChangeDynamically() { - assertThat(new DynamicServerConfig(new Configuration()).isAllowedConfig( - ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME.key())) + assertThat( + new DynamicServerConfig(new Configuration()) + .isAllowedConfig( + ConfigOptions + .SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME + .key())) .isTrue(); } diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java index eea8578ff2a..d5ca84c3173 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java @@ -97,6 +97,7 @@ import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -139,7 +140,7 @@ void testResolvesMultipleLakeMissesWithoutPrewriteRollback() throws Exception { TestingHistoricalLakeLookupManager lakeLookupManager = new TestingHistoricalLakeLookupManager(lookupConfiguration()); HistoricalPartitionManager historicalPartitionManager = - new HistoricalPartitionManager( + createNonCleanupManager( new HistoricalPartitionTaskExecutor(lookupConfiguration()), lakeLookupManager); @@ -207,7 +208,7 @@ void testWalFullRowUpsertDoesNotLookupLake() throws Exception { TestingHistoricalLakeLookupManager lakeLookupManager = new TestingHistoricalLakeLookupManager(lookupConfiguration()); HistoricalPartitionManager historicalPartitionManager = - new HistoricalPartitionManager( + createNonCleanupManager( new HistoricalPartitionTaskExecutor(lookupConfiguration()), lakeLookupManager); @@ -256,7 +257,7 @@ void testHistoricalInsertUpdateAndDelete() throws Exception { TestingHistoricalLakeLookupManager lakeLookupManager = new TestingHistoricalLakeLookupManager(lookupConfiguration()); HistoricalPartitionManager historicalPartitionManager = - new HistoricalPartitionManager( + createNonCleanupManager( new HistoricalPartitionTaskExecutor(lookupConfiguration()), lakeLookupManager); @@ -446,7 +447,7 @@ void testUpdateAndDeleteFromLakeFallback() throws Exception { TestingHistoricalLakeLookupManager lakeLookupManager = new TestingHistoricalLakeLookupManager(lookupConfiguration()); HistoricalPartitionManager historicalPartitionManager = - new HistoricalPartitionManager( + createNonCleanupManager( new HistoricalPartitionTaskExecutor(lookupConfiguration()), lakeLookupManager); @@ -547,7 +548,7 @@ void testLeaderChangeDuringLakeLookupFencesHistoricalWrite() throws Exception { TestingHistoricalLakeLookupManager lakeLookupManager = new TestingHistoricalLakeLookupManager(lookupConfiguration()); HistoricalPartitionManager historicalPartitionManager = - new HistoricalPartitionManager( + createNonCleanupManager( new HistoricalPartitionTaskExecutor(lookupConfiguration()), lakeLookupManager); CountDownLatch lakeLookupStarted = new CountDownLatch(1); @@ -604,7 +605,7 @@ void testRecoversHistoricalOverlayFromLakeCommitOffset() throws Exception { TestingHistoricalLakeLookupManager lakeLookupManager = new TestingHistoricalLakeLookupManager(lookupConfiguration()); HistoricalPartitionManager historicalPartitionManager = - new HistoricalPartitionManager( + createNonCleanupManager( new HistoricalPartitionTaskExecutor(lookupConfiguration()), lakeLookupManager); @@ -750,8 +751,7 @@ void testCleansFullyTieredHistoricalOverlayAfterWriteIdleTime() throws Exception TestingHistoricalLakeLookupManager lakeLookupManager = new TestingHistoricalLakeLookupManager(cleanupConf); HistoricalPartitionManager historicalPartitionManager = - createCleanupManager( - cleanupConf, executor, lakeLookupManager, replica, Long.MAX_VALUE); + createCleanupManager(cleanupConf, executor, lakeLookupManager, Long.MAX_VALUE); byte[] primaryKey = new CompactedKeyEncoder(HISTORICAL_KEY_TYPE).encodeKey(row(1, "us")); Object[] valueObjects = new Object[] {1, "us", ORIGINAL_PARTITION, "v1"}; @@ -774,60 +774,26 @@ void testCleansFullyTieredHistoricalOverlayAfterWriteIdleTime() throws Exception assertThat(putFuture.get(10, TimeUnit.SECONDS).failed()).isFalse(); flushAndWait(originalKvTablet, Long.MAX_VALUE); - manualClock.advanceTime(Duration.ofMinutes(1)); - // The idle policy cannot clean until lake progress covers the local WAL. - assertThat(executor.numQueuedRunnables()).isZero(); - assertThat(replica.getKvTablet()).isSameAs(originalKvTablet); - long tieredOffset = replica.getLocalLogEndOffset(); historicalPartitionManager.onLakeProgress(replica, 9L, tieredOffset - 1); assertThat(executor.numQueuedRunnables()).isZero(); replica.getLogTablet().updateLakeLogEndOffset(tieredOffset); historicalPartitionManager.onLakeProgress(replica, 10L, tieredOffset); - assertThat(executor.numQueuedRunnables()).isOne(); - assertThat(lakeLookupManager.requiredLakeSnapshotIds).isEmpty(); - - // A write admitted after idle cleanup was scheduled must either cancel that cleanup - // or run against the newly created overlay. It must never be lost during the reset. - CompletableFuture laterPut = - putHistoricalRecords( - historicalPartitionManager, - replica, - batch( - HISTORICAL_KEY_TYPE, - tableInfo.getRowType(), - Tuple2.of(new Object[] {1, "us"}, valueObjects))); - executor.triggerAll(); - assertThat(laterPut.get(10, TimeUnit.SECONDS).failed()).isFalse(); + // Lake normally catches up before the overlay becomes idle. Cleanup must wake at the + // write deadline even if no further lake progress notification arrives. + assertThat(executor.numQueuedRunnables()).isZero(); + assertThat(executor.getActiveNonPeriodicScheduledTask()).hasSize(1); assertThat(replica.getKvTablet()).isSameAs(originalKvTablet); - assertThat(lakeLookupManager.requiredLakeSnapshotIds).isEmpty(); - - flushAndWait(originalKvTablet, Long.MAX_VALUE); manualClock.advanceTime(Duration.ofMinutes(1)); - long latestTieredOffset = replica.getLocalLogEndOffset(); - replica.getLogTablet().updateLakeLogEndOffset(latestTieredOffset); - - Configuration longerIdleTimeConf = new Configuration(cleanupConf); - longerIdleTimeConf.set( - ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME, - Duration.ofMinutes(2)); - historicalPartitionManager.validate(longerIdleTimeConf); - historicalPartitionManager.reconfigure(longerIdleTimeConf); - historicalPartitionManager.onLakeProgress(replica, 11L, latestTieredOffset); + executor.triggerNonPeriodicScheduledTasks(); + assertThat(executor.numQueuedRunnables()).isOne(); assertThat(lakeLookupManager.requiredLakeSnapshotIds).isEmpty(); executor.triggerAll(); - assertThat(replica.getKvTablet()).isSameAs(originalKvTablet); - - Configuration shorterIdleTimeConf = new Configuration(cleanupConf); - historicalPartitionManager.validate(shorterIdleTimeConf); - historicalPartitionManager.reconfigure(shorterIdleTimeConf); - historicalPartitionManager.onLakeProgress(replica, 12L, latestTieredOffset); - executor.triggerAll(); KvTablet cleanedKvTablet = replica.getKvTablet(); assertThat(cleanedKvTablet).isNotNull().isNotSameAs(originalKvTablet); - assertThat(lakeLookupManager.requiredLakeSnapshotIds).containsExactly(12L); + assertThat(lakeLookupManager.requiredLakeSnapshotIds).containsExactly(10L); assertThat(cleanedKvTablet.getRocksDBKv().limitScan(10)).isEmpty(); assertThat(cleanedKvTablet.lookupHistoricalLocal(ORIGINAL_PARTITION, primaryKey)) .isEqualTo(KvStateLookupResult.notFound()); @@ -859,7 +825,8 @@ void testRejectsNegativeCleanupIdleTimeDuringReconfiguration() { new HistoricalPartitionTaskExecutor(cleanupConf, executor), new TestingHistoricalLakeLookupManager(cleanupConf), manualClock, - Long.MAX_VALUE); + Long.MAX_VALUE, + new TestingCleanupScheduler(executor)); Configuration invalidConf = new Configuration(cleanupConf); invalidConf.set( ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME, @@ -892,8 +859,7 @@ void testCleanupRequiresLakeAndLocalOffsetsToMatch() throws Exception { TestingHistoricalLakeLookupManager lakeLookupManager = new TestingHistoricalLakeLookupManager(cleanupConf); HistoricalPartitionManager historicalPartitionManager = - createCleanupManager( - cleanupConf, executor, lakeLookupManager, replica, Long.MAX_VALUE); + createCleanupManager(cleanupConf, executor, lakeLookupManager, Long.MAX_VALUE); KvRecordBatch records = batch( @@ -964,7 +930,6 @@ void testMaxSizeBlocksWritesUntilOverlayIsTieredAndCleaned() throws Exception { cleanupConf, executor, new TestingHistoricalLakeLookupManager(cleanupConf), - replica, 1L); KvRecordBatch firstBatch = @@ -1026,7 +991,7 @@ void testHistoricalLookupThrottledWhenPermitsExhausted() throws Exception { ManuallyTriggeredScheduledExecutorService executor = new ManuallyTriggeredScheduledExecutorService(); HistoricalPartitionManager historicalPartitionManager = - new HistoricalPartitionManager( + createNonCleanupManager( new HistoricalPartitionTaskExecutor(lookupConfiguration(), executor), new TestingHistoricalLakeLookupManager(lookupConfiguration())); @@ -1231,7 +1196,6 @@ private HistoricalPartitionManager createCleanupManager( Configuration configuration, ManuallyTriggeredScheduledExecutorService executor, TestingHistoricalLakeLookupManager lakeLookupManager, - Replica replica, long maxHistoricalKvSizeBytes) { HistoricalPartitionManager manager = new HistoricalPartitionManager( @@ -1239,11 +1203,47 @@ private HistoricalPartitionManager createCleanupManager( new HistoricalPartitionTaskExecutor(configuration, executor), lakeLookupManager, manualClock, - maxHistoricalKvSizeBytes); - manager.onLeaderActivated(replica); + maxHistoricalKvSizeBytes, + new TestingCleanupScheduler(executor)); return manager; } + private HistoricalPartitionManager createNonCleanupManager( + HistoricalPartitionTaskExecutor taskExecutor, + HistoricalLakeLookupManager lakeLookupManager) { + return new HistoricalPartitionManager( + new Configuration(), + taskExecutor, + lakeLookupManager, + manualClock, + Long.MAX_VALUE, + null); + } + + private static final class TestingCleanupScheduler + implements org.apache.fluss.utils.concurrent.Scheduler { + private final ManuallyTriggeredScheduledExecutorService executor; + + private TestingCleanupScheduler(ManuallyTriggeredScheduledExecutorService executor) { + this.executor = executor; + } + + @Override + public void startup() {} + + @Override + public void shutdown() {} + + @Override + public ScheduledFuture schedule( + String name, Runnable task, long delayMs, long periodMs) { + if (periodMs > 0L) { + return executor.scheduleAtFixedRate(task, delayMs, periodMs, TimeUnit.MILLISECONDS); + } + return executor.schedule(task, delayMs, TimeUnit.MILLISECONDS); + } + } + private static CompletableFuture putHistoricalRecords( HistoricalPartitionManager manager, Replica replica, KvRecordBatch records) { return manager.putKv( From 7f8c03fef098dbfae5f938a1cc76fedb7d97d0aa Mon Sep 17 00:00:00 2001 From: luoyuxia Date: Fri, 28 Aug 2026 09:43:15 +0800 Subject: [PATCH 4/4] [server] Refine historical KV cleanup state handling Return a retriable KV storage error while local historical KV state is being initialized or rebuilt. Clarify cleanup-state naming and terminology, and extend the Paimon integration test through post-recovery writes and restarted tiering. Co-Authored-By: Codex AI-Model: gpt-5 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 59/59 --- .../fluss/client/write/RecordAccumulator.java | 11 +- .../apache/fluss/config/ConfigOptions.java | 4 +- .../lookup/HistoricalPartitionITCase.java | 111 +++++++++++++--- ...pendOnlyArrowBatchCaseSensitivityTest.java | 6 +- .../apache/fluss/server/replica/Replica.java | 106 ++++++++++----- .../fluss/server/replica/ReplicaManager.java | 6 +- .../HistoricalPartitionManager.java | 117 ++++++++++------- .../HistoricalPartitionManagerTest.java | 122 ++++++++++++------ 8 files changed, 334 insertions(+), 149 deletions(-) diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java b/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java index 84ee7331e19..19362d8a948 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java @@ -364,6 +364,9 @@ void routeWritesTo( return; } + // Pair with appendNewBatch(). For a historical route change, either a normal batch is + // registered as incomplete first and rejects the switch, or batch creation observes the + // historical target. synchronized (existing) { if (existing.targetPath.equals(targetPath)) { existing.partitionId = targetPartitionId; @@ -703,7 +706,13 @@ private RecordAppendResult appendNewBatch( writeBatches.get(physicalTablePath), "Write batches for %s must exist.", physicalTablePath); - synchronized (bucketAndWriteBatches) { + // Only historical-enabled tables need to coordinate with routeWritesTo(). Other tables + // reuse the deque monitor already held by the caller, avoiding cross-bucket serialization. + Object routeLock = + tableInfo.getTableConfig().isHistoricalPartitionEnabled() + ? bucketAndWriteBatches + : deque; + synchronized (routeLock) { RecordAppendResult appendResult = tryAppend(writeRecord, callback, deque); if (appendResult != null) { // Somebody else found us a batch, return the one we waited for! Hopefully this diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java index fe2d538cfaa..c6502568a9c 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java @@ -442,9 +442,9 @@ public class ConfigOptions { public static final ConfigOption SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME = key("server.historical-partition.kv-cleanup.idle-time") .durationType() - .defaultValue(Duration.ofMinutes(30)) + .defaultValue(Duration.ofHours(3)) .withDescription( - "The idle time after which fully tiered historical KV write state in the local overlay can be cleaned. " + "The idle time after all local historical KV writes are tiered before the local state can be cleaned. " + "Set to 0 to disable idle cleanup."); public static final ConfigOption SERVER_DATA_DISK_WRITE_LIMIT_RATIO = diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/HistoricalPartitionITCase.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/HistoricalPartitionITCase.java index 0da14e9f710..20b05d55f28 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/HistoricalPartitionITCase.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/HistoricalPartitionITCase.java @@ -86,33 +86,79 @@ void testWriteAndTierHistoricalKvToPaimon() throws Exception { long tableId = createTable( tablePath, - partitionedPkDescriptor(schema, true, EXPIRED_PARTITION_RETENTION)); + partitionedDescriptor(schema, true, EXPIRED_PARTITION_RETENTION)); try { long historicalPartitionId = waitUntilHistoricalPartitionReady(tablePath, tableId); - InternalRow expectedRow = dataRow(true, 1, "unused", "Alice"); + InternalRow tieredRow = dataRow(true, 1, "unused", "Alice"); assertThat(admin.listPartitionInfos(tablePath).get()) .noneMatch(p -> EXPIRED_PARTITION_NAME.equals(p.getPartitionName())); - writeRows(tablePath, Collections.singletonList(expectedRow), false); + writeRows(tablePath, Collections.singletonList(tieredRow), false); // Historical writes must not recreate the expired original partition. assertThat(admin.listPartitionInfos(tablePath).get()) .noneMatch(p -> EXPIRED_PARTITION_NAME.equals(p.getPartitionName())); TableBucket historicalBucket = new TableBucket(tableId, historicalPartitionId, 0); assertThat(getLeaderReplica(historicalBucket).getLocalLogEndOffset()).isEqualTo(1); - JobClient jobClient = buildTieringJob(execEnv); - try { - assertReplicaStatus(historicalBucket, 1); - checkFlussOffsetsInSnapshot( - tablePath, Collections.singletonMap(historicalBucket, 1L)); - assertThat(readPaimonRows(tablePath)) - .containsExactly("1|" + EXPIRED_PARTITION_NAME + "|Alice"); - } finally { - jobClient.cancel().get(); - } + tierAndVerifyPaimonRows( + tablePath, historicalBucket, 1L, "1|" + EXPIRED_PARTITION_NAME + "|Alice"); + + // Leave an untiered update after the Paimon snapshot. Restart recovery must apply this + // changelog over the tiered row. + InternalRow updatedRow = dataRow(true, 1, "unused", "Alice-updated"); + writeRows(tablePath, Collections.singletonList(updatedRow), false); + // FULL changelog emits UPDATE_BEFORE and UPDATE_AFTER for the update. + assertThat(getLeaderReplica(historicalBucket).getLocalLogEndOffset()).isEqualTo(3); + assertThat(getLeaderReplica(historicalBucket).getLakeLogEndOffset()).isEqualTo(1); + + restartLeaderAndVerifyLookup(tablePath, historicalBucket, schema, updatedRow); + + // Verify that the recovered local state can resolve the previous value for another + // update, and that a newly started tiering job can synchronize the resulting changelog. + InternalRow postRecoveryRow = dataRow(true, 1, "unused", "Alice-after-recovery"); + writeRows(tablePath, Collections.singletonList(postRecoveryRow), false); + assertThat(getLeaderReplica(historicalBucket).getLocalLogEndOffset()).isEqualTo(5); + assertThat(getLeaderReplica(historicalBucket).getLakeLogEndOffset()).isEqualTo(1); + tierAndVerifyPaimonRows( + tablePath, + historicalBucket, + 5L, + "1|" + EXPIRED_PARTITION_NAME + "|Alice-after-recovery"); + } finally { + dropTable(tablePath); + } + } + + @Test + void testWriteAndTierHistoricalLogToPaimon() throws Exception { + TablePath tablePath = TablePath.of(DEFAULT_DB, "historical_log_write_tiering"); + long tableId = + createTable( + tablePath, + partitionedDescriptor( + partitionedLogSchema(), true, EXPIRED_PARTITION_RETENTION)); - restartLeaderAndVerifyLookup(tablePath, historicalBucket, schema, expectedRow); + try { + long historicalPartitionId = waitUntilHistoricalPartitionReady(tablePath, tableId); + List rows = + Arrays.asList( + row(1, EXPIRED_PARTITION_NAME, "Alice"), + row(2, EXPIRED_PARTITION_NAME, "Bob")); + + writeRows(tablePath, rows, true); + // Historical writes must not recreate the expired original partition. + assertThat(admin.listPartitionInfos(tablePath).get()) + .noneMatch(p -> EXPIRED_PARTITION_NAME.equals(p.getPartitionName())); + + TableBucket historicalBucket = new TableBucket(tableId, historicalPartitionId, 0); + assertThat(getLeaderReplica(historicalBucket).getLocalLogEndOffset()).isEqualTo(2); + tierAndVerifyPaimonRows( + tablePath, + historicalBucket, + 2L, + "1|" + EXPIRED_PARTITION_NAME + "|Alice", + "2|" + EXPIRED_PARTITION_NAME + "|Bob"); } finally { dropTable(tablePath); } @@ -128,7 +174,7 @@ void testLookupExpiredPartitionFromPaimon(boolean defaultBucketKey) throws Excep ? "historical_lookup_default_bucket" : "historical_lookup_bucket_subset"); Schema oldSchema = partitionedPkSchema(defaultBucketKey); - long tableId = createTable(tablePath, partitionedPkDescriptor(oldSchema, false)); + long tableId = createTable(tablePath, partitionedDescriptor(oldSchema, false)); // Enable historical lookup through ALTER TABLE to cover dynamic creation of the // coordinator-owned historical system partition. @@ -298,6 +344,23 @@ private List readPaimonRows(TablePath tablePath) throws Exception { return actualRows; } + private void tierAndVerifyPaimonRows( + TablePath tablePath, + TableBucket historicalBucket, + long expectedLogEndOffset, + String... expectedRows) + throws Exception { + JobClient jobClient = buildTieringJob(execEnv); + try { + assertReplicaStatus(historicalBucket, expectedLogEndOffset); + checkFlussOffsetsInSnapshot( + tablePath, Collections.singletonMap(historicalBucket, expectedLogEndOffset)); + assertThat(readPaimonRows(tablePath)).containsExactlyInAnyOrder(expectedRows); + } finally { + jobClient.cancel().get(); + } + } + private void restartLeaderAndVerifyLookup( TablePath tablePath, TableBucket historicalBucket, @@ -345,6 +408,14 @@ private static Schema partitionedPkSchema(boolean defaultBucketKey) { .build(); } + private static Schema partitionedLogSchema() { + return Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("dt", DataTypes.STRING()) + .column("name", DataTypes.STRING()) + .build(); + } + private static Schema evolvedPartitionedPkSchema(boolean defaultBucketKey) { if (defaultBucketKey) { return Schema.newBuilder() @@ -365,19 +436,19 @@ private static Schema evolvedPartitionedPkSchema(boolean defaultBucketKey) { .build(); } - private static TableDescriptor partitionedPkDescriptor( + private static TableDescriptor partitionedDescriptor( Schema schema, boolean historicalPartitionEnabled) { - return partitionedPkDescriptor( + return partitionedDescriptor( schema, historicalPartitionEnabled, INITIAL_PARTITION_RETENTION); } - private static TableDescriptor partitionedPkDescriptor( + private static TableDescriptor partitionedDescriptor( Schema schema, boolean historicalPartitionEnabled, int partitionRetention) { TableDescriptor.Builder builder = TableDescriptor.builder() .schema(schema) - // This is the default bucket key for (id, dt), and a strict subset of the - // physical primary key for (id, sub_id, dt). + // For primary-key tables, id is the default bucket key for (id, dt) and a + // strict subset of the physical primary key for (id, sub_id, dt). .distributedBy(1, "id") .partitionedBy("dt") .property(ConfigOptions.TABLE_AUTO_PARTITION_ENABLED, true) diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyArrowBatchCaseSensitivityTest.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyArrowBatchCaseSensitivityTest.java index e3d961d6605..31b664e1e55 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyArrowBatchCaseSensitivityTest.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyArrowBatchCaseSensitivityTest.java @@ -28,6 +28,7 @@ import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.paimon.FileStore; +import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.io.BundleRecords; import org.apache.paimon.table.BucketMode; @@ -48,7 +49,6 @@ import static org.apache.fluss.metadata.TableDescriptor.TIMESTAMP_COLUMN_NAME; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -150,10 +150,10 @@ private Object[] writeAndRead( fileStoreTable, tableWrite, paimonRowType, 0, legacyTable); ArrowBatchData batch = new ArrowBatchData(root.slice(0, root.getRowCount()), 0L, 1L, 1)) { - helper.writeArrowBatch(batch, null); + helper.writeArrowBatch(batch, BinaryRow.EMPTY_ROW, false); ArgumentCaptor captor = ArgumentCaptor.forClass(BundleRecords.class); - verify(tableWrite).writeBundle(isNull(), eq(0), captor.capture()); + verify(tableWrite).writeBundle(eq(BinaryRow.EMPTY_ROW), eq(0), captor.capture()); Iterator rows = captor.getValue().iterator(); assertThat(rows.hasNext()).isTrue(); InternalRow row = rows.next(); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java index ef312728215..34f4cba9246 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java @@ -222,11 +222,11 @@ public final class Replica { private volatile @Nullable KvTablet kvTablet; private volatile @Nullable CloseableRegistry closeableRegistryForKv; private @Nullable PeriodicSnapshotManager kvSnapshotManager; - // The lake log end offset used as the durable base of the current historical KV overlay. + // The lake log end offset from which the current local historical KV state was rebuilt. private volatile long historicalKvBaseOffset = -1L; - // Replaced together with the historical KV overlay so stale cleanup tasks can be fenced by - // identity without external replica lifecycle callbacks. - private volatile @Nullable HistoricalWriteState historicalWriteState; + // Replaced whenever the local historical KV state is rebuilt so delayed cleanup tasks for an + // earlier state can be ignored. + private volatile @Nullable HistoricalKvCleanupState historicalKvCleanupState; /** * Server-wide {@link ScannerManager}. Active sessions for this bucket are closed in {@link @@ -332,8 +332,8 @@ public long logicalStorageLogSize() { public long logicalStorageKvSize() { if (isLeader() && isKvTable()) { if (isHistoricalPartition()) { - // Historical KV tablets do not create snapshots, so account for the local overlay - // using live SST files instead. + // Historical KV tablets do not create snapshots, so use live SST files to account + // for their local state instead. KvTablet currentKvTablet = kvTablet; return currentKvTablet == null ? 0L : currentKvTablet.liveSstFilesSize(); } @@ -385,22 +385,22 @@ public long getLakeLogEndOffset() { return logTablet.getLakeLogEndOffset(); } - /** Returns the state owned by the active historical KV overlay, or null if none is active. */ - public @Nullable HistoricalWriteState getHistoricalWriteState() { - return historicalWriteState; + /** Returns the cleanup state for the local historical KV state, or null if it is not ready. */ + public @Nullable HistoricalKvCleanupState getHistoricalKvCleanupState() { + return historicalKvCleanupState; } /** - * Drops and recreates a fully tiered historical KV overlay if leadership and offsets still - * match. + * Drops and recreates local historical KV state after its writes are fully tiered, provided + * leadership and offsets still match. * - * @param expectedLeaderEpoch leader epoch captured when cleanup was scheduled - * @param logEndOffset matching lake and local log end offset that triggered cleanup - * @param beforeCleanup action to run before the overlay is dropped - * @return whether the overlay was cleaned + * @param expectedLeaderEpoch leader epoch required when cleanup runs + * @param tieredLogEndOffset fully tiered log end offset required when cleanup runs + * @param beforeCleanup action to run before the local KV state is dropped + * @return whether the local KV state was cleaned */ public boolean cleanupHistoricalKv( - int expectedLeaderEpoch, long logEndOffset, Runnable beforeCleanup) { + int expectedLeaderEpoch, long tieredLogEndOffset, Runnable beforeCleanup) { checkNotNull(beforeCleanup, "beforeCleanup must not be null"); return inWriteLock( leaderIsrUpdateLock, @@ -410,19 +410,19 @@ public boolean cleanupHistoricalKv( // the current lake and local offsets match while this task still references an // older snapshot. if (leaderEpoch != expectedLeaderEpoch - || localLogEndOffset != logEndOffset + || localLogEndOffset != tieredLogEndOffset || !isEligibleForHistoricalKvCleanup(localLogEndOffset)) { return false; } LOG.info( - "Cleaning historical KV overlay for {} at local log end offset {} " + "Cleaning local historical KV state for {} at log end offset {} " + "covered by lake log end offset {}.", tableBucket, localLogEndOffset, logTablet.getLakeLogEndOffset()); - // A lookup started after the rebuilt empty overlay is published must open a - // lake view that covers the state removed by this cleanup. + // A lookup started after the empty local KV state is rebuilt must open a lake + // view that covers the data removed by this cleanup. beforeCleanup.run(); dropKv(); // TODO: Retry rebuilding this historical bucket instead of waiting for @@ -778,7 +778,7 @@ private void logTableConfigChanges(TableInfo oldTableInfo, TableInfo newTableInf } private boolean isEligibleForHistoricalKvCleanup(long localLogEndOffset) { - // The overlay must have a known lake base, contain writes after that base, and have all + // Local KV state must have a known lake base, contain writes after that base, and have all // those writes covered by lake before it can be discarded. return isLeader() && isHistoricalPartition() @@ -833,7 +833,7 @@ private void createKv() { lastError); } if (isHistoricalPartition()) { - historicalWriteState = new HistoricalWriteState(clock.milliseconds()); + historicalKvCleanupState = new HistoricalKvCleanupState(clock.milliseconds()); } else { startPeriodicKvSnapshot(snapshotUsed.orElse(null)); } @@ -854,7 +854,7 @@ private void dropKv() { kvManager.dropKv(tableBucket); kvTablet = null; } - historicalWriteState = null; + historicalKvCleanupState = null; historicalKvBaseOffset = -1L; } @@ -912,8 +912,8 @@ private Optional initKvTablet() { // get the offset from which, we should restore from. default is 0 long restoreStartOffset = isHistoricalPartition() ? historicalRecoveryStartOffset() : 0; - // The lake snapshot is the durable base for a historical overlay. Historical replicas - // therefore never restore a normal KV snapshot, even if one exists from older code. + // Lake is the durable base for local historical KV state. Historical replicas therefore + // never restore a normal KV snapshot, even if one exists from older code. Optional optCompletedSnapshot = isHistoricalPartition() ? Optional.empty() : getLatestSnapshot(tableBucket); try { @@ -1363,7 +1363,7 @@ public List findKeysRequiringLakeLookup( }); } - /** Writes records to the local historical KV overlay of the leader replica. */ + /** Writes records to the local historical KV state of the leader replica. */ public LogAppendInfo putHistoricalRecordsToLeader( KvRecordBatch kvRecords, @Nullable int[] targetColumns, @@ -1412,7 +1412,7 @@ private void validateHistoricalWrite(int expectedLeaderEpoch, int requiredAcks) validateInSyncReplicaSize(requiredAcks); } - /** Looks up keys from the local historical KV overlay of the leader replica. */ + /** Looks up keys from the local historical KV state of the leader replica. */ public List lookupHistoricalLocal( String originalPartitionName, List keys) throws Exception { return inReadLock( @@ -2636,16 +2636,17 @@ public PeriodicSnapshotManager getKvSnapshotManager() { return kvSnapshotManager; } - /** Write activity and maximum-size state owned by one historical KV overlay. */ + /** Tracks write activity and cleanup conditions for the current local historical KV state. */ @ThreadSafe - public static final class HistoricalWriteState { - // Latched when the live SST size reaches the maximum. Replacing the overlay replaces this - // state, so transient RocksDB size changes cannot resume writes prematurely. + public static final class HistoricalKvCleanupState { + // Latched when the live SST size reaches the maximum. Rebuilding the local KV state resets + // this flag, while transient RocksDB size changes do not resume writes prematurely. private final AtomicBoolean maxSizeReached = new AtomicBoolean(); + private volatile @Nullable CleanupCandidate cleanupCandidate; private volatile long lastWriteMs; - private HistoricalWriteState(long lastWriteMs) { + private HistoricalKvCleanupState(long lastWriteMs) { this.lastWriteMs = lastWriteMs; } @@ -2659,6 +2660,18 @@ public boolean markMaxSizeReached() { return maxSizeReached.compareAndSet(false, true); } + /** Updates the cleanup candidate for the current local historical KV state. */ + public void updateCleanupCandidate( + long lakeSnapshotId, int expectedLeaderEpoch, long tieredLogEndOffset) { + cleanupCandidate = + new CleanupCandidate(lakeSnapshotId, expectedLeaderEpoch, tieredLogEndOffset); + } + + /** Returns the cleanup candidate, or null if none is available. */ + public @Nullable CleanupCandidate cleanupCandidate() { + return cleanupCandidate; + } + /** Records the latest historical write activity time. */ public void recordWrite(long timestampMs) { lastWriteMs = timestampMs; @@ -2668,5 +2681,34 @@ public void recordWrite(long timestampMs) { public long lastWriteMs() { return lastWriteMs; } + + /** A candidate for cleaning up local historical KV state. */ + public static final class CleanupCandidate { + private final long lakeSnapshotId; + private final int expectedLeaderEpoch; + private final long tieredLogEndOffset; + + private CleanupCandidate( + long lakeSnapshotId, int expectedLeaderEpoch, long tieredLogEndOffset) { + this.lakeSnapshotId = lakeSnapshotId; + this.expectedLeaderEpoch = expectedLeaderEpoch; + this.tieredLogEndOffset = tieredLogEndOffset; + } + + /** Returns the lake snapshot ID that covers the local KV state. */ + public long lakeSnapshotId() { + return lakeSnapshotId; + } + + /** Returns the leader epoch required to run cleanup. */ + public int expectedLeaderEpoch() { + return expectedLeaderEpoch; + } + + /** Returns the log end offset covered by the lake snapshot. */ + public long tieredLogEndOffset() { + return tieredLogEndOffset; + } + } } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java index bea89b18ffb..37b47d3d4ed 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java @@ -1487,9 +1487,9 @@ private void updateWithLakeTableSnapshot(Replica replica) throws Exception { .getLogEndOffset(tb) .ifPresent(replica.getLogTablet()::updateLakeLogEndOffset); if (replica.isHistoricalPartition()) { - // The historical overlay will be rebuilt from this snapshot's lake offset. + // Local historical KV state will be rebuilt from this snapshot's lake offset. // Refresh a cached lookuper before it becomes the fallback for data omitted - // from the rebuilt overlay. + // from the rebuilt local state. historicalPartitionManager.requireLakeSnapshot( replica.getTableBucket().getTableId(), snapshotId); } @@ -1498,7 +1498,7 @@ private void updateWithLakeTableSnapshot(Replica replica) throws Exception { if (replica.isHistoricalPartition()) { // Historical recovery uses the lake offset as its durable base and replays the // retained WAL from that offset. Reject leader activation if the latest lake - // progress cannot be loaded, instead of rebuilding the overlay from stale state. + // progress cannot be loaded, instead of rebuilding local KV from stale state. throw e; } // Lake commit cleanup can race with this best-effort refresh and remove the diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java index 7bd230e3010..93d720ae6fe 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java @@ -24,6 +24,7 @@ import org.apache.fluss.exception.ConfigException; import org.apache.fluss.exception.HistoricalPartitionThrottledException; import org.apache.fluss.exception.InvalidPartitionException; +import org.apache.fluss.exception.KvStorageException; import org.apache.fluss.lake.lakestorage.LakeTableLookuper; import org.apache.fluss.metadata.ResolvedPartitionSpec; import org.apache.fluss.metadata.TableBucket; @@ -42,7 +43,7 @@ import org.apache.fluss.server.kv.historical.HistoricalValueLookup; import org.apache.fluss.server.log.LogAppendInfo; import org.apache.fluss.server.replica.Replica; -import org.apache.fluss.server.replica.Replica.HistoricalWriteState; +import org.apache.fluss.server.replica.Replica.HistoricalKvCleanupState; import org.apache.fluss.server.storage.LocalDiskManager; import org.apache.fluss.utils.ByteArraySlice; import org.apache.fluss.utils.ByteArrayWrapper; @@ -147,23 +148,15 @@ public void onLakeProgress(Replica replica, long lakeSnapshotId, long lakeLogEnd if (lakeLogEndOffset != localLogEndOffset) { return; } - HistoricalWriteState state = replica.getHistoricalWriteState(); - if (state == null) { + HistoricalKvCleanupState cleanupState = replica.getHistoricalKvCleanupState(); + if (cleanupState == null) { return; } - boolean maxSizeReached = state.maxSizeReached(); - // Lake progress establishes the cleanup candidate. Idle cleanup waits until the last write - // reaches its deadline, then enters the ordered queue behind already accepted writes. - scheduleCleanup( - replica, - state, - maxSizeReached, - lakeSnapshotId, - expectedLeaderEpoch, - lakeLogEndOffset); + cleanupState.updateCleanupCandidate(lakeSnapshotId, expectedLeaderEpoch, lakeLogEndOffset); + tryScheduleCleanup(replica, cleanupState); } - /** Looks up historical keys from the local overlay and then lake storage. */ + /** Looks up historical keys from local KV state and then lake storage. */ public CompletableFuture lookup( Replica replica, LookupDataForBucket lookupData, @@ -195,7 +188,7 @@ public CompletableFuture lookup( } } - /** Writes records to the local overlay of a historical partition. */ + /** Writes records to the local KV state of a historical partition. */ public CompletableFuture putKv( Replica replica, PutKvDataForBucket putData, @@ -207,23 +200,26 @@ public CompletableFuture putKv( checkNotNull( putData.originalPartitionName(), "originalPartitionName must not be null"); - HistoricalWriteState state = - checkNotNull( - replica.getHistoricalWriteState(), - "No active historical KV overlay for " + replica.getTableBucket()); - if (state.maxSizeReached()) { + HistoricalKvCleanupState cleanupState = replica.getHistoricalKvCleanupState(); + if (cleanupState == null) { + throw new KvStorageException( + "Local historical KV state is not ready for " + + replica.getTableBucket() + + " because its KV tablet is being initialized or rebuilt."); + } + if (cleanupState.maxSizeReached()) { return CompletableFuture.completedFuture( maxSizeThrottledResult( putData, originalPartitionName, maxHistoricalKvSizeBytes)); } long liveSstSize = replica.logicalStorageKvSize(); if (liveSstSize >= maxHistoricalKvSizeBytes) { - markMaxSizeReached(replica, state); + markMaxSizeReached(replica, cleanupState); return CompletableFuture.completedFuture( maxSizeThrottledResult( putData, originalPartitionName, maxHistoricalKvSizeBytes)); } - state.recordWrite(clock.milliseconds()); + cleanupState.recordWrite(clock.milliseconds()); return taskExecutor.submitOrdered( putData.tableBucket(), () -> { @@ -383,33 +379,55 @@ LogAppendInfo processPut( requiredAcks); } - private void markMaxSizeReached(Replica replica, HistoricalWriteState state) { - if (state.markMaxSizeReached()) { + private void markMaxSizeReached(Replica replica, HistoricalKvCleanupState cleanupState) { + if (cleanupState.markMaxSizeReached()) { LOG.warn( "Pausing historical writes for {} because its live SST size reached the " + "maximum size {} bytes.", replica.getTableBucket(), maxHistoricalKvSizeBytes); + + tryScheduleCleanup(replica, cleanupState); + } + } + + private void tryScheduleCleanup(Replica replica, HistoricalKvCleanupState cleanupState) { + HistoricalKvCleanupState.CleanupCandidate candidate = cleanupState.cleanupCandidate(); + if (candidate == null) { + return; } + scheduleCleanup( + replica, + cleanupState, + cleanupState.maxSizeReached(), + candidate.lakeSnapshotId(), + candidate.expectedLeaderEpoch(), + candidate.tieredLogEndOffset()); } private void scheduleCleanup( Replica replica, - HistoricalWriteState state, + HistoricalKvCleanupState cleanupState, boolean maxSizeReached, long lakeSnapshotId, int expectedLeaderEpoch, - long logEndOffset) { + long tieredLogEndOffset) { + // Reject cleanup if the replica no longer matches the cleanup state, leader epoch, or + // tiered log end offset used when it was scheduled. if (closed - || replica.getHistoricalWriteState() != state + || replica.getHistoricalKvCleanupState() != cleanupState || expectedLeaderEpoch != replica.getLeaderEpoch() - || replica.getLocalLogEndOffset() != logEndOffset) { + || replica.getLocalLogEndOffset() != tieredLogEndOffset) { return; } if (!maxSizeReached && deferIdleCleanupIfNeeded( - replica, state, lakeSnapshotId, expectedLeaderEpoch, logEndOffset)) { + replica, + cleanupState, + lakeSnapshotId, + expectedLeaderEpoch, + tieredLogEndOffset)) { return; } @@ -420,11 +438,11 @@ && deferIdleCleanupIfNeeded( () -> runCleanup( replica, - state, + cleanupState, maxSizeReached, lakeSnapshotId, expectedLeaderEpoch, - logEndOffset)); + tieredLogEndOffset)); cleanupFuture.whenComplete( (ignored, error) -> { if (error != null) { @@ -438,27 +456,31 @@ && deferIdleCleanupIfNeeded( private void runCleanup( Replica replica, - HistoricalWriteState state, + HistoricalKvCleanupState cleanupState, boolean maxSizeReached, long lakeSnapshotId, int expectedLeaderEpoch, - long logEndOffset) { - if (replica.getHistoricalWriteState() != state + long tieredLogEndOffset) { + if (replica.getHistoricalKvCleanupState() != cleanupState || expectedLeaderEpoch != replica.getLeaderEpoch()) { return; } if (!maxSizeReached && deferIdleCleanupIfNeeded( - replica, state, lakeSnapshotId, expectedLeaderEpoch, logEndOffset)) { + replica, + cleanupState, + lakeSnapshotId, + expectedLeaderEpoch, + tieredLogEndOffset)) { return; } if (replica.cleanupHistoricalKv( expectedLeaderEpoch, - logEndOffset, + tieredLogEndOffset, () -> requireLakeSnapshot(replica.getTableBucket().getTableId(), lakeSnapshotId))) { LOG.info( - "Cleaned {} historical KV overlay for {}.", + "Cleaned {} local historical KV state for {}.", maxSizeReached ? "max-size-triggered" : "idle-triggered", replica.getTableBucket()); } @@ -470,15 +492,15 @@ && deferIdleCleanupIfNeeded( */ private boolean deferIdleCleanupIfNeeded( Replica replica, - HistoricalWriteState state, + HistoricalKvCleanupState cleanupState, long lakeSnapshotId, int expectedLeaderEpoch, - long logEndOffset) { + long tieredLogEndOffset) { long idleTimeMs = cleanupIdleTimeMs; if (idleTimeMs <= 0L) { return true; } - long delayMs = remainingIdleCleanupDelayMs(state, idleTimeMs); + long delayMs = remainingIdleCleanupDelayMs(cleanupState, idleTimeMs); if (delayMs <= 0L) { return false; } @@ -488,19 +510,20 @@ private boolean deferIdleCleanupIfNeeded( () -> scheduleCleanup( replica, - state, + cleanupState, false, lakeSnapshotId, expectedLeaderEpoch, - logEndOffset), + tieredLogEndOffset), delayMs); return true; } /** Returns the remaining delay before idle cleanup is eligible, or {@code 0} if it is due. */ - private long remainingIdleCleanupDelayMs(HistoricalWriteState state, long idleTimeMs) { + private long remainingIdleCleanupDelayMs( + HistoricalKvCleanupState cleanupState, long idleTimeMs) { long now = clock.milliseconds(); - long lastWriteMs = state.lastWriteMs(); + long lastWriteMs = cleanupState.lastWriteMs(); if (now < lastWriteMs) { // A backward clock jump restarts the idle window instead of cleaning prematurely. return idleTimeMs; @@ -532,12 +555,12 @@ private static PutKvResultForBucket maxSizeThrottledResult( + putData.tableBucket() + " (original partition " + originalPartitionName - + ") because its historical KV overlay reached the live " + + ") because its local historical KV state reached the live " + "SST maximum size of " + maxHistoricalKvSize + " bytes. New writes are paused until lake tiering " - + "covers all previously accepted writes and the local " - + "overlay cleanup completes.")), + + "covers all previously accepted writes and cleanup of " + + "the local historical KV state completes.")), originalPartitionName); } diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java index d5ca84c3173..526623b050e 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java @@ -22,7 +22,6 @@ import org.apache.fluss.config.AutoPartitionTimeUnit; import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; -import org.apache.fluss.exception.ConfigException; import org.apache.fluss.exception.HistoricalPartitionThrottledException; import org.apache.fluss.exception.InvalidPartitionException; import org.apache.fluss.lake.lakestorage.LakeTableLookuper; @@ -269,7 +268,7 @@ void testHistoricalInsertUpdateAndDelete() throws Exception { byte[] primaryKey = new CompactedKeyEncoder(keyType).encodeKey(row(1, "us")); try { - // The first write misses both local state and lake, so it creates a local overlay. + // The first write misses both local KV state and lake, so it creates a local value. KvRecordBatch insertBatch = batch( keyType, @@ -328,7 +327,7 @@ void testHistoricalInsertUpdateAndDelete() throws Exception { row(1, "us", "20240108", "another")); assertThat(lakeLookupManager.lookupCount).hasValue(2); - // Exercise the ReplicaManager entry point; the update should reuse the local overlay. + // Exercise the ReplicaManager entry point; the update should reuse local KV state. KvRecordBatch updateBatch = batch( keyType, @@ -372,7 +371,7 @@ void testHistoricalInsertUpdateAndDelete() throws Exception { row(1, "us", "20240108", "another")); assertThat(lakeLookupManager.lookupCount).hasValue(2); - // Historical lookup should observe the updated value from the local overlay. + // Historical lookup should observe the updated value from local KV state. CompletableFuture> lookupResponse = new CompletableFuture<>(); replicaManager.historicalLookups( @@ -599,7 +598,7 @@ void testLeaderChangeDuringLakeLookupFencesHistoricalWrite() throws Exception { } @Test - void testRecoversHistoricalOverlayFromLakeCommitOffset() throws Exception { + void testRecoversLocalHistoricalKvFromLakeCommitOffset() throws Exception { TableInfo tableInfo = registerHistoricalTableAndBecomeLeader(); Replica replica = replicaManager.getReplicaOrException(TABLE_BUCKET); TestingHistoricalLakeLookupManager lakeLookupManager = @@ -686,7 +685,7 @@ void testRecoversHistoricalOverlayFromLakeCommitOffset() throws Exception { // Persist the exclusive end offset of the first write as the lake recovery point. The // replica has not received this offset locally, so becoming leader must load it before - // creating the historical overlay. + // creating the local historical KV state. long lakeCommitOffset = firstAppend.lastOffset() + 1; new LakeTableHelper(zkClient, DEFAULT_REMOTE_DATA_DIR) .registerLakeTableSnapshotV1( @@ -695,8 +694,8 @@ void testRecoversHistoricalOverlayFromLakeCommitOffset() throws Exception { 1L, Collections.singletonMap(TABLE_BUCKET, lakeCommitOffset))); assertThat(replica.getLakeLogEndOffset()).isEqualTo(-1L); - // Dropping and recreating the leader KV tablet forces the overlay to be rebuilt only - // from WAL after the lake commit offset. The recovered tombstone must remain + // Dropping and recreating the leader KV tablet rebuilds its state only from WAL after + // the lake commit offset. The recovered tombstone must remain // authoritative over lake fallback. assertThat(replica.makeFollower(followerState())).isTrue(); CompletableFuture> leaderFuture = @@ -736,7 +735,7 @@ void testRecoversHistoricalOverlayFromLakeCommitOffset() throws Exception { } @Test - void testCleansFullyTieredHistoricalOverlayAfterWriteIdleTime() throws Exception { + void testCleansLocalHistoricalKvAfterTieringAndWriteIdleTime() throws Exception { TableInfo tableInfo = registerHistoricalTableAndBecomeLeader(); Replica replica = replicaManager.getReplicaOrException(TABLE_BUCKET); KvTablet originalKvTablet = replica.getKvTablet(); @@ -780,8 +779,8 @@ void testCleansFullyTieredHistoricalOverlayAfterWriteIdleTime() throws Exception replica.getLogTablet().updateLakeLogEndOffset(tieredOffset); historicalPartitionManager.onLakeProgress(replica, 10L, tieredOffset); - // Lake normally catches up before the overlay becomes idle. Cleanup must wake at the - // write deadline even if no further lake progress notification arrives. + // Lake normally catches up before local KV state becomes idle. Cleanup must wake at + // the write deadline even if no further lake progress notification arrives. assertThat(executor.numQueuedRunnables()).isZero(); assertThat(executor.getActiveNonPeriodicScheduledTask()).hasSize(1); assertThat(replica.getKvTablet()).isSameAs(originalKvTablet); @@ -808,41 +807,13 @@ void testCleansFullyTieredHistoricalOverlayAfterWriteIdleTime() throws Exception (lookupTimeNanos, lookupFileDownloaded) -> {}); executor.triggerAll(); assertThat(lookupFuture.get(10, TimeUnit.SECONDS).lookupValues()) + .extracting(ByteArraySlice::toByteArray) .containsExactly(lakeValue); } finally { historicalPartitionManager.close(); } } - @Test - void testRejectsNegativeCleanupIdleTimeDuringReconfiguration() { - Configuration cleanupConf = lookupConfiguration(); - ManuallyTriggeredScheduledExecutorService executor = - new ManuallyTriggeredScheduledExecutorService(); - HistoricalPartitionManager historicalPartitionManager = - new HistoricalPartitionManager( - cleanupConf, - new HistoricalPartitionTaskExecutor(cleanupConf, executor), - new TestingHistoricalLakeLookupManager(cleanupConf), - manualClock, - Long.MAX_VALUE, - new TestingCleanupScheduler(executor)); - Configuration invalidConf = new Configuration(cleanupConf); - invalidConf.set( - ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME, - Duration.ofMillis(-1)); - - try { - assertThatThrownBy(() -> historicalPartitionManager.validate(invalidConf)) - .isInstanceOf(ConfigException.class) - .hasMessageContaining( - ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME.key(), - "must not be negative"); - } finally { - historicalPartitionManager.close(); - } - } - @Test void testCleanupRequiresLakeAndLocalOffsetsToMatch() throws Exception { TableInfo tableInfo = registerHistoricalTableAndBecomeLeader(); @@ -914,7 +885,7 @@ void testCleanupRequiresLakeAndLocalOffsetsToMatch() throws Exception { } @Test - void testMaxSizeBlocksWritesUntilOverlayIsTieredAndCleaned() throws Exception { + void testMaxSizeBlocksWritesUntilAcceptedWritesAreTieredAndLocalKvIsCleaned() throws Exception { TableInfo tableInfo = registerHistoricalTableAndBecomeLeader(); Replica replica = replicaManager.getReplicaOrException(TABLE_BUCKET); KvTablet originalKvTablet = replica.getKvTablet(); @@ -984,6 +955,75 @@ void testMaxSizeBlocksWritesUntilOverlayIsTieredAndCleaned() throws Exception { } } + @Test + void testMaxSizeCleanupWhenLakeCaughtUpBeforeLimitIsObserved() throws Exception { + TableInfo tableInfo = registerHistoricalTableAndBecomeLeader(); + Replica replica = replicaManager.getReplicaOrException(TABLE_BUCKET); + KvTablet originalKvTablet = replica.getKvTablet(); + assertThat(originalKvTablet).isNotNull(); + + Configuration cleanupConf = lookupConfiguration(); + cleanupConf.set( + ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME, Duration.ZERO); + ManuallyTriggeredScheduledExecutorService executor = + new ManuallyTriggeredScheduledExecutorService(); + TestingHistoricalLakeLookupManager lakeLookupManager = + new TestingHistoricalLakeLookupManager(cleanupConf); + HistoricalPartitionManager historicalPartitionManager = + createCleanupManager(cleanupConf, executor, lakeLookupManager, 1L); + + KvRecordBatch firstBatch = + batch( + HISTORICAL_KEY_TYPE, + tableInfo.getRowType(), + Tuple2.of( + new Object[] {1, "us"}, + new Object[] {1, "us", ORIGINAL_PARTITION, "v1"})); + KvRecordBatch secondBatch = + batch( + HISTORICAL_KEY_TYPE, + tableInfo.getRowType(), + Tuple2.of( + new Object[] {2, "eu"}, + new Object[] {2, "eu", ORIGINAL_PARTITION, "v2"})); + + try { + CompletableFuture firstPut = + putHistoricalRecords(historicalPartitionManager, replica, firstBatch); + executor.triggerAll(); + assertThat(firstPut.get(10, TimeUnit.SECONDS).failed()).isFalse(); + flushAndWait(originalKvTablet, Long.MAX_VALUE); + try (FlushOptions flushOptions = new FlushOptions().setWaitForFlush(true)) { + originalKvTablet.getRocksDBKv().getDb().flush(flushOptions); + } + assertThat(originalKvTablet.liveSstFilesSize()).isPositive(); + + // Lake catches up before another request observes that the SST size reached the limit. + long tieredOffset = replica.getLocalLogEndOffset(); + replica.getLogTablet().updateLakeLogEndOffset(tieredOffset); + historicalPartitionManager.onLakeProgress(replica, 12L, tieredOffset); + assertThat(executor.numQueuedRunnables()).isZero(); + + PutKvResultForBucket blockedWrite = + putHistoricalRecords(historicalPartitionManager, replica, secondBatch) + .get(10, TimeUnit.SECONDS); + assertThat(blockedWrite.getError().error()) + .isEqualTo(Errors.HISTORICAL_PARTITION_THROTTLED); + assertThat(executor.numQueuedRunnables()).isOne(); + + executor.triggerAll(); + assertThat(replica.getKvTablet()).isNotNull().isNotSameAs(originalKvTablet); + assertThat(lakeLookupManager.requiredLakeSnapshotIds).containsExactly(12L); + + CompletableFuture resumedWrite = + putHistoricalRecords(historicalPartitionManager, replica, secondBatch); + executor.triggerAll(); + assertThat(resumedWrite.get(10, TimeUnit.SECONDS).failed()).isFalse(); + } finally { + historicalPartitionManager.close(); + } + } + @Test void testHistoricalLookupThrottledWhenPermitsExhausted() throws Exception { registerHistoricalTableAndBecomeLeader();