diff --git a/fluss-client/src/main/java/org/apache/fluss/client/utils/ClientRpcMessageUtils.java b/fluss-client/src/main/java/org/apache/fluss/client/utils/ClientRpcMessageUtils.java index 3c7512945dc..1a40afc92be 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/utils/ClientRpcMessageUtils.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/utils/ClientRpcMessageUtils.java @@ -147,6 +147,10 @@ public static ProduceLogRequest makeProduceLogRequest( if (tableBucket.getPartitionId() != null) { pbProduceLogReqForBucket.setPartitionId(tableBucket.getPartitionId()); } + if (readyBatch.writeBatch().getOriginalPartitionName() != null) { + pbProduceLogReqForBucket.setOriginalPartitionName( + readyBatch.writeBatch().getOriginalPartitionName()); + } }); return request; } @@ -202,6 +206,11 @@ public static PutKvRequest makePutKvRequest( if (tableBucket.getPartitionId() != null) { pbPutKvReqForBucket.setPartitionId(tableBucket.getPartitionId()); } + KvWriteBatch kvWriteBatch = (KvWriteBatch) readyBatch.writeBatch(); + if (kvWriteBatch.getOriginalPartitionName() != null) { + pbPutKvReqForBucket.setOriginalPartitionName( + kvWriteBatch.getOriginalPartitionName()); + } }); return request; } diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/AbstractRowLogWriteBatch.java b/fluss-client/src/main/java/org/apache/fluss/client/write/AbstractRowLogWriteBatch.java index 104f8f29e33..d9afe697600 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/AbstractRowLogWriteBatch.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/AbstractRowLogWriteBatch.java @@ -26,6 +26,8 @@ import org.apache.fluss.record.bytesview.BytesView; import org.apache.fluss.row.InternalRow; +import javax.annotation.Nullable; + import java.io.IOException; import java.util.List; @@ -49,11 +51,19 @@ protected AbstractRowLogWriteBatch( PhysicalTablePath physicalTablePath, int schemaId, WriteFormat writeFormat, + @Nullable String originalPartitionName, long createdMs, AbstractPagedOutputView outputView, MemoryLogRecordsRowBuilder recordsBuilder, String buildErrorMessage) { - super(tableId, bucketId, physicalTablePath, schemaId, writeFormat, createdMs); + super( + tableId, + bucketId, + physicalTablePath, + schemaId, + writeFormat, + originalPartitionName, + createdMs); this.outputView = outputView; this.recordsBuilder = recordsBuilder; this.buildErrorMessage = buildErrorMessage; diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/ArrowLogWriteBatch.java b/fluss-client/src/main/java/org/apache/fluss/client/write/ArrowLogWriteBatch.java index a6894b98b2e..4c4f293aa8f 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/ArrowLogWriteBatch.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/ArrowLogWriteBatch.java @@ -58,9 +58,17 @@ public ArrowLogWriteBatch( int schemaId, ArrowWriter arrowWriter, AbstractPagedOutputView outputView, + @Nullable String originalPartitionName, long createdMs, @Nullable LogRecordBatchStatisticsCollector statisticsCollector) { - super(tableId, bucketId, physicalTablePath, schemaId, WriteFormat.ARROW_LOG, createdMs); + super( + tableId, + bucketId, + physicalTablePath, + schemaId, + WriteFormat.ARROW_LOG, + originalPartitionName, + createdMs); this.outputView = outputView; this.recordsBuilder = MemoryLogRecordsArrowBuilder.builder( diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/CompactedLogWriteBatch.java b/fluss-client/src/main/java/org/apache/fluss/client/write/CompactedLogWriteBatch.java index 81bedff0fa0..1e3fc0b8dbc 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/CompactedLogWriteBatch.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/CompactedLogWriteBatch.java @@ -25,6 +25,7 @@ import org.apache.fluss.row.compacted.CompactedRow; import org.apache.fluss.rpc.messages.ProduceLogRequest; +import javax.annotation.Nullable; import javax.annotation.concurrent.NotThreadSafe; import static org.apache.fluss.utils.Preconditions.checkArgument; @@ -46,6 +47,7 @@ public CompactedLogWriteBatch( int schemaId, int writeLimit, AbstractPagedOutputView outputView, + @Nullable String originalPartitionName, long createdMs) { super( tableId, @@ -53,6 +55,7 @@ public CompactedLogWriteBatch( physicalTablePath, schemaId, WriteFormat.COMPACTED_LOG, + originalPartitionName, createdMs, outputView, MemoryLogRecordsCompactedBuilder.builder(schemaId, writeLimit, outputView, true), diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/IndexedLogWriteBatch.java b/fluss-client/src/main/java/org/apache/fluss/client/write/IndexedLogWriteBatch.java index 2bb496cbfe3..2b50402218b 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/IndexedLogWriteBatch.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/IndexedLogWriteBatch.java @@ -24,6 +24,7 @@ import org.apache.fluss.row.indexed.IndexedRow; import org.apache.fluss.rpc.messages.ProduceLogRequest; +import javax.annotation.Nullable; import javax.annotation.concurrent.NotThreadSafe; import static org.apache.fluss.utils.Preconditions.checkArgument; @@ -45,6 +46,7 @@ public IndexedLogWriteBatch( int schemaId, int writeLimit, AbstractPagedOutputView outputView, + @Nullable String originalPartitionName, long createdMs) { super( tableId, @@ -52,6 +54,7 @@ public IndexedLogWriteBatch( physicalTablePath, schemaId, WriteFormat.INDEXED_LOG, + originalPartitionName, createdMs, outputView, MemoryLogRecordsIndexedBuilder.builder(schemaId, writeLimit, outputView, true), diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/KvWriteBatch.java b/fluss-client/src/main/java/org/apache/fluss/client/write/KvWriteBatch.java index 4ca01e133b2..0a5c16db688 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/KvWriteBatch.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/KvWriteBatch.java @@ -63,6 +63,7 @@ public KvWriteBatch( AbstractPagedOutputView outputView, @Nullable int[] targetColumns, MergeMode mergeMode, + @Nullable String originalPartitionName, long createdMs) { super( tableId, @@ -70,6 +71,7 @@ public KvWriteBatch( physicalTablePath, schemaId, WriteFormat.fromKvFormat(kvFormat), + originalPartitionName, createdMs); this.outputView = outputView; this.recordsBuilder = diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java b/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java index ffa18ad9455..19362d8a948 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java @@ -32,6 +32,7 @@ import org.apache.fluss.metadata.PhysicalTablePath; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; import org.apache.fluss.metrics.MetricNames; import org.apache.fluss.record.LogRecordBatchStatisticsCollector; import org.apache.fluss.row.arrow.ArrowWriter; @@ -67,6 +68,7 @@ import static org.apache.fluss.record.LogRecordBatchFormat.NO_BATCH_SEQUENCE; import static org.apache.fluss.record.LogRecordBatchFormat.NO_WRITER_ID; import static org.apache.fluss.shaded.arrow.org.apache.arrow.memory.BufferAllocatorUtil.createBufferAllocator; +import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; import static org.apache.fluss.utils.Preconditions.checkNotNull; /* This file is based on source code of Apache Kafka Project (https://kafka.apache.org/), licensed by the Apache @@ -115,6 +117,9 @@ public final class RecordAccumulator { private final ConcurrentMap writeBatches = new CopyOnWriteMap<>(); + /** Tables observed by this writer with historical partition support enabled. */ + private final Set historicalPartitionEnabledTables = ConcurrentHashMap.newKeySet(); + private final IncompleteBatches incomplete; private final Map nodesDrainIndex; @@ -198,6 +203,9 @@ public RecordAppendResult append( throws Exception { PhysicalTablePath physicalTablePath = writeRecord.getPhysicalTablePath(); TableInfo tableInfo = writeRecord.getTableInfo(); + if (tableInfo.getTableConfig().isHistoricalPartitionEnabled()) { + historicalPartitionEnabledTables.add(tableInfo.getTablePath()); + } // The metadata may return null for the partition id, but it is fine to pass null here, // because we will fill the partitionId in bucketReady() before send the batch. Optional partitionIdOpt = cluster.getPartitionId(physicalTablePath); @@ -206,7 +214,9 @@ public RecordAppendResult append( physicalTablePath, k -> new BucketAndWriteBatches( - partitionIdOpt.orElse(null), tableInfo.isPartitioned())); + partitionIdOpt.orElse(null), + tableInfo.isPartitioned(), + physicalTablePath)); // We keep track of the number of appending thread to make sure we do not miss batches in // abortIncompleteBatches(). @@ -331,6 +341,78 @@ public void reEnqueue(ReadyWriteBatch readyWriteBatch) { } } + /** + * Routes writes for an original partition path to the given physical target. + * + *

The accumulator keeps queues keyed by {@code originalPath}, while metadata lookup, leader + * discovery, and RPC sending use {@code targetPath}. The target may therefore be either the + * original partition itself or the shared historical partition. + * + *

A normal target can be replaced by the historical target only while the original path has + * no incomplete batch. This method never moves queued or inflight batches between physical + * partitions. + * + * @throws FlussRuntimeException if a different target was fixed previously + */ + void routeWritesTo( + PhysicalTablePath originalPath, PhysicalTablePath targetPath, long targetPartitionId) { + BucketAndWriteBatches resolvedTarget = + new BucketAndWriteBatches(targetPartitionId, true, targetPath); + // Install the route atomically before append can create the first queue for this path. + BucketAndWriteBatches existing = writeBatches.putIfAbsent(originalPath, resolvedTarget); + if (existing == null) { + return; + } + + // Pair with appendNewBatch(). For a historical route change, either a normal batch is + // registered as incomplete first and rejects the switch, or batch creation observes the + // historical target. + synchronized (existing) { + if (existing.targetPath.equals(targetPath)) { + existing.partitionId = targetPartitionId; + return; + } + if (!existing.isHistoricalWriteTarget() && resolvedTarget.isHistoricalWriteTarget()) { + if (hasIncompleteBatchFor(originalPath)) { + throw new FlussRuntimeException( + String.format( + "Cannot route writes for %s to %s while this writer has " + + "incomplete writes to %s.", + originalPath, targetPath, existing.targetPath)); + } + existing.targetPath = targetPath; + existing.partitionId = targetPartitionId; + return; + } + + throw new FlussRuntimeException( + String.format( + "Cannot route writes for %s to %s because this writer already routed " + + "the partition to %s.", + originalPath, targetPath, existing.targetPath)); + } + } + + private boolean hasIncompleteBatchFor(PhysicalTablePath physicalTablePath) { + for (WriteBatch batch : incomplete.copyAll()) { + if (batch.physicalTablePath().equals(physicalTablePath)) { + return true; + } + } + return false; + } + + /** Returns whether this original path is already routed to the historical partition. */ + boolean hasHistoricalWriteTarget(PhysicalTablePath originalPath) { + BucketAndWriteBatches writeTarget = writeBatches.get(originalPath); + return writeTarget != null && writeTarget.isHistoricalWriteTarget(); + } + + /** Returns whether the target belongs to a table with historical partition support enabled. */ + boolean isHistoricalPartitionEnabled(PhysicalTablePath targetPath) { + return historicalPartitionEnabledTables.contains(targetPath.getTablePath()); + } + /** Abort all incomplete batches (whether they have been sent or not). */ public void abortAllBatches(final Exception reason) { for (WriteBatch batch : incomplete.copyAll()) { @@ -357,7 +439,8 @@ private Deque getOrCreateDeque( k -> new BucketAndWriteBatches( tableBucket.getPartitionId(), - physicalTablePath.getPartitionName() != null)); + physicalTablePath.getPartitionName() != null, + physicalTablePath)); return bucketAndWriteBatches.batches.computeIfAbsent( tableBucket.getBucket(), k -> new ArrayDeque<>()); } @@ -485,8 +568,9 @@ private long bucketReady( Cluster cluster, long nextReadyCheckDelayMs) { // first check this table has partitionId. + PhysicalTablePath targetPath = bucketAndWriteBatches.targetPath; if (bucketAndWriteBatches.isPartitionedTable && bucketAndWriteBatches.partitionId == null) { - Optional optionIdOpt = cluster.getPartitionId(physicalTablePath); + Optional optionIdOpt = cluster.getPartitionId(targetPath); if (optionIdOpt.isPresent()) { bucketAndWriteBatches.partitionId = optionIdOpt.get(); } else { @@ -495,7 +579,7 @@ private long bucketReady( physicalTablePath); // TODO: we shouldn't add unready partitions to unknownLeaderTables, // because it cases PartitionNotExistException later - unknownLeaderTables.add(physicalTablePath); + unknownLeaderTables.add(targetPath); return nextReadyCheckDelayMs; } } @@ -531,10 +615,10 @@ private long bucketReady( int bucketId = entry.getKey(); Optional tableIdOpt = cluster.getTableId(physicalTablePath.getTablePath()); if (!tableIdOpt.isPresent()) { - unknownLeaderTables.add(physicalTablePath); + unknownLeaderTables.add(targetPath); } else { TableBucket tableBucket = - cluster.getTableBucket(tableIdOpt.get(), physicalTablePath, bucketId); + cluster.getTableBucket(tableIdOpt.get(), targetPath, bucketId); // If this bucket is throttled, don't mark its node as ready. // Instead, factor the remaining throttle time into the next check delay. @@ -556,7 +640,7 @@ private long bucketReady( // This is a bucket for which leader is not known, but messages are // available to send. Note that entries are currently not removed from // batches when deque is empty. - unknownLeaderTables.add(physicalTablePath); + unknownLeaderTables.add(targetPath); } else { nextReadyCheckDelayMs = batchReady( @@ -616,31 +700,49 @@ private RecordAppendResult appendNewBatch( Deque deque, List segments) throws Exception { - RecordAppendResult appendResult = tryAppend(writeRecord, callback, deque); - if (appendResult != null) { - // Somebody else found us a batch, return the one we waited for! Hopefully this doesn't - // happen often... - return appendResult; - } - PhysicalTablePath physicalTablePath = writeRecord.getPhysicalTablePath(); - PreAllocatedPagedOutputView outputView = new PreAllocatedPagedOutputView(segments); - int schemaId = tableInfo.getSchemaId(); - WriteFormat writeFormat = writeRecord.getWriteFormat(); - final WriteBatch batch = - createWriteBatch( - writeRecord, - bucketId, - tableInfo, - writeFormat, - physicalTablePath, - outputView, - schemaId); + BucketAndWriteBatches bucketAndWriteBatches = + checkNotNull( + writeBatches.get(physicalTablePath), + "Write batches for %s must exist.", + physicalTablePath); + // Only historical-enabled tables need to coordinate with routeWritesTo(). Other tables + // reuse the deque monitor already held by the caller, avoiding cross-bucket serialization. + Object routeLock = + tableInfo.getTableConfig().isHistoricalPartitionEnabled() + ? bucketAndWriteBatches + : deque; + synchronized (routeLock) { + RecordAppendResult appendResult = tryAppend(writeRecord, callback, deque); + if (appendResult != null) { + // Somebody else found us a batch, return the one we waited for! Hopefully this + // doesn't happen often... + return appendResult; + } - batch.tryAppend(writeRecord, callback); - deque.addLast(batch); - incomplete.add(batch); - return new RecordAppendResult(deque.size() > 1 || batch.isClosed(), true, false); + PreAllocatedPagedOutputView outputView = new PreAllocatedPagedOutputView(segments); + int schemaId = tableInfo.getSchemaId(); + WriteFormat writeFormat = writeRecord.getWriteFormat(); + String originalPartitionName = + bucketAndWriteBatches.isHistoricalWriteTarget() + ? checkNotNull(physicalTablePath.getPartitionName()) + : null; + final WriteBatch batch = + createWriteBatch( + writeRecord, + bucketId, + tableInfo, + writeFormat, + physicalTablePath, + outputView, + schemaId, + originalPartitionName); + + batch.tryAppend(writeRecord, callback); + deque.addLast(batch); + incomplete.add(batch); + return new RecordAppendResult(deque.size() > 1 || batch.isClosed(), true, false); + } } private WriteBatch createWriteBatch( @@ -650,7 +752,8 @@ private WriteBatch createWriteBatch( WriteFormat writeFormat, PhysicalTablePath physicalTablePath, PreAllocatedPagedOutputView outputView, - int schemaId) { + int schemaId, + @Nullable String originalPartitionName) { // If the table is kv table we need to create a kv batch, otherwise we create a log batch. switch (writeFormat) { case COMPACTED_KV: @@ -665,6 +768,7 @@ private WriteBatch createWriteBatch( outputView, writeRecord.getTargetColumns(), writeRecord.getMergeMode(), + originalPartitionName, clock.milliseconds()); case ARROW_LOG: @@ -688,6 +792,7 @@ private WriteBatch createWriteBatch( tableInfo.getSchemaId(), arrowWriter, outputView, + originalPartitionName, clock.milliseconds(), statisticsCollector); @@ -699,6 +804,7 @@ private WriteBatch createWriteBatch( schemaId, outputView.getPreAllocatedSize(), outputView, + originalPartitionName, clock.milliseconds()); case INDEXED_LOG: @@ -709,6 +815,7 @@ private WriteBatch createWriteBatch( tableInfo.getSchemaId(), outputView.getPreAllocatedSize(), outputView, + originalPartitionName, clock.milliseconds()); default: @@ -1013,6 +1120,12 @@ private List getAllBucketsInCurrentNode(Integer currentNode, Clu List buckets = new ArrayList<>(); Set physicalTablePaths = cluster.getBucketLocationsByPath().keySet(); for (PhysicalTablePath path : physicalTablePaths) { + BucketAndWriteBatches bucketAndWriteBatches = writeBatches.get(path); + // A historical route uses the original path only as the accumulator queue key. Its + // actual bucket locations come from the historical target and are added below. + if (bucketAndWriteBatches != null && bucketAndWriteBatches.isHistoricalWriteTarget()) { + continue; + } List bucketsForTable = cluster.getAvailableBucketsForPhysicalTablePath(path); for (BucketLocation bucket : bucketsForTable) { @@ -1023,6 +1136,31 @@ private List getAllBucketsInCurrentNode(Integer currentNode, Clu } } } + + // Historical queues remain keyed by their original partition path. Add a location using + // that queue key while retaining the historical bucket as the RPC target. + for (Map.Entry entry : writeBatches.entrySet()) { + BucketAndWriteBatches bucketAndWriteBatches = entry.getValue(); + PhysicalTablePath originalPath = entry.getKey(); + if (!bucketAndWriteBatches.isHistoricalWriteTarget()) { + continue; + } + for (BucketLocation bucketLocation : + cluster.getAvailableBucketsForPhysicalTablePath( + bucketAndWriteBatches.targetPath)) { + if (bucketLocation.getLeader() != null + && Objects.equals(currentNode, bucketLocation.getLeader())) { + // Keep the original path so drain can find its original-keyed queue. The + // TableBucket, leader, and replicas still describe the historical RPC target. + buckets.add( + new BucketLocation( + originalPath, + bucketLocation.getTableBucket(), + bucketLocation.getLeader(), + bucketLocation.getReplicas())); + } + } + } return buckets; } @@ -1162,13 +1300,24 @@ public void destroyResources() { /** Per table bucket and write batches. */ private static class BucketAndWriteBatches { public final boolean isPartitionedTable; + /** The physical partition used for metadata lookup, leader discovery, and write RPCs. */ + private volatile PhysicalTablePath targetPath; + public volatile @Nullable Long partitionId; // Write batches for each bucket in queue. public final Map> batches = new CopyOnWriteMap<>(); - public BucketAndWriteBatches(@Nullable Long partitionId, boolean isPartitionedTable) { + private BucketAndWriteBatches( + @Nullable Long partitionId, + boolean isPartitionedTable, + PhysicalTablePath targetPath) { this.partitionId = partitionId; this.isPartitionedTable = isPartitionedTable; + this.targetPath = targetPath; + } + + public boolean isHistoricalWriteTarget() { + return HISTORICAL_PARTITION_VALUE.equals(targetPath.getPartitionName()); } } } diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java b/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java index b16780035f7..29600e2521b 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java @@ -45,6 +45,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nullable; import javax.annotation.concurrent.GuardedBy; import java.util.ArrayList; @@ -52,10 +53,12 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import static org.apache.fluss.client.utils.ClientRpcMessageUtils.makeProduceLogRequest; import static org.apache.fluss.client.utils.ClientRpcMessageUtils.makePutKvRequest; +import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; import static org.apache.fluss.utils.Preconditions.checkArgument; import static org.apache.fluss.utils.Preconditions.checkNotNull; @@ -228,8 +231,8 @@ private void sendWriteData() throws Exception { // TODO: this try-catch is not needed when we don't update metadata for // unready partitions Throwable t = ExceptionUtils.stripExecutionException(e); - if (t.getCause() instanceof PartitionNotExistException) { - // ignore this exception, this is probably happen because the partition + if (t instanceof PartitionNotExistException) { + abortIfHistoricalWriteTargetMissing(readyCheckResult.unknownLeaderTables); } else { throw e; } @@ -250,10 +253,8 @@ private void sendWriteData() throws Exception { // get the list of batches prepare to send. Map> batches = accumulator.drain(clusterSnapshot, readyNodes, maxRequestSize); - if (!batches.isEmpty()) { addToInflightBatches(batches); - // TODO add logic for batch expire. sendWriteRequests(batches); @@ -392,24 +393,79 @@ private void sendWriteRequest(int destination, short acks, List batches); } else { writeBatchByTable.forEach( - (tableId, writeBatches) -> { - if (isLogBatches(writeBatches)) { - sendProduceLogRequestAndHandleResponse( - gateway, - makeProduceLogRequest( - tableId, acks, maxRequestTimeoutMs, writeBatches), - tableId, - writeBatches); - } else { - sendPutKvRequestAndHandleResponse( - gateway, - makePutKvRequest( - tableId, acks, maxRequestTimeoutMs, writeBatches), - tableId, - writeBatches); - } - }); + (tableId, writeBatches) -> + sendWriteRequestsForTable(gateway, tableId, acks, writeBatches)); + } + } + + /** + * Sends normal and historical batches in separate requests. + * + *

Normal and historical writes cannot share a request. Both write protocols correlate + * historical responses by {@link TableBucket} and original partition name, so different + * original partitions targeting the same historical table bucket can remain in one request. + */ + private void sendWriteRequestsForTable( + TabletServerGateway gateway, + long tableId, + short acks, + List writeBatches) { + boolean logBatches = isLogBatches(writeBatches); + List normalBatches = new ArrayList<>(); + List historicalBatches = new ArrayList<>(); + + for (ReadyWriteBatch readyWriteBatch : writeBatches) { + if (readyWriteBatch.writeBatch().getOriginalPartitionName() == null) { + normalBatches.add(readyWriteBatch); + } else { + historicalBatches.add(readyWriteBatch); + } + } + + sendBatchesInRequest(gateway, tableId, acks, logBatches, normalBatches); + sendBatchesInRequest(gateway, tableId, acks, logBatches, historicalBatches); + } + + private void sendBatchesInRequest( + TabletServerGateway gateway, + long tableId, + short acks, + boolean logBatches, + List writeBatches) { + if (writeBatches.isEmpty()) { + return; + } + if (logBatches) { + sendProduceLogRequestAndHandleResponse( + gateway, + makeProduceLogRequest(tableId, acks, maxRequestTimeoutMs, writeBatches), + tableId, + writeBatches); + } else { + sendPutKvRequestAndHandleResponse( + gateway, + makePutKvRequest(tableId, acks, maxRequestTimeoutMs, writeBatches), + tableId, + writeBatches); + } + } + + private static Map toWriteBatchesByKey( + List writeBatches) { + Map writeBatchesByKey = new HashMap<>(); + for (ReadyWriteBatch readyWriteBatch : writeBatches) { + WriteBatch writeBatch = readyWriteBatch.writeBatch(); + WriteBatchKey key = + new WriteBatchKey( + readyWriteBatch.tableBucket(), writeBatch.getOriginalPartitionName()); + ReadyWriteBatch previous = writeBatchesByKey.put(key, readyWriteBatch); + checkArgument( + previous == null, + "A write request contains duplicate table bucket %s and original partition %s.", + readyWriteBatch.tableBucket(), + writeBatch.getOriginalPartitionName()); } + return writeBatchesByKey; } /** @@ -430,8 +486,7 @@ private void sendProduceLogRequestAndHandleResponse( ProduceLogRequest request, long tableId, List writeBatches) { - Map recordsByBucket = new HashMap<>(); - writeBatches.forEach(batch -> recordsByBucket.put(batch.tableBucket(), batch)); + Map writeBatchesByKey = toWriteBatchesByKey(writeBatches); long startTime = System.currentTimeMillis(); gateway.produceLog(request) .whenComplete( @@ -442,7 +497,7 @@ private void sendProduceLogRequestAndHandleResponse( handleWriteRequestException(e, writeBatches); } else { handleProduceLogResponse( - produceLogResponse, tableId, recordsByBucket); + produceLogResponse, tableId, writeBatchesByKey); } }); } @@ -452,8 +507,7 @@ private void sendPutKvRequestAndHandleResponse( PutKvRequest request, long tableId, List writeBatches) { - Map recordsByBucket = new HashMap<>(); - writeBatches.forEach(batch -> recordsByBucket.put(batch.tableBucket(), batch)); + Map writeBatchesByKey = toWriteBatchesByKey(writeBatches); long startTime = System.currentTimeMillis(); gateway.putKv(request) .whenComplete( @@ -463,7 +517,7 @@ private void sendPutKvRequestAndHandleResponse( if (e != null) { handleWriteRequestException(e, writeBatches); } else { - handlePutKvResponse(putKvResponse, tableId, recordsByBucket); + handlePutKvResponse(putKvResponse, tableId, writeBatchesByKey); } }); } @@ -471,7 +525,7 @@ private void sendPutKvRequestAndHandleResponse( private void handleProduceLogResponse( ProduceLogResponse response, long tableId, - Map recordsByBucket) { + Map writeBatchesByKey) { Set invalidMetadataTablesSet = new HashSet<>(); for (PbProduceLogRespForBucket logRespForBucket : response.getBucketsRespsList()) { TableBucket tb = @@ -481,7 +535,13 @@ private void handleProduceLogResponse( ? logRespForBucket.getPartitionId() : null, logRespForBucket.getBucketId()); - ReadyWriteBatch writeBatch = recordsByBucket.get(tb); + ReadyWriteBatch writeBatch = + writeBatchesByKey.get( + new WriteBatchKey( + tb, + logRespForBucket.hasOriginalPartitionName() + ? logRespForBucket.getOriginalPartitionName() + : null)); if (logRespForBucket.hasErrorCode()) { Set invalidMetadataTables = handleWriteBatchException( @@ -497,7 +557,7 @@ private void handleProduceLogResponse( private void handlePutKvResponse( PutKvResponse putKvResponse, long tableId, - Map recordsByBucket) { + Map writeBatchesByKey) { Set invalidMetadataTablesSet = new HashSet<>(); for (PbPutKvRespForBucket respForBucket : putKvResponse.getBucketsRespsList()) { TableBucket tb = @@ -511,7 +571,13 @@ private void handlePutKvResponse( accumulator.updateThrottle(tb, respForBucket.getPressure()); } - ReadyWriteBatch writeBatch = recordsByBucket.get(tb); + ReadyWriteBatch writeBatch = + writeBatchesByKey.get( + new WriteBatchKey( + tb, + respForBucket.hasOriginalPartitionName() + ? respForBucket.getOriginalPartitionName() + : null)); if (writeBatch == null) { continue; } @@ -549,6 +615,14 @@ private Set handleWriteBatchException( ReadyWriteBatch readyWriteBatch, ApiError error) { Set invalidMetadataTables = new HashSet<>(); WriteBatch writeBatch = readyWriteBatch.writeBatch(); + // Historical queues use the original path as their accumulator key, so capture the actual + // RPC target before any retry handling. + PhysicalTablePath requestTargetPath = + writeBatch.getOriginalPartitionName() == null + ? writeBatch.physicalTablePath() + : PhysicalTablePath.of( + writeBatch.physicalTablePath().getTablePath(), + HISTORICAL_PARTITION_VALUE); if (error.exception() instanceof StorageBackpressureException) { // Hard rejection: the storage engine reached its slowdown trigger and rejected the // write. Map it to full pressure (internal hard-rejection value 1.0f) so the bucket is @@ -617,7 +691,10 @@ private Set handleWriteBatchException( readyWriteBatch.tableBucket(), error.exception()); } - invalidMetadataTables.add(writeBatch.physicalTablePath()); + // A historical batch remains keyed by its original partition path in the + // accumulator, but its RPC is sent to the internal historical partition. Invalidate + // the actual RPC target so the retry refreshes the historical bucket metadata. + invalidMetadataTables.add(requestTargetPath); } } else { LOG.warn( @@ -632,6 +709,52 @@ private Set handleWriteBatchException( return invalidMetadataTables; } + /** + * Aborts pending writes when metadata confirms their target is missing. Such writes cannot make + * progress without a leader, and rerouting existing batches to the historical target is unsafe + * because their outcome and idempotent state may belong to the original target. + */ + private void abortIfHistoricalWriteTargetMissing(Set unknownLeaderTables) + throws Exception { + for (PhysicalTablePath targetPath : unknownLeaderTables) { + if (!accumulator.isHistoricalPartitionEnabled(targetPath)) { + continue; + } + try { + metadataUpdater.checkAndUpdatePartitionMetadata(targetPath); + } catch (Exception e) { + Throwable t = ExceptionUtils.stripExecutionException(e); + if (t instanceof PartitionNotExistException) { + // This target was considered usable when its batches were enqueued or first + // attempted, but is now confirmed missing. Transparently rerouting those + // batches to the historical partition is unsafe: an original-target attempt + // may have been accepted despite a lost response, and writer ID / batch + // sequence state cannot be reused across different physical TableBuckets. A + // safe failover must stop draining this path, wait for its in-flight requests, + // classify ambiguous outcomes, reset writer state, and preserve per-bucket + // ordering. This race requires partition retirement to overlap a writer that + // still holds the original route, so it is expected to be uncommon; fail + // closed for now. + // TODO: Implement safe in-flight historical failover if this path occurs + // frequently in practice. + // Retrying a historical-enabled table without a leader would leave its + // batches queued indefinitely. Fail only after checking the target itself so + // ordinary writes in the bulk metadata request keep their existing behavior. + PartitionNotExistException missingTargetException = + new PartitionNotExistException( + "Write target " + + targetPath + + " for a historical-partition-enabled table no " + + "longer exists according to refreshed metadata."); + missingTargetException.initCause(t); + maybeAbortBatches(missingTargetException); + return; + } + throw e; + } + } + } + private void updateWriterMetrics(Map> batches) { batches.values() .forEach( @@ -702,4 +825,35 @@ private void awaitNextReadyCheck(long delayMs) throws InterruptedException { void destroyResources() { accumulator.destroyResources(); } + + private static final class WriteBatchKey { + private final TableBucket tableBucket; + + // Distinguishes historical writes from different original partitions that share a target + // bucket. Normal writes have no original partition name. + private final @Nullable String originalPartitionName; + + private WriteBatchKey(TableBucket tableBucket, @Nullable String originalPartitionName) { + this.tableBucket = tableBucket; + this.originalPartitionName = originalPartitionName; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof WriteBatchKey)) { + return false; + } + WriteBatchKey that = (WriteBatchKey) o; + return tableBucket.equals(that.tableBucket) + && Objects.equals(originalPartitionName, that.originalPartitionName); + } + + @Override + public int hashCode() { + return Objects.hash(tableBucket, originalPartitionName); + } + } } diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/WriteBatch.java b/fluss-client/src/main/java/org/apache/fluss/client/write/WriteBatch.java index 754f9caa706..6c730bbb5e5 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/WriteBatch.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/WriteBatch.java @@ -54,6 +54,14 @@ public abstract class WriteBatch { protected final List callbacks = new ArrayList<>(); private final AtomicReference finalState = new AtomicReference<>(null); private final AtomicInteger attempts = new AtomicInteger(0); + /** + * The original partition name for a batch targeting the historical system partition. + * + *

It is null for a normal write and contains the logical partition namespace for a + * historical write. + */ + private final @Nullable String originalPartitionName; + protected boolean reopened; protected int recordCount; private long drainedMs; @@ -64,12 +72,14 @@ public WriteBatch( PhysicalTablePath physicalTablePath, int schemaId, WriteFormat writeFormat, + @Nullable String originalPartitionName, long createdMs) { this.physicalTablePath = physicalTablePath; this.createdMs = createdMs; this.tableId = tableId; this.schemaId = schemaId; this.writeFormat = checkNotNull(writeFormat, "write format must be not null"); + this.originalPartitionName = originalPartitionName; this.bucketId = bucketId; this.requestFuture = new RequestFuture(); this.recordCount = 0; @@ -193,6 +203,11 @@ public PhysicalTablePath physicalTablePath() { return physicalTablePath; } + /** Returns the original partition name for a historical write, or null for a normal write. */ + public @Nullable String getOriginalPartitionName() { + return originalPartitionName; + } + public RequestFuture getRequestFuture() { return requestFuture; } diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/WriterClient.java b/fluss-client/src/main/java/org/apache/fluss/client/write/WriterClient.java index ad8c7870547..e82bc57f8fb 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/write/WriterClient.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/write/WriterClient.java @@ -28,10 +28,12 @@ import org.apache.fluss.config.Configuration; import org.apache.fluss.exception.FlussRuntimeException; import org.apache.fluss.exception.IllegalConfigurationException; +import org.apache.fluss.exception.PartitionNotExistException; import org.apache.fluss.metadata.PhysicalTablePath; import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.rpc.gateway.TabletServerGateway; import org.apache.fluss.rpc.metrics.ClientMetricGroup; +import org.apache.fluss.utils.AutoPartitionStrategy; import org.apache.fluss.utils.CopyOnWriteMap; import org.apache.fluss.utils.clock.SystemClock; import org.apache.fluss.utils.concurrent.ExecutorThreadFactory; @@ -42,6 +44,9 @@ import javax.annotation.concurrent.ThreadSafe; import java.time.Duration; +import java.time.Instant; +import java.time.ZonedDateTime; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.concurrent.ExecutorService; @@ -51,6 +56,9 @@ import static org.apache.fluss.config.ConfigOptions.NoKeyAssigner.ROUND_ROBIN; import static org.apache.fluss.config.ConfigOptions.NoKeyAssigner.STICKY; import static org.apache.fluss.utils.ExceptionUtils.toException; +import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; +import static org.apache.fluss.utils.PartitionUtils.generateAutoPartitionTime; +import static org.apache.fluss.utils.PartitionUtils.isPastAutoPartition; /** * A client that write records to server. @@ -194,7 +202,12 @@ private void doSend(WriteRecord record, WriteCallback callback) { PhysicalTablePath physicalTablePath = record.getPhysicalTablePath(); // Skip the call entirely on non-partitioned tables; there is no partition to create. if (tableInfo.isPartitioned()) { - dynamicPartitionCreator.checkAndCreatePartitionAsync(physicalTablePath, tableInfo); + if (mayBeExpiredHistoricalPartition(physicalTablePath, tableInfo, Instant.now())) { + resolveHistoricalWriteTarget(physicalTablePath); + } else { + dynamicPartitionCreator.checkAndCreatePartitionAsync( + physicalTablePath, tableInfo); + } } // maybe create bucket assigner. @@ -240,6 +253,69 @@ private void doSend(WriteRecord record, WriteCallback callback) { } } + /** + * Returns whether a partition is old enough that it may have expired under its retention + * policy. + * + *

This client-side precheck uses the time zone resolved from the table configuration. A + * {@code true} result does not confirm that the partition is missing; the caller must refresh + * metadata before routing the write to a historical partition. If the table does not explicitly + * configure a time zone and the Client and Coordinator use different defaults, they may + * classify partitions near the retention boundary differently. Late classification may fail a + * write to an already removed original partition, while early classification only causes an + * extra metadata refresh. + */ + static boolean mayBeExpiredHistoricalPartition( + PhysicalTablePath physicalTablePath, TableInfo tableInfo, Instant now) { + String partitionName = physicalTablePath.getPartitionName(); + AutoPartitionStrategy strategy = tableInfo.getTableConfig().getAutoPartitionStrategy(); + if (partitionName == null + || !tableInfo.getTableConfig().isHistoricalPartitionEnabled() + || strategy.numToRetain() < 0) { + return false; + } + + if (!isPastAutoPartition(partitionName, strategy, now)) { + return false; + } + ZonedDateTime currentDateTime = + ZonedDateTime.ofInstant(now, strategy.timeZone().toZoneId()); + String earliestRetainedPartition = + generateAutoPartitionTime( + currentDateTime, -strategy.numToRetain(), strategy.timeUnit(), strategy); + return partitionName.compareTo(earliestRetainedPartition) < 0; + } + + private synchronized void resolveHistoricalWriteTarget(PhysicalTablePath originalPath) { + if (accumulator.hasHistoricalWriteTarget(originalPath)) { + return; + } + + // The time check only limits metadata traffic. Invalidate a potentially stale cached route + // and authoritatively choose the target before the record enters the queue. + metadataUpdater.invalidPhysicalTableBucketAndPartitionMeta( + Collections.singleton(originalPath)); + PhysicalTablePath targetPath = originalPath; + try { + if (!metadataUpdater.checkAndUpdatePartitionMetadata(originalPath)) { + throw new FlussRuntimeException( + "Failed to resolve write target for " + originalPath + '.'); + } + } catch (PartitionNotExistException ignored) { + targetPath = + PhysicalTablePath.of(originalPath.getTablePath(), HISTORICAL_PARTITION_VALUE); + // TODO: Activate this target only after Server retirement guarantees that all accepted + // original writes have been tiered to the lake. + if (!metadataUpdater.checkAndUpdatePartitionMetadata(targetPath)) { + throw new PartitionNotExistException( + "Historical partition " + targetPath + " does not exist."); + } + } + + accumulator.routeWritesTo( + originalPath, targetPath, metadataUpdater.getPartitionIdOrElseThrow(targetPath)); + } + private void maybeAbortBatches(Throwable t) { if (accumulator.hasIncomplete()) { LOG.error("Aborting all pending write batches due to fatal error", t); diff --git a/fluss-client/src/test/java/org/apache/fluss/client/utils/ClientRpcMessageUtilsTest.java b/fluss-client/src/test/java/org/apache/fluss/client/utils/ClientRpcMessageUtilsTest.java index 3ed17da7da5..153c3dc13f0 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/utils/ClientRpcMessageUtilsTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/utils/ClientRpcMessageUtilsTest.java @@ -140,6 +140,7 @@ private KvWriteBatch createKvWriteBatch(int bucketId, MergeMode mergeMode) throw outputView, null, mergeMode, + null, System.currentTimeMillis()); } } diff --git a/fluss-client/src/test/java/org/apache/fluss/client/write/ArrowLogWriteBatchTest.java b/fluss-client/src/test/java/org/apache/fluss/client/write/ArrowLogWriteBatchTest.java index f87f0c01a35..c9b98e1dfca 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/write/ArrowLogWriteBatchTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/write/ArrowLogWriteBatchTest.java @@ -137,6 +137,7 @@ void testAppendWithPreAllocatedMemorySegments() throws Exception { DATA1_ROW_TYPE, DEFAULT_COMPRESSION), new PreAllocatedPagedOutputView(memorySegmentList), + null, System.currentTimeMillis(), null); assertThat(arrowLogWriteBatch.pooledMemorySegments()).isEqualTo(memorySegmentList); @@ -213,6 +214,7 @@ void testArrowCompressionRatioEstimated() throws Exception { DATA1_TABLE_INFO.getSchemaId(), arrowWriter, new PreAllocatedPagedOutputView(memorySegmentList), + null, System.currentTimeMillis(), null); @@ -315,6 +317,7 @@ private ArrowLogWriteBatch createArrowLogWriteBatch(TableBucket tb, int maxSizeI DATA1_ROW_TYPE, DEFAULT_COMPRESSION), new UnmanagedPagedOutputView(128), + null, System.currentTimeMillis(), null); } diff --git a/fluss-client/src/test/java/org/apache/fluss/client/write/CompactedLogWriteBatchTest.java b/fluss-client/src/test/java/org/apache/fluss/client/write/CompactedLogWriteBatchTest.java index dc8fbce9d32..2461c2a7134 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/write/CompactedLogWriteBatchTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/write/CompactedLogWriteBatchTest.java @@ -254,6 +254,7 @@ private CompactedLogWriteBatch createLogWriteBatch( DATA1_TABLE_INFO.getSchemaId(), writeLimit, new PreAllocatedPagedOutputView(Collections.singletonList(memorySegment)), + null, System.currentTimeMillis()); } diff --git a/fluss-client/src/test/java/org/apache/fluss/client/write/IndexedLogWriteBatchTest.java b/fluss-client/src/test/java/org/apache/fluss/client/write/IndexedLogWriteBatchTest.java index 331be4209f6..a6269ad64b3 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/write/IndexedLogWriteBatchTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/write/IndexedLogWriteBatchTest.java @@ -216,6 +216,7 @@ private IndexedLogWriteBatch createLogWriteBatch( DATA1_TABLE_INFO.getSchemaId(), writeLimit, new PreAllocatedPagedOutputView(Collections.singletonList(memorySegment)), + null, System.currentTimeMillis()); } diff --git a/fluss-client/src/test/java/org/apache/fluss/client/write/KvWriteBatchTest.java b/fluss-client/src/test/java/org/apache/fluss/client/write/KvWriteBatchTest.java index 7b61976a038..b0faf06e679 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/write/KvWriteBatchTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/write/KvWriteBatchTest.java @@ -230,6 +230,7 @@ private KvWriteBatch createKvWriteBatch( outputView, null, MergeMode.DEFAULT, + null, System.currentTimeMillis()); } @@ -326,6 +327,7 @@ private KvWriteBatch createKvWriteBatchWithMergeMode(TableBucket tb, MergeMode m outputView, null, mergeMode, + null, System.currentTimeMillis()); } diff --git a/fluss-client/src/test/java/org/apache/fluss/client/write/SenderTest.java b/fluss-client/src/test/java/org/apache/fluss/client/write/SenderTest.java index 69d9c669c9e..c3228c37a69 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/write/SenderTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/write/SenderTest.java @@ -19,17 +19,23 @@ import org.apache.fluss.client.metadata.TestingMetadataUpdater; import org.apache.fluss.client.metrics.TestingWriterMetricGroup; +import org.apache.fluss.cluster.BucketLocation; import org.apache.fluss.cluster.Cluster; import org.apache.fluss.cluster.ServerNode; +import org.apache.fluss.config.AutoPartitionTimeUnit; import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; import org.apache.fluss.config.MemorySize; import org.apache.fluss.exception.AuthorizationException; import org.apache.fluss.exception.NetworkException; +import org.apache.fluss.exception.PartitionNotExistException; import org.apache.fluss.exception.TableNotExistException; import org.apache.fluss.exception.TimeoutException; +import org.apache.fluss.metadata.DataLakeFormat; import org.apache.fluss.metadata.PhysicalTablePath; +import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.metadata.TablePath; import org.apache.fluss.record.MemoryLogRecords; @@ -44,7 +50,9 @@ import org.apache.fluss.rpc.messages.PutKvRequest; import org.apache.fluss.rpc.messages.PutKvResponse; import org.apache.fluss.rpc.protocol.Errors; +import org.apache.fluss.server.entity.ProduceLogDataForBucket; import org.apache.fluss.server.tablet.TestTabletServerGateway; +import org.apache.fluss.types.DataTypes; import org.apache.fluss.utils.clock.SystemClock; import org.junit.jupiter.api.AfterEach; @@ -52,10 +60,13 @@ import org.junit.jupiter.api.Test; import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Deque; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -76,12 +87,13 @@ import static org.apache.fluss.record.TestData.DATA2_TABLE_ID; import static org.apache.fluss.record.TestData.DEFAULT_REMOTE_DATA_DIR; import static org.apache.fluss.rpc.protocol.Errors.SCHEMA_NOT_EXIST; -import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getProduceLogData; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.makeProduceLogResponse; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.makePutKvResponse; +import static org.apache.fluss.server.utils.ServerRpcMessageUtils.toProduceLogDataForBuckets; import static org.apache.fluss.testutils.DataTestUtils.compactedRow; import static org.apache.fluss.testutils.DataTestUtils.row; import static org.apache.fluss.testutils.common.CommonTestUtils.retry; +import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; import static org.assertj.core.api.Assertions.assertThat; /** ITCase for {@link Sender}. */ @@ -114,6 +126,130 @@ public void teardown() throws Exception { sender.destroyResources(); } + @Test + void testPotentialExpirationUsesAutoPartitionTimeUnit() { + TableInfo tableInfo = createHistoricalTableInfo(AutoPartitionTimeUnit.HOUR, 48); + Instant now = Instant.parse("2026-08-24T00:00:00Z"); + + assertThat( + WriterClient.mayBeExpiredHistoricalPartition( + PhysicalTablePath.of(tableInfo.getTablePath(), "2026082123"), + tableInfo, + now)) + .isTrue(); + assertThat( + WriterClient.mayBeExpiredHistoricalPartition( + PhysicalTablePath.of(tableInfo.getTablePath(), "2026082200"), + tableInfo, + now)) + .isFalse(); + } + + @Test + void testFailsWriteAfterMetadataConfirmsPartitionMissing() throws Exception { + sender.destroyResources(); + TableInfo tableInfo = createHistoricalTableInfo(); + PhysicalTablePath originalPath = PhysicalTablePath.of(tableInfo.getTablePath(), "20990101"); + TableBucket originalBucket = new TableBucket(tableInfo.getTableId(), 21L, 0); + metadataUpdater = missingPartitionMetadataUpdater(tableInfo); + metadataUpdater.updateCluster( + partitionedCluster( + tableInfo, Collections.singletonMap(originalPath, originalBucket))); + sender = setupWithIdempotenceState(); + + CompletableFuture future = + appendKvRecord(tableInfo, originalPath, 1, metadataUpdater.getCluster()); + sender.runOnce(); + + TestTabletServerGateway gateway = node1Gateway(); + gateway.response( + 0, createPutKvResponse(originalBucket, Errors.UNKNOWN_TABLE_OR_BUCKET_EXCEPTION)); + assertThat(future).isNotDone(); + + sender.runOnce(); + assertThat(future.get()) + .isInstanceOf(PartitionNotExistException.class) + .hasMessageContaining(originalPath.toString()) + .hasCauseInstanceOf(PartitionNotExistException.class); + } + + @Test + void testNormalAndHistoricalPutRequests() throws Exception { + sender.destroyResources(); + TableInfo tableInfo = createHistoricalTableInfo(); + PhysicalTablePath activePath = PhysicalTablePath.of(tableInfo.getTablePath(), "20990101"); + PhysicalTablePath firstOriginalPath = + PhysicalTablePath.of(tableInfo.getTablePath(), "20000101"); + PhysicalTablePath secondOriginalPath = + PhysicalTablePath.of(tableInfo.getTablePath(), "20000102"); + PhysicalTablePath historicalPath = + PhysicalTablePath.of(tableInfo.getTablePath(), HISTORICAL_PARTITION_VALUE); + TableBucket activeBucket = new TableBucket(tableInfo.getTableId(), 21L, 0); + TableBucket historicalBucket = new TableBucket(tableInfo.getTableId(), 22L, 0); + + Map tableBucketsByPath = new HashMap<>(); + tableBucketsByPath.put(activePath, activeBucket); + tableBucketsByPath.put(historicalPath, historicalBucket); + metadataUpdater = + new TestingMetadataUpdater( + Collections.singletonMap(tableInfo.getTablePath(), tableInfo)); + metadataUpdater.updateCluster(partitionedCluster(tableInfo, tableBucketsByPath)); + sender = setupWithIdempotenceState(); + + CompletableFuture activeFuture = + appendKvRecord(tableInfo, activePath, 1, metadataUpdater.getCluster()); + accumulator.routeWritesTo( + firstOriginalPath, historicalPath, historicalBucket.getPartitionId()); + accumulator.routeWritesTo( + secondOriginalPath, historicalPath, historicalBucket.getPartitionId()); + CompletableFuture firstHistoricalFuture = + appendKvRecord(tableInfo, firstOriginalPath, 2, metadataUpdater.getCluster()); + CompletableFuture secondHistoricalFuture = + appendKvRecord(tableInfo, secondOriginalPath, 3, metadataUpdater.getCluster()); + + sender.runOnce(); + + TestTabletServerGateway gateway = node1Gateway(); + assertThat(gateway.pendingRequestSize()).isEqualTo(2); + + PutKvRequest normalRequest = (PutKvRequest) gateway.getRequest(0); + assertThat(normalRequest.getBucketsReqsCount()).isOne(); + assertThat(normalRequest.getBucketsReqAt(0).getPartitionId()) + .isEqualTo(activeBucket.getPartitionId()); + assertThat(normalRequest.getBucketsReqAt(0).hasOriginalPartitionName()).isFalse(); + + PutKvRequest historicalRequest = (PutKvRequest) gateway.getRequest(1); + assertThat(historicalRequest.getBucketsReqsCount()).isEqualTo(2); + Set originalPartitionNames = new HashSet<>(); + for (int i = 0; i < historicalRequest.getBucketsReqsCount(); i++) { + assertThat(historicalRequest.getBucketsReqAt(i).getPartitionId()) + .isEqualTo(historicalBucket.getPartitionId()); + originalPartitionNames.add( + historicalRequest.getBucketsReqAt(i).getOriginalPartitionName()); + } + assertThat(originalPartitionNames) + .containsExactlyInAnyOrder( + firstOriginalPath.getPartitionName(), + secondOriginalPath.getPartitionName()); + + gateway.response(0, createPutKvResponse(activeBucket, 1L)); + gateway.response( + 0, + makePutKvResponse( + Arrays.asList( + PutKvResultForBucket.historicalSuccess( + historicalBucket, + 1L, + secondOriginalPath.getPartitionName()), + PutKvResultForBucket.historicalSuccess( + historicalBucket, + 1L, + firstOriginalPath.getPartitionName())))); + assertThat(activeFuture.get()).isNull(); + assertThat(firstHistoricalFuture.get()).isNull(); + assertThat(secondHistoricalFuture.get()).isNull(); + } + @Test void testSimple() throws Exception { long offset = 0; @@ -1124,6 +1260,111 @@ private void resetTableInfosWith(TableInfo tableInfo) { metadataUpdater.updateTableInfos(tableInfos); } + private static TableInfo createHistoricalTableInfo() { + return createHistoricalTableInfo(AutoPartitionTimeUnit.DAY, 7); + } + + private static TableInfo createHistoricalTableInfo( + AutoPartitionTimeUnit timeUnit, int numToRetain) { + Schema schema = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("dt", DataTypes.STRING()) + .primaryKey("id", "dt") + .build(); + TableDescriptor descriptor = + TableDescriptor.builder() + .schema(schema) + .partitionedBy("dt") + .distributedBy(1) + .property(ConfigOptions.TABLE_AUTO_PARTITION_ENABLED, true) + .property(ConfigOptions.TABLE_AUTO_PARTITION_TIME_UNIT, timeUnit) + .property(ConfigOptions.TABLE_AUTO_PARTITION_NUM_RETENTION, numToRetain) + .property(ConfigOptions.TABLE_AUTO_PARTITION_TIMEZONE, "UTC") + .property(ConfigOptions.TABLE_DATALAKE_ENABLED, true) + .property(ConfigOptions.TABLE_DATALAKE_FORMAT, DataLakeFormat.PAIMON) + .property(ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_ENABLED, true) + .build(); + return TableInfo.of( + DATA1_TABLE_PATH_PK, + DATA1_TABLE_ID_PK, + 1, + descriptor, + DEFAULT_REMOTE_DATA_DIR, + 1L, + 1L); + } + + private static TestingMetadataUpdater missingPartitionMetadataUpdater(TableInfo tableInfo) { + return new TestingMetadataUpdater( + Collections.singletonMap(tableInfo.getTablePath(), tableInfo)) { + @Override + public void updatePhysicalTableMetadata(Set physicalTablePaths) { + throw new PartitionNotExistException("Partition does not exist."); + } + + @Override + public boolean checkAndUpdatePartitionMetadata(PhysicalTablePath physicalTablePath) { + throw new PartitionNotExistException("Partition does not exist."); + } + }; + } + + private static Cluster partitionedCluster( + TableInfo tableInfo, Map tableBucketsByPath) { + int[] replicas = new int[] {TestingMetadataUpdater.NODE1.id()}; + Map> bucketLocationsByPath = new HashMap<>(); + Map partitionIdsByPath = new HashMap<>(); + tableBucketsByPath.forEach( + (physicalTablePath, tableBucket) -> { + bucketLocationsByPath.put( + physicalTablePath, + Collections.singletonList( + new BucketLocation( + physicalTablePath, + tableBucket, + TestingMetadataUpdater.NODE1.id(), + replicas))); + partitionIdsByPath.put(physicalTablePath, tableBucket.getPartitionId()); + }); + return new Cluster( + Collections.singletonMap( + TestingMetadataUpdater.NODE1.id(), TestingMetadataUpdater.NODE1), + TestingMetadataUpdater.COORDINATOR, + bucketLocationsByPath, + Collections.singletonMap(tableInfo.getTablePath(), tableInfo.getTableId()), + partitionIdsByPath); + } + + private CompletableFuture appendKvRecord( + TableInfo tableInfo, PhysicalTablePath physicalTablePath, int id, Cluster cluster) + throws Exception { + BinaryRow row = + compactedRow( + tableInfo.getRowType(), + new Object[] {id, physicalTablePath.getPartitionName()}); + byte[] key = + new CompactedKeyEncoder( + tableInfo.getRowType(), + tableInfo.getSchema().getPrimaryKeyIndexes()) + .encodeKey(row); + CompletableFuture future = new CompletableFuture<>(); + accumulator.append( + WriteRecord.forUpsert( + tableInfo, + physicalTablePath, + row, + key, + key, + WriteFormat.COMPACTED_KV, + null), + (tableBucket, logEndOffset, error) -> future.complete(error), + cluster, + 0, + false); + return future; + } + private void appendToAccumulator(TableBucket tb, GenericRow row, WriteCallback writeCallback) throws Exception { appendToAccumulator(DATA1_TABLE_INFO, tb, row, writeCallback); @@ -1169,6 +1410,11 @@ private void appendKvToAccumulator( false); } + private TestTabletServerGateway node1Gateway() { + return (TestTabletServerGateway) + metadataUpdater.newTabletServerClientForNode(TestingMetadataUpdater.NODE1.id()); + } + private ApiMessage getRequest(TableBucket tb, int index) { TestTabletServerGateway gateway = (TestTabletServerGateway) @@ -1309,14 +1555,24 @@ private IdempotenceManager createIdempotenceManager(boolean idempotenceEnabled) } private static boolean hasIdempotentRecords(TableBucket tb, ProduceLogRequest request) { - MemoryLogRecords memoryLogRecords = getProduceLogData(request).get(tb); + MemoryLogRecords memoryLogRecords = getProduceLogRecords(request, tb); return memoryLogRecords.batchIterator().next().writerId() != NO_WRITER_ID; } private static void assertBatchSequenceEquals( TableBucket tb, ProduceLogRequest request, int expectedBatchSequence) { - MemoryLogRecords memoryLogRecords = getProduceLogData(request).get(tb); + MemoryLogRecords memoryLogRecords = getProduceLogRecords(request, tb); assertThat(memoryLogRecords.batchIterator().next().batchSequence()) .isEqualTo(expectedBatchSequence); } + + private static MemoryLogRecords getProduceLogRecords( + ProduceLogRequest request, TableBucket tableBucket) { + for (ProduceLogDataForBucket bucketData : toProduceLogDataForBuckets(request)) { + if (bucketData.tableBucket().equals(tableBucket)) { + return bucketData.records(); + } + } + throw new IllegalArgumentException("No records found for table bucket " + tableBucket); + } } diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java index 16301f4612f..c6502568a9c 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java @@ -439,6 +439,14 @@ public class ConfigOptions { .withDescription( "The duration after which an idle historical partition table lookuper is removed from the cache."); + public static final ConfigOption SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME = + key("server.historical-partition.kv-cleanup.idle-time") + .durationType() + .defaultValue(Duration.ofHours(3)) + .withDescription( + "The idle time after all local historical KV writes are tiered before the local state can be cleaned. " + + "Set to 0 to disable idle cleanup."); + public static final ConfigOption SERVER_DATA_DISK_WRITE_LIMIT_RATIO = key("server.data-disk.write-limit-ratio") .doubleType() @@ -1925,11 +1933,12 @@ public class ConfigOptions { .booleanType() .defaultValue(false) .withDescription( - "Whether to enable historical partition lookup for the table. " + "Whether to enable historical partition access for the table. " + "When enabled, the coordinator creates and retains a system partition " - + "for routing lookups of expired partitions to lake storage. " - + "Currently, this option only supports auto-partitioned Paimon primary " - + "key tables with a single partition key. Disabled by default. " + + "for routing writes to expired partitions and, for primary-key tables, " + + "lookups of expired partitions to lake storage. Currently, this option " + + "only supports auto-partitioned Paimon tables with a single partition " + + "key. Disabled by default. " + "After changing this option, restart existing lookup jobs that need " + "to look up historical partition data so that their clients load the " + "updated table configuration."); diff --git a/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java b/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java index ea046e3b11d..d1b41983768 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java @@ -129,7 +129,7 @@ public boolean isDataLakeEnabled() { return config.get(ConfigOptions.TABLE_DATALAKE_ENABLED); } - /** Whether historical partition lookup is enabled. */ + /** Whether historical partition access is enabled. */ public boolean isHistoricalPartitionEnabled() { return config.get(ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_ENABLED); } diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/enumerator/TieringSourceEnumerator.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/enumerator/TieringSourceEnumerator.java index f1bf98527af..eb27397c9c6 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/enumerator/TieringSourceEnumerator.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/enumerator/TieringSourceEnumerator.java @@ -158,7 +158,7 @@ public void start() { this.coordinatorGateway = GatewayClientProxy.createGatewayProxy( metadataUpdater::getCoordinatorServer, rpcClient, CoordinatorGateway.class); - this.splitGenerator = new TieringSplitGenerator(flussAdmin); + this.splitGenerator = new TieringSplitGenerator(flussAdmin, metadataUpdater); LOG.info("Starting register Tiering Service to Fluss Coordinator..."); try { diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/split/TieringSplitGenerator.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/split/TieringSplitGenerator.java index a4b2638f309..5dd0bad2374 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/split/TieringSplitGenerator.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/split/TieringSplitGenerator.java @@ -22,8 +22,10 @@ import org.apache.fluss.client.initializer.OffsetsInitializer.BucketOffsetsRetriever; import org.apache.fluss.client.metadata.KvSnapshots; import org.apache.fluss.client.metadata.LakeSnapshot; +import org.apache.fluss.client.metadata.MetadataUpdater; import org.apache.fluss.exception.LakeTableSnapshotNotExistException; import org.apache.fluss.metadata.PartitionInfo; +import org.apache.fluss.metadata.PhysicalTablePath; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.metadata.TablePath; @@ -36,6 +38,7 @@ import javax.annotation.Nullable; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; @@ -43,6 +46,7 @@ import java.util.stream.IntStream; import static org.apache.fluss.client.table.scanner.log.LogScanner.EARLIEST_OFFSET; +import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; import static org.apache.fluss.utils.Preconditions.checkState; /** A generator for lake splits. */ @@ -51,9 +55,11 @@ public class TieringSplitGenerator { private static final Logger LOG = LoggerFactory.getLogger(TieringSplitGenerator.class); private final Admin flussAdmin; + private final MetadataUpdater metadataUpdater; - public TieringSplitGenerator(Admin flussAdmin) { + public TieringSplitGenerator(Admin flussAdmin, MetadataUpdater metadataUpdater) { this.flussAdmin = flussAdmin; + this.metadataUpdater = metadataUpdater; } public List generateTableSplits(TableInfo tableInfo) throws Exception { @@ -92,6 +98,21 @@ public List generateTableSplits(TableInfo tableInfo) throws Except Collectors.toMap( PartitionInfo::getPartitionId, PartitionInfo::getPartitionName)); + if (tableInfo.getTableConfig().isHistoricalPartitionEnabled()) { + // The internal historical partition is intentionally omitted from + // listPartitionInfos(), but tiering must consume it to synchronize historical + // writes to the lake table. Resolve it explicitly and include it in the splits. + PhysicalTablePath historicalPath = + PhysicalTablePath.of(tablePath, HISTORICAL_PARTITION_VALUE); + // Partition metadata is decoded using the tableId-to-path mapping already present + // in the Cluster, so initialize the table metadata before requesting the internal + // partition directly. + metadataUpdater.checkAndUpdateTableMetadata(Collections.singleton(tablePath)); + metadataUpdater.checkAndUpdatePartitionMetadata(historicalPath); + partitionNameById.put( + metadataUpdater.getPartitionIdOrElseThrow(historicalPath), + HISTORICAL_PARTITION_VALUE); + } return generatePartitionTableSplit( tableInfo, partitionNameById, bucketOffsetsRetriever, lakeSnapshotInfo); @@ -112,6 +133,7 @@ private List generatePartitionTableSplit( for (Map.Entry partitionNameByIdEntry : partitionNameById.entrySet()) { long partitionId = partitionNameByIdEntry.getKey(); String partitionName = partitionNameByIdEntry.getValue(); + boolean historicalPartition = HISTORICAL_PARTITION_VALUE.equals(partitionName); Map latestBucketsOffset = bucketOffsetsRetriever.latestOffsets( partitionName, @@ -120,21 +142,31 @@ private List generatePartitionTableSplit( .collect(Collectors.toList())); KvSnapshots latestKvSnapshots = null; if (tableInfo.hasPrimaryKey()) { - // get the table partition latest kv snapshot info - try { + if (historicalPartition) { + // Historical KV replicas use the lake snapshot as their durable base and tier + // only the retained WAL, so they have no local KV snapshots to tier. latestKvSnapshots = - flussAdmin - .getLatestKvSnapshots(tableInfo.getTablePath(), partitionName) - .get(); - } catch (Exception e) { - throw new FlinkRuntimeException( - String.format( - "Failed to get table snapshot for table %s and partition %s", - tableInfo.getTablePath(), partitionName), - ExceptionUtils.stripCompletionException(e)); + new KvSnapshots( + tableInfo.getTableId(), + partitionId, + Collections.emptyMap(), + Collections.emptyMap()); + } else { + try { + latestKvSnapshots = + flussAdmin + .getLatestKvSnapshots( + tableInfo.getTablePath(), partitionName) + .get(); + } catch (Exception e) { + throw new FlinkRuntimeException( + String.format( + "Failed to get table snapshot for table %s and partition %s", + tableInfo.getTablePath(), partitionName), + ExceptionUtils.stripCompletionException(e)); + } } } - splits.addAll( generateTableSplit( tableInfo, diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeCommitter.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeCommitter.java index c23cc373cda..7b16280eaf8 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeCommitter.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeCommitter.java @@ -35,6 +35,7 @@ import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.TableSnapshot; import org.apache.paimon.table.sink.CommitCallback; +import org.apache.paimon.table.sink.CommitMessage; import org.apache.paimon.table.sink.TableCommitImpl; import org.apache.paimon.utils.SnapshotManager; import org.slf4j.Logger; @@ -91,7 +92,9 @@ public PaimonCommittable toCommittable(List paimonWriteResult throws IOException { ManifestCommittable committable = new ManifestCommittable(COMMIT_IDENTIFIER); for (PaimonWriteResult paimonWriteResult : paimonWriteResults) { - committable.addFileCommittable(paimonWriteResult.commitMessage()); + for (CommitMessage commitMessage : paimonWriteResult.commitMessages()) { + committable.addFileCommittable(commitMessage); + } } return new PaimonCommittable(committable); } diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeWriter.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeWriter.java index 9b82403b6ce..a4bb0d44a41 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeWriter.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeWriter.java @@ -32,7 +32,6 @@ import org.apache.paimon.CoreOptions; import org.apache.paimon.catalog.Catalog; import org.apache.paimon.table.FileStoreTable; -import org.apache.paimon.table.sink.CommitMessage; import java.io.IOException; import java.util.Collections; @@ -40,6 +39,7 @@ import java.util.Map; import static org.apache.fluss.lake.paimon.utils.PaimonConversions.toPaimon; +import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; /** Implementation of {@link LakeWriter} for Paimon. */ public class PaimonLakeWriter implements LakeWriter, SupportsRecordBatchWrite { @@ -58,6 +58,8 @@ public PaimonLakeWriter( List partitionKeys = fileStoreTable.partitionKeys(); RowType flussRowType = writerInitContext.tableInfo().getRowType(); + boolean historicalPartition = + HISTORICAL_PARTITION_VALUE.equals(writerInitContext.partition()); // FIP-27: detect whether the target Paimon table is a clean table (only user columns) or a // legacy table (carrying the three Fluss system columns). Writers emit system columns only @@ -72,7 +74,8 @@ public PaimonLakeWriter( writerInitContext.partition(), partitionKeys, flussRowType, - paimonIncludingSystemColumns) + paimonIncludingSystemColumns, + historicalPartition) : new MergeTreeWriter( fileStoreTable, writerInitContext.tableBucket(), @@ -80,7 +83,8 @@ public PaimonLakeWriter( partitionKeys, flussRowType, writerInitContext.ioTmpDirs(), - paimonIncludingSystemColumns); + paimonIncludingSystemColumns, + historicalPartition); } @Override @@ -113,13 +117,11 @@ public void write(RecordBatch recordBatch) throws IOException { @Override public PaimonWriteResult complete() throws IOException { - CommitMessage commitMessage; try { - commitMessage = recordWriter.complete(); + return new PaimonWriteResult(recordWriter.complete()); } catch (Exception e) { throw new IOException("Failed to complete Paimon write.", e); } - return new PaimonWriteResult(commitMessage); } @Override diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonWriteResult.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonWriteResult.java index 70575c00e16..b2d40fbb25f 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonWriteResult.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonWriteResult.java @@ -20,19 +20,29 @@ import org.apache.paimon.table.sink.CommitMessage; import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; /** The write result of Paimon lake writer to pass to committer to commit. */ -public class PaimonWriteResult implements Serializable { +public final class PaimonWriteResult implements Serializable { private static final long serialVersionUID = 1L; - private final CommitMessage commitMessage; + private final List commitMessages; - public PaimonWriteResult(CommitMessage commitMessage) { - this.commitMessage = commitMessage; + /** Creates a write result containing all commit messages produced by one lake writer. */ + public PaimonWriteResult(List commitMessages) { + checkNotNull(commitMessages, "commitMessages must not be null"); + checkArgument(!commitMessages.isEmpty(), "commitMessages must not be empty"); + this.commitMessages = Collections.unmodifiableList(new ArrayList<>(commitMessages)); } - public CommitMessage commitMessage() { - return commitMessage; + /** Returns all commit messages produced by the lake writer. */ + public List commitMessages() { + return commitMessages; } } diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonWriteResultSerializer.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonWriteResultSerializer.java index 7efb3d37548..3ffecc9855d 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonWriteResultSerializer.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonWriteResultSerializer.java @@ -19,10 +19,14 @@ import org.apache.fluss.lake.serializer.SimpleVersionedSerializer; +import org.apache.paimon.io.DataInputViewStreamWrapper; +import org.apache.paimon.io.DataOutputSerializer; import org.apache.paimon.table.sink.CommitMessage; import org.apache.paimon.table.sink.CommitMessageSerializer; +import java.io.ByteArrayInputStream; import java.io.IOException; +import java.util.List; /** The {@link SimpleVersionedSerializer} for {@link PaimonWriteResult}. */ public class PaimonWriteResultSerializer implements SimpleVersionedSerializer { @@ -38,8 +42,9 @@ public int getVersion() { @Override public byte[] serialize(PaimonWriteResult paimonWriteResult) throws IOException { - CommitMessage commitMessage = paimonWriteResult.commitMessage(); - return messageSer.serialize(commitMessage); + DataOutputSerializer output = new DataOutputSerializer(64); + messageSer.serializeList(paimonWriteResult.commitMessages(), output); + return output.getCopyOfBuffer(); } @Override @@ -52,7 +57,11 @@ public PaimonWriteResult deserialize(int version, byte[] serialized) throws IOEx + version + "."); } - CommitMessage commitMessage = messageSer.deserialize(messageSer.getVersion(), serialized); - return new PaimonWriteResult(commitMessage); + try (DataInputViewStreamWrapper input = + new DataInputViewStreamWrapper(new ByteArrayInputStream(serialized))) { + List commitMessages = + messageSer.deserializeList(messageSer.getVersion(), input); + return new PaimonWriteResult(commitMessages); + } } } diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/RecordWriter.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/RecordWriter.java index 2260d553bc9..25d43d8400f 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/RecordWriter.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/RecordWriter.java @@ -31,6 +31,7 @@ import java.util.List; import static org.apache.fluss.lake.paimon.utils.PaimonConversions.toPaimonPartition; +import static org.apache.fluss.utils.Preconditions.checkNotNull; import static org.apache.fluss.utils.Preconditions.checkState; /** A base interface to write {@link LogRecord} to Paimon. */ @@ -40,7 +41,9 @@ public abstract class RecordWriter implements AutoCloseable { protected final RowType tableRowType; protected final int bucket; protected final List partitionKeys; - protected final BinaryRow partition; + protected final boolean historicalPartition; + // Null for historical writers, which derive the original partition from each record. + protected final @Nullable BinaryRow fixedPartition; protected final FlussRecordAsPaimonRow flussRecordAsPaimonRow; public RecordWriter( @@ -50,17 +53,21 @@ public RecordWriter( @Nullable String partition, List partitionKeys, org.apache.fluss.types.RowType flussRowType, - boolean paimonIncludingSystemColumns) { + boolean paimonIncludingSystemColumns, + boolean historicalPartition) { this.tableWrite = tableWrite; this.tableRowType = tableRowType; this.bucket = tableBucket.getBucket(); this.partitionKeys = partitionKeys; - if (partition == null || partitionKeys.isEmpty()) { + this.historicalPartition = historicalPartition; + if (historicalPartition) { + this.fixedPartition = null; + } else if (partition == null || partitionKeys.isEmpty()) { // non-partitioned table - this.partition = BinaryRow.EMPTY_ROW; + this.fixedPartition = BinaryRow.EMPTY_ROW; } else { // eagerly resolve BinaryRow partition from partition name string - this.partition = resolvePartition(partition, partitionKeys, flussRowType); + this.fixedPartition = resolvePartition(partition, partitionKeys, flussRowType); } this.flussRecordAsPaimonRow = new FlussRecordAsPaimonRow( @@ -69,19 +76,31 @@ public RecordWriter( public abstract void write(LogRecord record) throws Exception; - CommitMessage complete() throws Exception { + List complete() throws Exception { List commitMessages = tableWrite.prepareCommit(); - checkState( - commitMessages.size() == 1, - "The size of CommitMessage must be 1, but got %s.", - commitMessages); - return commitMessages.get(0); + // A normal writer targets one fixed partition, while a historical writer may write to + // multiple original partitions and therefore produce multiple commit messages. + if (!historicalPartition) { + checkState( + commitMessages.size() == 1, + "The size of CommitMessage must be 1, but got %s.", + commitMessages); + } + return commitMessages; } public void close() throws Exception { tableWrite.close(); } + /** Sets the current Fluss record and returns the Paimon partition it should be written to. */ + protected BinaryRow prepareRecordAndGetPartition(LogRecord record) { + flussRecordAsPaimonRow.setFlussRecord(record); + return historicalPartition + ? tableWrite.getPartition(flussRecordAsPaimonRow) + : checkNotNull(fixedPartition); + } + /** * Resolves a Paimon {@link BinaryRow} partition from the partition name string by parsing each * partition value to its typed Fluss representation, constructing a synthetic row, and diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyArrowBatchHelper.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyArrowBatchHelper.java index ac3b75cad3b..83518179ddf 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyArrowBatchHelper.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyArrowBatchHelper.java @@ -44,6 +44,8 @@ import java.util.ArrayList; import java.util.List; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + /** * Helper class that encapsulates Arrow-dependent batch writing logic for append-only tables. * @@ -106,7 +108,11 @@ class AppendOnlyArrowBatchHelper implements AutoCloseable { * system columns (__bucket, __offset, __timestamp) and uses Paimon's {@link ArrowBundleRecords} * for efficient batch writing. */ - void writeArrowBatch(ArrowBatchData arrowBatchData, BinaryRow partition) throws Exception { + void writeArrowBatch( + ArrowBatchData arrowBatchData, + @Nullable BinaryRow fixedPartition, + boolean historicalPartition) + throws Exception { int writtenBucket = bucket; if (fileStoreTable.store().bucketMode() == BucketMode.BUCKET_UNAWARE) { writtenBucket = 0; @@ -119,7 +125,7 @@ void writeArrowBatch(ArrowBatchData arrowBatchData, BinaryRow partition) throws // the Paimon table schema. Write it directly without enriching system columns. ArrowBundleRecords cleanRecords = new ArrowBundleRecords(originalRoot, tableRowType, CASE_SENSITIVE); - tableWrite.writeBundle(partition, writtenBucket, cleanRecords); + writeArrowBundle(cleanRecords, fixedPartition, historicalPartition, writtenBucket); return; } @@ -133,7 +139,25 @@ void writeArrowBatch(ArrowBatchData arrowBatchData, BinaryRow partition) throws ArrowBundleRecords arrowBundleRecords = new ArrowBundleRecords(enrichedRoot, tableRowType, CASE_SENSITIVE); - tableWrite.writeBundle(partition, writtenBucket, arrowBundleRecords); + writeArrowBundle(arrowBundleRecords, fixedPartition, historicalPartition, writtenBucket); + } + + private void writeArrowBundle( + ArrowBundleRecords arrowBundleRecords, + @Nullable BinaryRow fixedPartition, + boolean historicalPartition, + int writtenBucket) + throws Exception { + if (historicalPartition) { + // writeBundle accepts one fixed partition, but a historical batch may contain rows + // from multiple original partitions. + for (InternalRow row : arrowBundleRecords) { + BinaryRow partition = tableWrite.getPartition(row); + tableWrite.getWrite().write(partition, writtenBucket, row); + } + } else { + tableWrite.writeBundle(checkNotNull(fixedPartition), writtenBucket, arrowBundleRecords); + } } /** diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyWriter.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyWriter.java index 23f61a33171..a6d4f4292fd 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyWriter.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyWriter.java @@ -23,6 +23,7 @@ import org.apache.fluss.record.LogRecord; import org.apache.fluss.types.RowType; +import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.table.BucketMode; import org.apache.paimon.table.FileStoreTable; @@ -54,7 +55,8 @@ public AppendOnlyWriter( @Nullable String partition, List partitionKeys, RowType flussRowType, - boolean paimonIncludingSystemColumns) { + boolean paimonIncludingSystemColumns, + boolean historicalPartition) { //noinspection unchecked super( (TableWriteImpl) @@ -65,14 +67,15 @@ public AppendOnlyWriter( partition, partitionKeys, flussRowType, - paimonIncludingSystemColumns); + paimonIncludingSystemColumns, + historicalPartition); this.fileStoreTable = fileStoreTable; this.paimonIncludingSystemColumns = paimonIncludingSystemColumns; } @Override public void write(LogRecord record) throws Exception { - flussRecordAsPaimonRow.setFlussRecord(record); + BinaryRow targetPartition = prepareRecordAndGetPartition(record); // hacky, call internal method tableWrite.getWrite() to support // to write to given partition, otherwise, it'll always extract a partition from Paimon row @@ -82,7 +85,7 @@ public void write(LogRecord record) throws Exception { if (fileStoreTable.store().bucketMode() == BucketMode.BUCKET_UNAWARE) { writtenBucket = 0; } - tableWrite.getWrite().write(partition, writtenBucket, flussRecordAsPaimonRow); + tableWrite.getWrite().write(targetPartition, writtenBucket, flussRecordAsPaimonRow); } /** @@ -104,7 +107,7 @@ public void writeArrowBatch(ArrowBatchData arrowBatchData) throws Exception { } else { helper = (AppendOnlyArrowBatchHelper) arrowBatchHelper; } - helper.writeArrowBatch(arrowBatchData, partition); + helper.writeArrowBatch(arrowBatchData, fixedPartition, historicalPartition); } @Override diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/mergetree/MergeTreeWriter.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/mergetree/MergeTreeWriter.java index 37aeef7afe6..4dbc8100cb5 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/mergetree/MergeTreeWriter.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/mergetree/MergeTreeWriter.java @@ -23,6 +23,7 @@ import org.apache.fluss.types.RowType; import org.apache.paimon.KeyValue; +import org.apache.paimon.data.BinaryRow; import org.apache.paimon.disk.IOManager; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.sink.RowKeyExtractor; @@ -44,23 +45,6 @@ public class MergeTreeWriter extends RecordWriter { private final IOManager ioManager; - public MergeTreeWriter( - FileStoreTable fileStoreTable, - TableBucket tableBucket, - @Nullable String partition, - List partitionKeys, - RowType flussRowType, - boolean paimonIncludingSystemColumns) { - this( - fileStoreTable, - tableBucket, - partition, - partitionKeys, - flussRowType, - (String[]) null, - paimonIncludingSystemColumns); - } - public MergeTreeWriter( FileStoreTable fileStoreTable, TableBucket tableBucket, @@ -68,7 +52,8 @@ public MergeTreeWriter( List partitionKeys, RowType flussRowType, @Nullable String[] ioTmpDirs, - boolean paimonIncludingSystemColumns) { + boolean paimonIncludingSystemColumns, + boolean historicalPartition) { this( fileStoreTable, createIOManager(ioTmpDirs), @@ -76,7 +61,8 @@ public MergeTreeWriter( partition, partitionKeys, flussRowType, - paimonIncludingSystemColumns); + paimonIncludingSystemColumns, + historicalPartition); } MergeTreeWriter( @@ -86,7 +72,8 @@ public MergeTreeWriter( @Nullable String partition, List partitionKeys, RowType flussRowType, - boolean paimonIncludingSystemColumns) { + boolean paimonIncludingSystemColumns, + boolean historicalPartition) { super( createTableWrite(fileStoreTable, ioManager), fileStoreTable.rowType(), @@ -94,7 +81,8 @@ public MergeTreeWriter( partition, partitionKeys, flussRowType, - paimonIncludingSystemColumns); + paimonIncludingSystemColumns, + historicalPartition); this.rowKeyExtractor = fileStoreTable.createRowKeyExtractor(); this.ioManager = ioManager; } @@ -128,7 +116,7 @@ public void close() throws Exception { @Override public void write(LogRecord record) throws Exception { - flussRecordAsPaimonRow.setFlussRecord(record); + BinaryRow targetPartition = prepareRecordAndGetPartition(record); rowKeyExtractor.setRecord(flussRecordAsPaimonRow); keyValue.replace( @@ -139,6 +127,6 @@ public void write(LogRecord record) throws Exception { // hacky, call internal method tableWrite.getWrite() to support // to write to given partition, otherwise, it'll always extract a partition from Paimon row // which may be costly - tableWrite.getWrite().write(partition, bucket, keyValue); + tableWrite.getWrite().write(targetPartition, bucket, keyValue); } } diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/HistoricalPartitionLookupITCase.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/HistoricalPartitionITCase.java similarity index 60% rename from fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/HistoricalPartitionLookupITCase.java rename to fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/HistoricalPartitionITCase.java index 091c72bc21b..20b05d55f28 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/HistoricalPartitionLookupITCase.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/HistoricalPartitionITCase.java @@ -38,12 +38,15 @@ import org.apache.fluss.types.DataTypes; import org.apache.flink.core.execution.JobClient; +import org.apache.paimon.utils.CloseableIterator; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; import java.time.Duration; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -56,8 +59,8 @@ import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; import static org.assertj.core.api.Assertions.assertThat; -/** End-to-end IT case for looking up expired Fluss partitions from Paimon. */ -class HistoricalPartitionLookupITCase extends FlinkPaimonTieringTestBase { +/** End-to-end IT case for historical partition writes, tiering, recovery, and lookup. */ +class HistoricalPartitionITCase extends FlinkPaimonTieringTestBase { private static final String EXPIRED_PARTITION_NAME = "20240101"; private static final String SECOND_EXPIRED_PARTITION_NAME = "20240102"; @@ -76,6 +79,91 @@ protected static void beforeAll() { FlinkPaimonTieringTestBase.beforeAll(FLUSS_CLUSTER_EXTENSION.getClientConfig()); } + @Test + void testWriteAndTierHistoricalKvToPaimon() throws Exception { + TablePath tablePath = TablePath.of(DEFAULT_DB, "historical_write_tiering"); + Schema schema = partitionedPkSchema(true); + long tableId = + createTable( + tablePath, + partitionedDescriptor(schema, true, EXPIRED_PARTITION_RETENTION)); + + try { + long historicalPartitionId = waitUntilHistoricalPartitionReady(tablePath, tableId); + + InternalRow tieredRow = dataRow(true, 1, "unused", "Alice"); + assertThat(admin.listPartitionInfos(tablePath).get()) + .noneMatch(p -> EXPIRED_PARTITION_NAME.equals(p.getPartitionName())); + writeRows(tablePath, Collections.singletonList(tieredRow), false); + // Historical writes must not recreate the expired original partition. + assertThat(admin.listPartitionInfos(tablePath).get()) + .noneMatch(p -> EXPIRED_PARTITION_NAME.equals(p.getPartitionName())); + + TableBucket historicalBucket = new TableBucket(tableId, historicalPartitionId, 0); + assertThat(getLeaderReplica(historicalBucket).getLocalLogEndOffset()).isEqualTo(1); + tierAndVerifyPaimonRows( + tablePath, historicalBucket, 1L, "1|" + EXPIRED_PARTITION_NAME + "|Alice"); + + // Leave an untiered update after the Paimon snapshot. Restart recovery must apply this + // changelog over the tiered row. + InternalRow updatedRow = dataRow(true, 1, "unused", "Alice-updated"); + writeRows(tablePath, Collections.singletonList(updatedRow), false); + // FULL changelog emits UPDATE_BEFORE and UPDATE_AFTER for the update. + assertThat(getLeaderReplica(historicalBucket).getLocalLogEndOffset()).isEqualTo(3); + assertThat(getLeaderReplica(historicalBucket).getLakeLogEndOffset()).isEqualTo(1); + + restartLeaderAndVerifyLookup(tablePath, historicalBucket, schema, updatedRow); + + // Verify that the recovered local state can resolve the previous value for another + // update, and that a newly started tiering job can synchronize the resulting changelog. + InternalRow postRecoveryRow = dataRow(true, 1, "unused", "Alice-after-recovery"); + writeRows(tablePath, Collections.singletonList(postRecoveryRow), false); + assertThat(getLeaderReplica(historicalBucket).getLocalLogEndOffset()).isEqualTo(5); + assertThat(getLeaderReplica(historicalBucket).getLakeLogEndOffset()).isEqualTo(1); + tierAndVerifyPaimonRows( + tablePath, + historicalBucket, + 5L, + "1|" + EXPIRED_PARTITION_NAME + "|Alice-after-recovery"); + } finally { + dropTable(tablePath); + } + } + + @Test + void testWriteAndTierHistoricalLogToPaimon() throws Exception { + TablePath tablePath = TablePath.of(DEFAULT_DB, "historical_log_write_tiering"); + long tableId = + createTable( + tablePath, + partitionedDescriptor( + partitionedLogSchema(), true, EXPIRED_PARTITION_RETENTION)); + + try { + long historicalPartitionId = waitUntilHistoricalPartitionReady(tablePath, tableId); + List rows = + Arrays.asList( + row(1, EXPIRED_PARTITION_NAME, "Alice"), + row(2, EXPIRED_PARTITION_NAME, "Bob")); + + writeRows(tablePath, rows, true); + // Historical writes must not recreate the expired original partition. + assertThat(admin.listPartitionInfos(tablePath).get()) + .noneMatch(p -> EXPIRED_PARTITION_NAME.equals(p.getPartitionName())); + + TableBucket historicalBucket = new TableBucket(tableId, historicalPartitionId, 0); + assertThat(getLeaderReplica(historicalBucket).getLocalLogEndOffset()).isEqualTo(2); + tierAndVerifyPaimonRows( + tablePath, + historicalBucket, + 2L, + "1|" + EXPIRED_PARTITION_NAME + "|Alice", + "2|" + EXPIRED_PARTITION_NAME + "|Bob"); + } finally { + dropTable(tablePath); + } + } + @ParameterizedTest(name = "defaultBucketKey={0}") @ValueSource(booleans = {true, false}) void testLookupExpiredPartitionFromPaimon(boolean defaultBucketKey) throws Exception { @@ -86,7 +174,7 @@ void testLookupExpiredPartitionFromPaimon(boolean defaultBucketKey) throws Excep ? "historical_lookup_default_bucket" : "historical_lookup_bucket_subset"); Schema oldSchema = partitionedPkSchema(defaultBucketKey); - long tableId = createTable(tablePath, partitionedPkDescriptor(oldSchema)); + long tableId = createTable(tablePath, partitionedDescriptor(oldSchema, false)); // Enable historical lookup through ALTER TABLE to cover dynamic creation of the // coordinator-owned historical system partition. @@ -232,6 +320,76 @@ protected FlussClusterExtension getFlussClusterExtension() { return FLUSS_CLUSTER_EXTENSION; } + private static long waitUntilHistoricalPartitionReady(TablePath tablePath, long tableId) + throws Exception { + Optional historicalPartition = + FLUSS_CLUSTER_EXTENSION + .getZooKeeperClient() + .getPartition(tablePath, HISTORICAL_PARTITION_VALUE); + assertThat(historicalPartition).isPresent(); + long partitionId = historicalPartition.get().getPartitionId(); + FLUSS_CLUSTER_EXTENSION.waitUntilTablePartitionReady(tableId, partitionId); + return partitionId; + } + + private List readPaimonRows(TablePath tablePath) throws Exception { + List actualRows = new ArrayList<>(); + try (CloseableIterator rows = + getPaimonRowCloseableIterator(tablePath)) { + while (rows.hasNext()) { + org.apache.paimon.data.InternalRow row = rows.next(); + actualRows.add(row.getInt(0) + "|" + row.getString(1) + "|" + row.getString(2)); + } + } + return actualRows; + } + + private void tierAndVerifyPaimonRows( + TablePath tablePath, + TableBucket historicalBucket, + long expectedLogEndOffset, + String... expectedRows) + throws Exception { + JobClient jobClient = buildTieringJob(execEnv); + try { + assertReplicaStatus(historicalBucket, expectedLogEndOffset); + checkFlussOffsetsInSnapshot( + tablePath, Collections.singletonMap(historicalBucket, expectedLogEndOffset)); + assertThat(readPaimonRows(tablePath)).containsExactlyInAnyOrder(expectedRows); + } finally { + jobClient.cancel().get(); + } + } + + private void restartLeaderAndVerifyLookup( + TablePath tablePath, + TableBucket historicalBucket, + Schema schema, + InternalRow expectedRow) + throws Exception { + int tabletServerId = FLUSS_CLUSTER_EXTENSION.waitAndGetLeader(historicalBucket); + FLUSS_CLUSTER_EXTENSION.stopTabletServer(tabletServerId); + try { + FLUSS_CLUSTER_EXTENSION.startTabletServer(tabletServerId); + FLUSS_CLUSTER_EXTENSION.waitAndGetLeaderReplica(historicalBucket); + + try (Connection connection = ConnectionFactory.createConnection(clientConf); + Table table = connection.getTable(tablePath)) { + InternalRow actualRow = + table.newLookup() + .createLookuper() + .lookup(lookupKey(true, 1, "unused")) + .get() + .getSingletonRow(); + assertThatRow(actualRow).withSchema(schema.getRowType()).isEqualTo(expectedRow); + } + } finally { + if (FLUSS_CLUSTER_EXTENSION.getTabletServerById(tabletServerId) == null) { + FLUSS_CLUSTER_EXTENSION.startTabletServer(tabletServerId); + } + } + } + private static Schema partitionedPkSchema(boolean defaultBucketKey) { if (defaultBucketKey) { return Schema.newBuilder() @@ -250,6 +408,14 @@ private static Schema partitionedPkSchema(boolean defaultBucketKey) { .build(); } + private static Schema partitionedLogSchema() { + return Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("dt", DataTypes.STRING()) + .column("name", DataTypes.STRING()) + .build(); + } + private static Schema evolvedPartitionedPkSchema(boolean defaultBucketKey) { if (defaultBucketKey) { return Schema.newBuilder() @@ -270,23 +436,36 @@ private static Schema evolvedPartitionedPkSchema(boolean defaultBucketKey) { .build(); } - private static TableDescriptor partitionedPkDescriptor(Schema schema) { - return TableDescriptor.builder() - .schema(schema) - // This is the default bucket key for (id, dt), and a strict subset of the physical - // primary key for (id, sub_id, dt). - .distributedBy(1, "id") - .partitionedBy("dt") - .property(ConfigOptions.TABLE_AUTO_PARTITION_ENABLED, true) - .property(ConfigOptions.TABLE_AUTO_PARTITION_KEY, "dt") - .property(ConfigOptions.TABLE_AUTO_PARTITION_TIME_UNIT, AutoPartitionTimeUnit.DAY) - .property( - ConfigOptions.TABLE_AUTO_PARTITION_NUM_RETENTION, - INITIAL_PARTITION_RETENTION) - .property(ConfigOptions.TABLE_AUTO_PARTITION_TIMEZONE, "UTC") - .property(ConfigOptions.TABLE_DATALAKE_ENABLED, true) - .property(ConfigOptions.TABLE_DATALAKE_FRESHNESS, Duration.ofMillis(500)) - .build(); + private static TableDescriptor partitionedDescriptor( + Schema schema, boolean historicalPartitionEnabled) { + return partitionedDescriptor( + schema, historicalPartitionEnabled, INITIAL_PARTITION_RETENTION); + } + + private static TableDescriptor partitionedDescriptor( + Schema schema, boolean historicalPartitionEnabled, int partitionRetention) { + TableDescriptor.Builder builder = + TableDescriptor.builder() + .schema(schema) + // For primary-key tables, id is the default bucket key for (id, dt) and a + // strict subset of the physical primary key for (id, sub_id, dt). + .distributedBy(1, "id") + .partitionedBy("dt") + .property(ConfigOptions.TABLE_AUTO_PARTITION_ENABLED, true) + .property(ConfigOptions.TABLE_AUTO_PARTITION_KEY, "dt") + .property( + ConfigOptions.TABLE_AUTO_PARTITION_TIME_UNIT, + AutoPartitionTimeUnit.DAY) + .property( + ConfigOptions.TABLE_AUTO_PARTITION_NUM_RETENTION, + partitionRetention) + .property(ConfigOptions.TABLE_AUTO_PARTITION_TIMEZONE, "UTC") + .property(ConfigOptions.TABLE_DATALAKE_ENABLED, true) + .property(ConfigOptions.TABLE_DATALAKE_FRESHNESS, Duration.ofMillis(500)); + if (historicalPartitionEnabled) { + builder.property(ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_ENABLED, true); + } + return builder.build(); } private static InternalRow dataRow( diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringTest.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringTest.java index a433af06c82..19c59600f92 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringTest.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringTest.java @@ -19,23 +19,31 @@ import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; +import org.apache.fluss.lake.batch.ArrowRecordBatch; import org.apache.fluss.lake.committer.CommittedLakeSnapshot; import org.apache.fluss.lake.committer.CommitterInitContext; import org.apache.fluss.lake.committer.LakeCommitter; import org.apache.fluss.lake.serializer.SimpleVersionedSerializer; import org.apache.fluss.lake.writer.LakeWriter; +import org.apache.fluss.lake.writer.SupportsRecordBatchWrite; import org.apache.fluss.lake.writer.WriterInitContext; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.record.ArrowBatchData; import org.apache.fluss.record.ChangeType; import org.apache.fluss.record.GenericRecord; import org.apache.fluss.record.LogRecord; import org.apache.fluss.row.BinaryString; import org.apache.fluss.row.GenericRow; +import org.apache.fluss.utils.UnshadedArrowReadUtils; import org.apache.fluss.utils.types.Tuple2; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.paimon.CoreOptions; import org.apache.paimon.catalog.Catalog; import org.apache.paimon.catalog.CatalogContext; @@ -77,11 +85,13 @@ import static org.apache.fluss.metadata.TableDescriptor.BUCKET_COLUMN_NAME; import static org.apache.fluss.metadata.TableDescriptor.OFFSET_COLUMN_NAME; import static org.apache.fluss.metadata.TableDescriptor.TIMESTAMP_COLUMN_NAME; +import static org.apache.fluss.record.ChangeType.APPEND_ONLY; import static org.apache.fluss.record.ChangeType.DELETE; import static org.apache.fluss.record.ChangeType.INSERT; import static org.apache.fluss.record.ChangeType.UPDATE_AFTER; import static org.apache.fluss.record.ChangeType.UPDATE_BEFORE; import static org.apache.fluss.record.TestData.DEFAULT_REMOTE_DATA_DIR; +import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; import static org.assertj.core.api.Assertions.assertThat; /** The UT for tiering to Paimon via {@link PaimonLakeTieringFactory}. */ @@ -212,6 +222,72 @@ void testTieringWriteTable(boolean isPrimaryKeyTable, boolean isPartitioned) thr } } + @Test + void testHistoricalPrimaryKeyTiering() throws Exception { + TablePath tablePath = TablePath.of("paimon", "test_historical_primary_key"); + TableInfo tableInfo = createHistoricalTable(tablePath, true); + long timestamp = 1_000L; + List records = + Arrays.asList( + historicalRecord(0L, timestamp, 1, "20240101", INSERT), + historicalRecord(1L, timestamp, 1, "20240102", INSERT)); + + PaimonWriteResult writeResult; + try (LakeWriter lakeWriter = + createLakeWriter(tablePath, 0, HISTORICAL_PARTITION_VALUE, 1L, tableInfo)) { + for (LogRecord record : records) { + lakeWriter.write(record); + } + writeResult = lakeWriter.complete(); + } + + SimpleVersionedSerializer serializer = + paimonLakeTieringFactory.getWriteResultSerializer(); + writeResult = + serializer.deserialize(serializer.getVersion(), serializer.serialize(writeResult)); + assertHistoricalPartitions(writeResult); + + try (LakeCommitter committer = + createLakeCommitter(tablePath, tableInfo, new Configuration())) { + committer.commit( + committer.toCommittable(Collections.singletonList(writeResult)), + Collections.emptyMap()); + } + assertThat(paimonCatalog.listPartitions(toPaimon(tablePath))) + .extracting(partition -> partition.spec().get("c3")) + .containsExactlyInAnyOrder("20240101", "20240102"); + } + + @Test + void testHistoricalArrowBatchTiering() throws Exception { + TablePath tablePath = TablePath.of("paimon", "test_historical_arrow"); + TableInfo tableInfo = createHistoricalTable(tablePath, false); + long baseOffset = 10L; + long timestamp = 1_000L; + List records = + Arrays.asList( + historicalRecord(baseOffset, timestamp, 1, "20240101", APPEND_ONLY), + historicalRecord(baseOffset + 1, timestamp, 2, "20240102", APPEND_ONLY)); + + PaimonWriteResult writeResult; + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE); + LakeWriter lakeWriter = + createLakeWriter(tablePath, 0, HISTORICAL_PARTITION_VALUE, 1L, tableInfo)) { + VectorSchemaRoot root = + VectorSchemaRoot.create( + UnshadedArrowReadUtils.toArrowSchema(tableInfo.getRowType()), + allocator); + try (ArrowRecordBatch arrowRecordBatch = + new ArrowRecordBatch(new ArrowBatchData(root, baseOffset, timestamp, 1))) { + writeArrowRows(root, records); + ((SupportsRecordBatchWrite) lakeWriter).write(arrowRecordBatch); + } + writeResult = lakeWriter.complete(); + } + + assertHistoricalPartitions(writeResult); + } + @Test void testEmptyCommitCreatesSnapshot() throws Exception { TablePath tablePath = TablePath.of("paimon", "test_empty_commit"); @@ -593,6 +669,12 @@ private void verifyLogTableRecordsThreePartition( actualRecords.close(); } + private void assertHistoricalPartitions(PaimonWriteResult writeResult) { + assertThat(writeResult.commitMessages()) + .extracting(message -> message.partition().getString(0).toString()) + .containsExactlyInAnyOrder("20240101", "20240102"); + } + private void verifyTableRecords( CloseableIterator actualRecords, List expectRecords, @@ -737,6 +819,28 @@ private GenericRecord toRecord(long offset, GenericRow row, ChangeType changeTyp return new GenericRecord(offset, System.currentTimeMillis(), changeType, row); } + private LogRecord historicalRecord( + long offset, long timestamp, int key, String partition, ChangeType changeType) { + GenericRow row = + GenericRow.of( + key, BinaryString.fromString("value"), BinaryString.fromString(partition)); + return new GenericRecord(offset, timestamp, changeType, row); + } + + private void writeArrowRows(VectorSchemaRoot root, List records) { + root.allocateNew(); + IntVector keyVector = (IntVector) root.getVector("c1"); + VarCharVector valueVector = (VarCharVector) root.getVector("c2"); + VarCharVector partitionVector = (VarCharVector) root.getVector("c3"); + for (int i = 0; i < records.size(); i++) { + org.apache.fluss.row.InternalRow row = records.get(i).getRow(); + keyVector.setSafe(i, row.getInt(0)); + valueVector.setSafe(i, row.getString(1).toBytes()); + partitionVector.setSafe(i, row.getString(2).toBytes()); + } + root.setRowCount(records.size()); + } + private CloseableIterator getPaimonRows( TablePath tablePath, @Nullable String partition, boolean isPrimaryKeyTable, int bucket) throws Exception { @@ -911,6 +1015,33 @@ private void createTable( doCreatePaimonTable(tablePath, builder); } + private TableInfo createHistoricalTable(TablePath tablePath, boolean isPrimaryKeyTable) + throws Exception { + createTable( + tablePath, + isPrimaryKeyTable, + true, + isPrimaryKeyTable ? 1 : null, + Collections.emptyMap()); + + org.apache.fluss.metadata.Schema.Builder schemaBuilder = + org.apache.fluss.metadata.Schema.newBuilder() + .column("c1", org.apache.fluss.types.DataTypes.INT()) + .column("c2", org.apache.fluss.types.DataTypes.STRING()) + .column("c3", org.apache.fluss.types.DataTypes.STRING()); + if (isPrimaryKeyTable) { + schemaBuilder.primaryKey("c1", "c3"); + } + TableDescriptor descriptor = + TableDescriptor.builder() + .schema(schemaBuilder.build()) + .partitionedBy("c3") + .distributedBy(1) + .property(ConfigOptions.TABLE_DATALAKE_ENABLED, true) + .build(); + return TableInfo.of(tablePath, 0, 1, descriptor, DEFAULT_REMOTE_DATA_DIR, 1L, 1L); + } + private void createMultiPartitionTable(TablePath tablePath) throws Exception { Schema.Builder builder = Schema.newBuilder() diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyArrowBatchCaseSensitivityTest.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyArrowBatchCaseSensitivityTest.java index e3d961d6605..31b664e1e55 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyArrowBatchCaseSensitivityTest.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyArrowBatchCaseSensitivityTest.java @@ -28,6 +28,7 @@ import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.paimon.FileStore; +import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.io.BundleRecords; import org.apache.paimon.table.BucketMode; @@ -48,7 +49,6 @@ import static org.apache.fluss.metadata.TableDescriptor.TIMESTAMP_COLUMN_NAME; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -150,10 +150,10 @@ private Object[] writeAndRead( fileStoreTable, tableWrite, paimonRowType, 0, legacyTable); ArrowBatchData batch = new ArrowBatchData(root.slice(0, root.getRowCount()), 0L, 1L, 1)) { - helper.writeArrowBatch(batch, null); + helper.writeArrowBatch(batch, BinaryRow.EMPTY_ROW, false); ArgumentCaptor captor = ArgumentCaptor.forClass(BundleRecords.class); - verify(tableWrite).writeBundle(isNull(), eq(0), captor.capture()); + verify(tableWrite).writeBundle(eq(BinaryRow.EMPTY_ROW), eq(0), captor.capture()); Iterator rows = captor.getValue().iterator(); assertThat(rows.hasNext()).isTrue(); InternalRow row = rows.next(); diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/ProduceLogResultForBucket.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/ProduceLogResultForBucket.java index 37efad6c646..434e4c68f43 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/ProduceLogResultForBucket.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/ProduceLogResultForBucket.java @@ -23,35 +23,69 @@ import org.apache.fluss.rpc.protocol.ApiError; import org.apache.fluss.rpc.protocol.Errors; +import javax.annotation.Nullable; + +import java.util.Objects; + /** Result of {@link ProduceLogRequest} for each table bucket. */ @Internal public class ProduceLogResultForBucket extends WriteResultForBucket { private final long baseOffset; + // Identifies the original partition for a historical write; null for a normal write. + private final @Nullable String originalPartitionName; public ProduceLogResultForBucket(TableBucket tableBucket, long baseOffset, long endOffset) { - this(tableBucket, baseOffset, endOffset, ApiError.NONE); + this(tableBucket, baseOffset, endOffset, ApiError.NONE, null); } public ProduceLogResultForBucket(TableBucket tableBucket, ApiError error) { - this(tableBucket, -1L, -1L, error); + this(tableBucket, -1L, -1L, error, null); + } + + public static ProduceLogResultForBucket historicalSuccess( + TableBucket tableBucket, + long baseOffset, + long endOffset, + String originalPartitionName) { + return new ProduceLogResultForBucket( + tableBucket, baseOffset, endOffset, ApiError.NONE, originalPartitionName); + } + + public static ProduceLogResultForBucket historicalFailure( + TableBucket tableBucket, ApiError error, String originalPartitionName) { + return new ProduceLogResultForBucket(tableBucket, -1L, -1L, error, originalPartitionName); } private ProduceLogResultForBucket( - TableBucket tableBucket, long baseOffset, long endOffset, ApiError error) { + TableBucket tableBucket, + long baseOffset, + long endOffset, + ApiError error, + @Nullable String originalPartitionName) { super(tableBucket, endOffset, error); this.baseOffset = baseOffset; + this.originalPartitionName = originalPartitionName; } public long getBaseOffset() { return baseOffset; } + /** Returns the original partition name for a historical write, or null for a normal write. */ + public @Nullable String getOriginalPartitionName() { + return originalPartitionName; + } + @Override public T copy(Errors newError) { //noinspection unchecked return (T) new ProduceLogResultForBucket( - tableBucket, baseOffset, getWriteLogEndOffset(), newError.toApiError()); + tableBucket, + baseOffset, + getWriteLogEndOffset(), + newError.toApiError(), + originalPartitionName); } @Override @@ -66,6 +100,12 @@ public boolean equals(Object o) { return false; } ProduceLogResultForBucket that = (ProduceLogResultForBucket) o; - return baseOffset == that.baseOffset; + return baseOffset == that.baseOffset + && Objects.equals(originalPartitionName, that.originalPartitionName); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), baseOffset, originalPartitionName); } } diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/client/ServerConnection.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/client/ServerConnection.java index 46ec0806110..f28cf1e9b76 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/client/ServerConnection.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/client/ServerConnection.java @@ -25,11 +25,14 @@ import org.apache.fluss.exception.InvalidServerTypeException; import org.apache.fluss.exception.NetworkException; import org.apache.fluss.exception.RetriableAuthenticationException; +import org.apache.fluss.exception.UnsupportedVersionException; import org.apache.fluss.rpc.messages.ApiMessage; import org.apache.fluss.rpc.messages.ApiVersionsRequest; import org.apache.fluss.rpc.messages.ApiVersionsResponse; import org.apache.fluss.rpc.messages.AuthenticateRequest; import org.apache.fluss.rpc.messages.AuthenticateResponse; +import org.apache.fluss.rpc.messages.ProduceLogRequest; +import org.apache.fluss.rpc.messages.PutKvRequest; import org.apache.fluss.rpc.metrics.ClientMetricGroup; import org.apache.fluss.rpc.metrics.ConnectionMetrics; import org.apache.fluss.rpc.protocol.ApiKeys; @@ -62,12 +65,16 @@ import java.util.concurrent.TimeUnit; import java.util.function.BiConsumer; +import static org.apache.fluss.rpc.util.CommonRpcMessageUtils.hasHistoricalProduce; +import static org.apache.fluss.rpc.util.CommonRpcMessageUtils.hasHistoricalPut; import static org.apache.fluss.utils.IOUtils.closeQuietly; /** Connection to a Netty server used by the {@link NettyClient}. */ @ThreadSafe final class ServerConnection { private static final Logger LOG = LoggerFactory.getLogger(ServerConnection.class); + private static final short HISTORICAL_PRODUCE_LOG_MIN_VERSION = 1; + private static final short HISTORICAL_PUT_KV_MIN_VERSION = 3; private final ServerNode node; @@ -309,6 +316,7 @@ private CompletableFuture doSend( if (serverApiVersions != null) { try { version = serverApiVersions.highestAvailableVersion(apiKey); + validateVersionCompatibility(apiKey, version, rawRequest); } catch (Exception e) { responseFuture.completeExceptionally(e); return responseFuture; @@ -361,6 +369,39 @@ private CompletableFuture doSend( } } + private void validateVersionCompatibility( + ApiKeys apiKey, short version, ApiMessage rawRequest) { + if (apiKey == ApiKeys.PRODUCE_LOG && version < HISTORICAL_PRODUCE_LOG_MIN_VERSION) { + ProduceLogRequest produceLogRequest = (ProduceLogRequest) rawRequest; + if (hasHistoricalProduce(produceLogRequest)) { + throw new UnsupportedVersionException( + "Historical partition writes require PRODUCE_LOG version " + + HISTORICAL_PRODUCE_LOG_MIN_VERSION + + " or newer, but server " + + node + + " negotiated version " + + version + + '.'); + } + } + + if (apiKey != ApiKeys.PUT_KV || version >= HISTORICAL_PUT_KV_MIN_VERSION) { + return; + } + + PutKvRequest putKvRequest = (PutKvRequest) rawRequest; + if (hasHistoricalPut(putKvRequest)) { + throw new UnsupportedVersionException( + "Historical partition writes require PUT_KV version " + + HISTORICAL_PUT_KV_MIN_VERSION + + " or newer, but server " + + node + + " negotiated version " + + version + + '.'); + } + } + private void handleApiVersionsResponse(ApiMessage response, Throwable cause) { if (cause != null) { close(cause); diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/ApiKeys.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/ApiKeys.java index baf4256650e..ffdd1978de9 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/ApiKeys.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/ApiKeys.java @@ -42,7 +42,8 @@ public enum ApiKeys { GET_TABLE_SCHEMA(1011, 0, 0, PUBLIC), GET_METADATA(1012, 0, 0, PUBLIC), UPDATE_METADATA(1013, 0, 0, PRIVATE), - PRODUCE_LOG(1014, 0, 0, PUBLIC), + // Version 1: Supports original_partition_name in requests and responses for historical writes. + PRODUCE_LOG(1014, 0, 1, PUBLIC), FETCH_LOG(1015, 0, 0, PUBLIC), // Version 0: Uses lake's encoder for primary key encoding (legacy behavior). diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/util/CommonRpcMessageUtils.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/util/CommonRpcMessageUtils.java index 219b51087a4..befd3fc7f3e 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/util/CommonRpcMessageUtils.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/util/CommonRpcMessageUtils.java @@ -34,6 +34,7 @@ import org.apache.fluss.rpc.messages.PbPartitionSpec; import org.apache.fluss.rpc.messages.PbRemoteLogFetchInfo; import org.apache.fluss.rpc.messages.PbRemoteLogSegment; +import org.apache.fluss.rpc.messages.ProduceLogRequest; import org.apache.fluss.rpc.messages.PutKvRequest; import org.apache.fluss.rpc.protocol.ApiError; import org.apache.fluss.security.acl.AccessControlEntry; @@ -83,6 +84,17 @@ public static boolean hasHistoricalPut(PutKvRequest putKvRequest) { && putKvRequest.getBucketsReqAt(0).hasOriginalPartitionName(); } + /** + * Returns whether the produce-log request is for historical partition writes. + * + *

Normal and historical write buckets cannot be mixed in the same request, so the first + * bucket determines the request type. + */ + public static boolean hasHistoricalProduce(ProduceLogRequest produceLogRequest) { + return produceLogRequest.getBucketsReqsCount() > 0 + && produceLogRequest.getBucketsReqAt(0).hasOriginalPartitionName(); + } + public static List toPbAclInfos(Collection aclBindings) { return aclBindings.stream() .map(CommonRpcMessageUtils::toPbAclInfo) diff --git a/fluss-rpc/src/main/proto/FlussApi.proto b/fluss-rpc/src/main/proto/FlussApi.proto index 9d5e05fbdae..1ae6b1f126b 100644 --- a/fluss-rpc/src/main/proto/FlussApi.proto +++ b/fluss-rpc/src/main/proto/FlussApi.proto @@ -886,6 +886,8 @@ message PbProduceLogReqForBucket { optional int64 partition_id = 1; required int32 bucket_id = 2; required bytes records = 3; + // The original partition name for a historical write; unset for a normal write. + optional string original_partition_name = 4; } message PbProduceLogRespForBucket { @@ -894,6 +896,8 @@ message PbProduceLogRespForBucket { optional int32 error_code = 3; optional string error_message = 4; optional int64 base_offset = 5; + // The original partition name echoed from a historical write request; unset for a normal write. + optional string original_partition_name = 6; } message PbFetchLogReqForTable { diff --git a/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/client/ServerConnectionTest.java b/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/client/ServerConnectionTest.java index 4368872ec73..3b4e0596f3e 100644 --- a/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/client/ServerConnectionTest.java +++ b/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/client/ServerConnectionTest.java @@ -23,6 +23,7 @@ import org.apache.fluss.config.Configuration; import org.apache.fluss.exception.DisconnectException; import org.apache.fluss.exception.InvalidServerTypeException; +import org.apache.fluss.exception.UnsupportedVersionException; import org.apache.fluss.metrics.Gauge; import org.apache.fluss.metrics.Metric; import org.apache.fluss.metrics.MetricType; @@ -33,11 +34,18 @@ import org.apache.fluss.rpc.TestingGatewayService; import org.apache.fluss.rpc.TestingTabletGatewayService; import org.apache.fluss.rpc.messages.ApiMessage; +import org.apache.fluss.rpc.messages.ApiVersionsRequest; +import org.apache.fluss.rpc.messages.ApiVersionsResponse; import org.apache.fluss.rpc.messages.GetTableSchemaRequest; import org.apache.fluss.rpc.messages.ListDatabasesRequest; import org.apache.fluss.rpc.messages.LookupRequest; +import org.apache.fluss.rpc.messages.PbApiVersion; import org.apache.fluss.rpc.messages.PbLookupReqForBucket; import org.apache.fluss.rpc.messages.PbTablePath; +import org.apache.fluss.rpc.messages.ProduceLogRequest; +import org.apache.fluss.rpc.messages.ProduceLogResponse; +import org.apache.fluss.rpc.messages.PutKvRequest; +import org.apache.fluss.rpc.messages.PutKvResponse; import org.apache.fluss.rpc.metrics.ClientMetricGroup; import org.apache.fluss.rpc.metrics.TestingClientMetricGroup; import org.apache.fluss.rpc.netty.client.ServerConnection.ConnectionState; @@ -239,7 +247,75 @@ public ChannelFuture connect(String host, int port) { .isInstanceOf(DisconnectException.class); } + @Test + void testRejectHistoricalWritesForOldServer() throws Exception { + nettyServer.close(); + buildNettyServer(new OldWriteGatewayService()); + + ServerConnection connection = + new ServerConnection( + bootstrap, + serverNode, + TestingClientMetricGroup.newInstance(), + clientAuthenticator, + (con, ignore) -> {}); + try { + assertThat(connection.send(ApiKeys.PUT_KV, putKvRequest(null)).get()) + .isInstanceOf(PutKvResponse.class); + + assertThatThrownBy( + () -> + connection + .send(ApiKeys.PUT_KV, putKvRequest("dt=20260823")) + .get()) + .rootCause() + .isInstanceOf(UnsupportedVersionException.class) + .hasMessageContaining("require PUT_KV version 3 or newer") + .hasMessageContaining("negotiated version 2"); + + assertThat(connection.send(ApiKeys.PRODUCE_LOG, produceLogRequest(null)).get()) + .isInstanceOf(ProduceLogResponse.class); + + assertThatThrownBy( + () -> + connection + .send( + ApiKeys.PRODUCE_LOG, + produceLogRequest("dt=20260823")) + .get()) + .rootCause() + .isInstanceOf(UnsupportedVersionException.class) + .hasMessageContaining("require PRODUCE_LOG version 1 or newer") + .hasMessageContaining("negotiated version 0"); + } finally { + connection.close().get(); + } + } + + private static PutKvRequest putKvRequest(String originalPartitionName) { + PutKvRequest request = new PutKvRequest().setTableId(1L).setAcks(1).setTimeoutMs(10_000); + request.addBucketsReq().setBucketId(0).setRecords(new byte[0]); + if (originalPartitionName != null) { + request.getBucketsReqAt(0).setOriginalPartitionName(originalPartitionName); + } + return request; + } + + private static ProduceLogRequest produceLogRequest(String originalPartitionName) { + ProduceLogRequest request = + new ProduceLogRequest().setTableId(1L).setAcks(1).setTimeoutMs(10_000); + request.addBucketsReq().setBucketId(0).setRecords(new byte[0]); + if (originalPartitionName != null) { + request.getBucketsReqAt(0).setOriginalPartitionName(originalPartitionName); + } + return request; + } + private void buildNettyServer() throws Exception { + buildNettyServer(new TestingTabletGatewayService()); + } + + private void buildNettyServer(TestingGatewayService gatewayService) throws Exception { try (NetUtils.Port availablePort = getAvailablePort(); NetUtils.Port availablePort2 = getAvailablePort()) { serverNode = @@ -248,7 +324,7 @@ private void buildNettyServer() throws Exception { serverNode2 = new ServerNode( 2, "localhost", availablePort2.getPort(), ServerType.TABLET_SERVER); - service = new TestingTabletGatewayService(); + service = gatewayService; MetricGroup metricGroup = NOPMetricsGroup.newInstance(); nettyServer = new NettyServer( @@ -263,6 +339,34 @@ private void buildNettyServer() throws Exception { } } + private static class OldWriteGatewayService extends TestingTabletGatewayService { + @Override + public CompletableFuture apiVersions(ApiVersionsRequest request) { + return super.apiVersions(request) + .thenApply( + response -> { + for (PbApiVersion apiVersion : response.getApiVersionsList()) { + if (apiVersion.getApiKey() == ApiKeys.PUT_KV.id) { + apiVersion.setMaxVersion(2); + } else if (apiVersion.getApiKey() == ApiKeys.PRODUCE_LOG.id) { + apiVersion.setMaxVersion(0); + } + } + return response; + }); + } + + @Override + public CompletableFuture putKv(PutKvRequest request) { + return CompletableFuture.completedFuture(new PutKvResponse()); + } + + @Override + public CompletableFuture produceLog(ProduceLogRequest request) { + return CompletableFuture.completedFuture(new ProduceLogResponse()); + } + } + private static class MockMetricRegistry extends NOPMetricRegistry { Map registeredMetrics = new HashMap<>(); diff --git a/fluss-rust/crates/fluss/proto/FlussApi.proto b/fluss-rust/crates/fluss/proto/FlussApi.proto index 9d5e05fbdae..1ae6b1f126b 100644 --- a/fluss-rust/crates/fluss/proto/FlussApi.proto +++ b/fluss-rust/crates/fluss/proto/FlussApi.proto @@ -886,6 +886,8 @@ message PbProduceLogReqForBucket { optional int64 partition_id = 1; required int32 bucket_id = 2; required bytes records = 3; + // The original partition name for a historical write; unset for a normal write. + optional string original_partition_name = 4; } message PbProduceLogRespForBucket { @@ -894,6 +896,8 @@ message PbProduceLogRespForBucket { optional int32 error_code = 3; optional string error_message = 4; optional int64 base_offset = 5; + // The original partition name echoed from a historical write request; unset for a normal write. + optional string original_partition_name = 6; } message PbFetchLogReqForTable { diff --git a/fluss-rust/crates/fluss/src/proto/fluss.rs b/fluss-rust/crates/fluss/src/proto/fluss.rs index 95e7045c57e..e29dbaea874 100644 --- a/fluss-rust/crates/fluss/src/proto/fluss.rs +++ b/fluss-rust/crates/fluss/src/proto/fluss.rs @@ -1165,6 +1165,9 @@ pub struct PbProduceLogReqForBucket { pub bucket_id: i32, #[prost(bytes = "bytes", required, tag = "3")] pub records: ::prost::bytes::Bytes, + /// The original partition name for a historical write; unset for a normal write. + #[prost(string, optional, tag = "4")] + pub original_partition_name: ::core::option::Option<::prost::alloc::string::String>, } #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct PbProduceLogRespForBucket { @@ -1178,6 +1181,9 @@ pub struct PbProduceLogRespForBucket { pub error_message: ::core::option::Option<::prost::alloc::string::String>, #[prost(int64, optional, tag = "5")] pub base_offset: ::core::option::Option, + /// The original partition name echoed from a historical write request; unset for a normal write. + #[prost(string, optional, tag = "6")] + pub original_partition_name: ::core::option::Option<::prost::alloc::string::String>, } #[derive(Clone, PartialEq, ::prost::Message)] pub struct PbFetchLogReqForTable { diff --git a/fluss-rust/crates/fluss/src/rpc/api_key.rs b/fluss-rust/crates/fluss/src/rpc/api_key.rs index e9720a4e2b6..1341d1e6180 100644 --- a/fluss-rust/crates/fluss/src/rpc/api_key.rs +++ b/fluss-rust/crates/fluss/src/rpc/api_key.rs @@ -93,7 +93,6 @@ impl ApiKey { | ApiKey::TableExists | ApiKey::GetTableSchema | ApiKey::MetaData - | ApiKey::ProduceLog | ApiKey::FetchLog | ApiKey::ListOffsets | ApiKey::GetLatestKvSnapshots @@ -129,6 +128,8 @@ impl ApiKey { | ApiKey::GetClusterHealth | ApiKey::ListRemoteLogManifests | ApiKey::ListKvSnapshots => Some(ApiVersionRange::new(ApiVersion(0), ApiVersion(0))), + // ProduceLog v1 adds historical partition context to requests and responses. + ApiKey::ProduceLog => Some(ApiVersionRange::new(ApiVersion(0), ApiVersion(1))), // PutKv v2 adds the storage backpressure error code; v3 adds historical partition // context to requests and responses. ApiKey::PutKv => Some(ApiVersionRange::new(ApiVersion(0), ApiVersion(3))), diff --git a/fluss-rust/crates/fluss/src/rpc/message/produce_log.rs b/fluss-rust/crates/fluss/src/rpc/message/produce_log.rs index de9dce118d2..041e0adde01 100644 --- a/fluss-rust/crates/fluss/src/rpc/message/produce_log.rs +++ b/fluss-rust/crates/fluss/src/rpc/message/produce_log.rs @@ -49,6 +49,7 @@ impl ProduceLogRequest { partition_id: ready_batch.table_bucket.partition_id(), bucket_id: ready_batch.table_bucket.bucket_id(), records: ready_batch.write_batch.build()?, + original_partition_name: None, }) } diff --git a/fluss-rust/crates/fluss/src/rpc/server_connection.rs b/fluss-rust/crates/fluss/src/rpc/server_connection.rs index a8b36cbecc2..66825ab265a 100644 --- a/fluss-rust/crates/fluss/src/rpc/server_connection.rs +++ b/fluss-rust/crates/fluss/src/rpc/server_connection.rs @@ -1196,7 +1196,7 @@ mod tests { min_version: 0, max_version: 3, }, - // ProduceLog: server v0..v2, client v0 only → negotiated v0 + // ProduceLog: server v0..v2, client v0..v1 → negotiated v1 PbApiVersion { api_key: 1014, min_version: 0, @@ -1238,7 +1238,7 @@ mod tests { negotiated .highest_available_version(ApiKey::ProduceLog) .unwrap(), - ApiVersion(0) + ApiVersion(1) ); // Disjoint range → error diff --git a/fluss-server/src/main/java/org/apache/fluss/server/DynamicServerConfig.java b/fluss-server/src/main/java/org/apache/fluss/server/DynamicServerConfig.java index 98d721e152e..5f6e860dbe4 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/DynamicServerConfig.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/DynamicServerConfig.java @@ -56,6 +56,7 @@ import static org.apache.fluss.config.ConfigOptions.REMOTE_DATA_DIRS_WEIGHTS; import static org.apache.fluss.config.ConfigOptions.SERVER_DATA_DISK_WRITE_LIMIT_RATIO; import static org.apache.fluss.config.ConfigOptions.SERVER_DATA_DISK_WRITE_RECOVER_RATIO; +import static org.apache.fluss.config.ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME; import static org.apache.fluss.config.ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUPER_CACHE_EXPIRE_AFTER_ACCESS; import static org.apache.fluss.config.ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO; import static org.apache.fluss.config.ConfigOptions.SERVER_SASL_CREDENTIALS; @@ -84,6 +85,7 @@ class DynamicServerConfig { KV_SNAPSHOT_INTERVAL.key(), SERVER_DATA_DISK_WRITE_RECOVER_RATIO.key(), SERVER_DATA_DISK_WRITE_LIMIT_RATIO.key(), + SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME.key(), SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO.key(), SERVER_HISTORICAL_PARTITION_LOOKUPER_CACHE_EXPIRE_AFTER_ACCESS.key(), // Config options for remote.data.dirs diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/HistoricalLookupCacheConfigValidator.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/HistoricalLookupCacheConfigValidator.java index c9d763f410a..337b9542d45 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/HistoricalLookupCacheConfigValidator.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/HistoricalLookupCacheConfigValidator.java @@ -24,11 +24,20 @@ import java.time.Duration; -/** Validates dynamic historical lookup cache settings. */ +/** Validates dynamic historical partition settings used outside the coordinator. */ final class HistoricalLookupCacheConfigValidator implements ServerReconfigurable { @Override public void validate(Configuration newConfig) throws ConfigException { + Duration newCleanupIdleTime = + newConfig.get(ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME); + if (newCleanupIdleTime.isNegative()) { + throw new ConfigException( + String.format( + "Invalid configuration for %s, it must not be negative.", + ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME.key())); + } + double newMaxRatio = newConfig.get( ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/entity/ProduceLogDataForBucket.java b/fluss-server/src/main/java/org/apache/fluss/server/entity/ProduceLogDataForBucket.java new file mode 100644 index 00000000000..96232246152 --- /dev/null +++ b/fluss-server/src/main/java/org/apache/fluss/server/entity/ProduceLogDataForBucket.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.entity; + +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.record.MemoryLogRecords; + +import javax.annotation.Nullable; + +/** Log records for one normal or historical partition bucket. */ +public class ProduceLogDataForBucket { + private final TableBucket tableBucket; + private final MemoryLogRecords records; + // Identifies the original partition for a historical write; null for a normal write. + private final @Nullable String originalPartitionName; + + public ProduceLogDataForBucket( + TableBucket tableBucket, + MemoryLogRecords records, + @Nullable String originalPartitionName) { + this.tableBucket = tableBucket; + this.records = records; + this.originalPartitionName = originalPartitionName; + } + + public TableBucket tableBucket() { + return tableBucket; + } + + public MemoryLogRecords records() { + return records; + } + + public @Nullable String originalPartitionName() { + return originalPartitionName; + } +} diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java index ee28567756f..34f4cba9246 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java @@ -222,6 +222,11 @@ public final class Replica { private volatile @Nullable KvTablet kvTablet; private volatile @Nullable CloseableRegistry closeableRegistryForKv; private @Nullable PeriodicSnapshotManager kvSnapshotManager; + // The lake log end offset from which the current local historical KV state was rebuilt. + private volatile long historicalKvBaseOffset = -1L; + // Replaced whenever the local historical KV state is rebuilt so delayed cleanup tasks for an + // earlier state can be ignored. + private volatile @Nullable HistoricalKvCleanupState historicalKvCleanupState; /** * Server-wide {@link ScannerManager}. Active sessions for this bucket are closed in {@link @@ -327,8 +332,8 @@ public long logicalStorageLogSize() { public long logicalStorageKvSize() { if (isLeader() && isKvTable()) { if (isHistoricalPartition()) { - // Historical KV tablets do not create snapshots, so account for the local overlay - // using live SST files instead. + // Historical KV tablets do not create snapshots, so use live SST files to account + // for their local state instead. KvTablet currentKvTablet = kvTablet; return currentKvTablet == null ? 0L : currentKvTablet.liveSstFilesSize(); } @@ -380,6 +385,53 @@ public long getLakeLogEndOffset() { return logTablet.getLakeLogEndOffset(); } + /** Returns the cleanup state for the local historical KV state, or null if it is not ready. */ + public @Nullable HistoricalKvCleanupState getHistoricalKvCleanupState() { + return historicalKvCleanupState; + } + + /** + * Drops and recreates local historical KV state after its writes are fully tiered, provided + * leadership and offsets still match. + * + * @param expectedLeaderEpoch leader epoch required when cleanup runs + * @param tieredLogEndOffset fully tiered log end offset required when cleanup runs + * @param beforeCleanup action to run before the local KV state is dropped + * @return whether the local KV state was cleaned + */ + public boolean cleanupHistoricalKv( + int expectedLeaderEpoch, long tieredLogEndOffset, Runnable beforeCleanup) { + checkNotNull(beforeCleanup, "beforeCleanup must not be null"); + return inWriteLock( + leaderIsrUpdateLock, + () -> { + long localLogEndOffset = logTablet.localLogEndOffset(); + // Keep the scheduled snapshot and offset paired: newer lake progress may make + // the current lake and local offsets match while this task still references an + // older snapshot. + if (leaderEpoch != expectedLeaderEpoch + || localLogEndOffset != tieredLogEndOffset + || !isEligibleForHistoricalKvCleanup(localLogEndOffset)) { + return false; + } + + LOG.info( + "Cleaning local historical KV state for {} at log end offset {} " + + "covered by lake log end offset {}.", + tableBucket, + localLogEndOffset, + logTablet.getLakeLogEndOffset()); + // A lookup started after the empty local KV state is rebuilt must open a lake + // view that covers the data removed by this cleanup. + beforeCleanup.run(); + dropKv(); + // TODO: Retry rebuilding this historical bucket instead of waiting for + // failover or restart. + createKv(); + return true; + }); + } + public boolean isDataLakeEnabled() { return getTableConfig().isDataLakeEnabled(); } @@ -725,6 +777,18 @@ private void logTableConfigChanges(TableInfo oldTableInfo, TableInfo newTableInf } } + private boolean isEligibleForHistoricalKvCleanup(long localLogEndOffset) { + // Local KV state must have a known lake base, contain writes after that base, and have all + // those writes covered by lake before it can be discarded. + return isLeader() + && isHistoricalPartition() + && isKvTable() + && kvTablet != null + && historicalKvBaseOffset >= 0L + && historicalKvBaseOffset < localLogEndOffset + && logTablet.getLakeLogEndOffset() == localLogEndOffset; + } + private void createKv() { try { // create a closeable registry for the closable related to kv @@ -740,23 +804,36 @@ private void createKv() { // init kv tablet and get the snapshot it uses to init if have any Optional snapshotUsed = Optional.empty(); + Exception lastError = null; for (int i = 1; i <= INIT_KV_TABLET_MAX_RETRY_TIMES; i++) { try { snapshotUsed = initKvTablet(); + lastError = null; break; } catch (Exception e) { + lastError = e; LOG.warn( - "Fail to init kv tablet for bucket {}, retrying for {} times", + "Failed to init kv tablet for bucket {} on attempt {}/{}.", tableBucket, i, + INIT_KV_TABLET_MAX_RETRY_TIMES, e); } } - // A historical KV tablet is a disposable overlay over the lake snapshot. It is recovered - // by replaying WAL from the lake log end offset and does not create its own KV snapshots. + if (lastError != null) { + try { + dropKv(); + } catch (Exception cleanupError) { + lastError.addSuppressed(cleanupError); + } + throw new KvStorageException( + String.format( + "Failed to create KV tablet for bucket %s after %s attempts.", + tableBucket, INIT_KV_TABLET_MAX_RETRY_TIMES), + lastError); + } if (isHistoricalPartition()) { - // TODO: Clean up historical KV state after the corresponding WAL is fully tiered to - // lake storage. + historicalKvCleanupState = new HistoricalKvCleanupState(clock.milliseconds()); } else { startPeriodicKvSnapshot(snapshotUsed.orElse(null)); } @@ -777,6 +854,8 @@ private void dropKv() { kvManager.dropKv(tableBucket); kvTablet = null; } + historicalKvCleanupState = null; + historicalKvBaseOffset = -1L; } private void mayFlushKv(long newHighWatermark) { @@ -833,8 +912,8 @@ private Optional initKvTablet() { // get the offset from which, we should restore from. default is 0 long restoreStartOffset = isHistoricalPartition() ? historicalRecoveryStartOffset() : 0; - // The lake snapshot is the durable base for a historical overlay. Historical replicas - // therefore never restore a normal KV snapshot, even if one exists from older code. + // Lake is the durable base for local historical KV state. Historical replicas therefore + // never restore a normal KV snapshot, even if one exists from older code. Optional optCompletedSnapshot = isHistoricalPartition() ? Optional.empty() : getLatestSnapshot(tableBucket); try { @@ -898,6 +977,9 @@ private Optional initKvTablet() { logTablet.updateMinRetainOffset(restoreStartOffset); recoverKvTablet(restoreStartOffset, rowCount, autoIncIDRange); + if (isHistoricalPartition()) { + historicalKvBaseOffset = restoreStartOffset; + } } catch (Exception e) { throw new KvStorageException( String.format( @@ -1044,8 +1126,10 @@ private void recoverKvTablet( private long historicalRecoveryStartOffset() { long lakeLogEndOffset = logTablet.getLakeLogEndOffset(); + long localLogEndOffset = logTablet.localLogEndOffset(); long logStartOffset = logTablet.logStartOffset(); - long recoveryStartOffset = lakeLogEndOffset >= 0 ? lakeLogEndOffset : 0L; + long recoveryStartOffset = + lakeLogEndOffset >= 0 ? Math.min(lakeLogEndOffset, localLogEndOffset) : 0L; checkState( recoveryStartOffset >= logStartOffset, "Cannot recover historical KV state: recovery start offset %s is before the " @@ -1161,9 +1245,14 @@ public LogAppendInfo appendRecordsToLeader(MemoryLogRecords memoryLogRecords, in "Leader not local for bucket %s on tabletServer %d", tableBucket, localTabletServerId)); } - if (isHistoricalPartition()) { + // Historical primary-key writes must go through PUT_KV so the server can + // preserve the original partition namespace and consult the lake on a local + // miss. Append-only records already contain their partition columns, so a log + // table can append them directly to its historical system partition. + if (isHistoricalPartition() && isKvTable()) { throw new InvalidPartitionException( - "Normal write request must not target a historical partition."); + "Produce-log request must not target the historical partition of " + + "a primary-key table."); } validateInSyncReplicaSize(requiredAcks); @@ -1274,7 +1363,7 @@ public List findKeysRequiringLakeLookup( }); } - /** Writes records to the local historical KV overlay of the leader replica. */ + /** Writes records to the local historical KV state of the leader replica. */ public LogAppendInfo putHistoricalRecordsToLeader( KvRecordBatch kvRecords, @Nullable int[] targetColumns, @@ -1323,7 +1412,7 @@ private void validateHistoricalWrite(int expectedLeaderEpoch, int requiredAcks) validateInSyncReplicaSize(requiredAcks); } - /** Looks up keys from the local historical KV overlay of the leader replica. */ + /** Looks up keys from the local historical KV state of the leader replica. */ public List lookupHistoricalLocal( String originalPartitionName, List keys) throws Exception { return inReadLock( @@ -2546,4 +2635,80 @@ public SchemaGetter getSchemaGetter() { public PeriodicSnapshotManager getKvSnapshotManager() { return kvSnapshotManager; } + + /** Tracks write activity and cleanup conditions for the current local historical KV state. */ + @ThreadSafe + public static final class HistoricalKvCleanupState { + // Latched when the live SST size reaches the maximum. Rebuilding the local KV state resets + // this flag, while transient RocksDB size changes do not resume writes prematurely. + private final AtomicBoolean maxSizeReached = new AtomicBoolean(); + + private volatile @Nullable CleanupCandidate cleanupCandidate; + private volatile long lastWriteMs; + + private HistoricalKvCleanupState(long lastWriteMs) { + this.lastWriteMs = lastWriteMs; + } + + /** Returns whether historical writes are paused by the maximum-size limit. */ + public boolean maxSizeReached() { + return maxSizeReached.get(); + } + + /** Latches the maximum-size limit and returns whether this call changed the state. */ + public boolean markMaxSizeReached() { + return maxSizeReached.compareAndSet(false, true); + } + + /** Updates the cleanup candidate for the current local historical KV state. */ + public void updateCleanupCandidate( + long lakeSnapshotId, int expectedLeaderEpoch, long tieredLogEndOffset) { + cleanupCandidate = + new CleanupCandidate(lakeSnapshotId, expectedLeaderEpoch, tieredLogEndOffset); + } + + /** Returns the cleanup candidate, or null if none is available. */ + public @Nullable CleanupCandidate cleanupCandidate() { + return cleanupCandidate; + } + + /** Records the latest historical write activity time. */ + public void recordWrite(long timestampMs) { + lastWriteMs = timestampMs; + } + + /** Returns the latest historical write activity time. */ + public long lastWriteMs() { + return lastWriteMs; + } + + /** A candidate for cleaning up local historical KV state. */ + public static final class CleanupCandidate { + private final long lakeSnapshotId; + private final int expectedLeaderEpoch; + private final long tieredLogEndOffset; + + private CleanupCandidate( + long lakeSnapshotId, int expectedLeaderEpoch, long tieredLogEndOffset) { + this.lakeSnapshotId = lakeSnapshotId; + this.expectedLeaderEpoch = expectedLeaderEpoch; + this.tieredLogEndOffset = tieredLogEndOffset; + } + + /** Returns the lake snapshot ID that covers the local KV state. */ + public long lakeSnapshotId() { + return lakeSnapshotId; + } + + /** Returns the leader epoch required to run cleanup. */ + public int expectedLeaderEpoch() { + return expectedLeaderEpoch; + } + + /** Returns the log end offset covered by the lake snapshot. */ + public long tieredLogEndOffset() { + return tieredLogEndOffset; + } + } + } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java index b3357019230..37b47d3d4ed 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java @@ -82,6 +82,7 @@ import org.apache.fluss.server.entity.NotifyLeaderAndIsrData; import org.apache.fluss.server.entity.NotifyLeaderAndIsrResultForBucket; import org.apache.fluss.server.entity.NotifyRemoteLogOffsetsData; +import org.apache.fluss.server.entity.ProduceLogDataForBucket; import org.apache.fluss.server.entity.PutKvDataForBucket; import org.apache.fluss.server.entity.StopReplicaData; import org.apache.fluss.server.entity.StopReplicaResultForBucket; @@ -126,6 +127,7 @@ import org.apache.fluss.utils.FileUtils; import org.apache.fluss.utils.FlussPaths; import org.apache.fluss.utils.clock.Clock; +import org.apache.fluss.utils.concurrent.FutureUtils; import org.apache.fluss.utils.concurrent.Scheduler; import org.slf4j.Logger; @@ -373,7 +375,8 @@ public ReplicaManager( localDiskManager, dataDir, dataDirVolumeBytes, - scheduler); + scheduler, + clock); registerMetrics(); } @@ -417,6 +420,7 @@ public int getCoordinatorEpoch() { public void validate(Configuration newConfig) throws ConfigException { // Type validation is already handled by DynamicServerConfig. // Here we only do basic sanity checks. + historicalPartitionManager.validate(newConfig); int newMinInSyncReplicas = newConfig.get(ConfigOptions.LOG_REPLICA_MIN_IN_SYNC_REPLICAS_NUMBER); if (newMinInSyncReplicas <= 0) { @@ -688,6 +692,47 @@ public void appendRecordsToLog( timeoutMs, requiredAcks, entriesPerBucket.size(), appendResult, responseCallback); } + /** Appends historical log batches while preserving each original partition in the response. */ + public void appendHistoricalRecordsToLog( + int timeoutMs, + int requiredAcks, + Collection entriesPerBucket, + @Nullable UserContext userContext, + Consumer> responseCallback) { + List> resultFutures = + new ArrayList<>(entriesPerBucket.size()); + for (ProduceLogDataForBucket bucketData : entriesPerBucket) { + CompletableFuture resultFuture = new CompletableFuture<>(); + resultFutures.add(resultFuture); + String originalPartitionName = + checkNotNull( + bucketData.originalPartitionName(), + "originalPartitionName must not be null"); + appendRecordsToLog( + timeoutMs, + requiredAcks, + Collections.singletonMap(bucketData.tableBucket(), bucketData.records()), + userContext, + bucketResults -> { + ProduceLogResultForBucket result = bucketResults.get(0); + ProduceLogResultForBucket historicalResult = + result.failed() + ? ProduceLogResultForBucket.historicalFailure( + result.getTableBucket(), + result.getError(), + originalPartitionName) + : ProduceLogResultForBucket.historicalSuccess( + result.getTableBucket(), + result.getBaseOffset(), + result.getWriteLogEndOffset(), + originalPartitionName); + resultFuture.complete(historicalResult); + }); + } + FutureUtils.combineAll(resultFutures) + .thenAccept(results -> responseCallback.accept(new ArrayList<>(results))); + } + /** * Fetch records from a replica. Currently, we will return the fetched records immediately. * @@ -1350,7 +1395,8 @@ public void notifyLakeTableOffset( lakeBucketOffsets.entrySet()) { TableBucket tb = lakeBucketOffsetEntry.getKey(); LakeBucketOffset lakeBucketOffset = lakeBucketOffsetEntry.getValue(); - LogTablet logTablet = getReplicaOrException(tb).getLogTablet(); + Replica replica = getReplicaOrException(tb); + LogTablet logTablet = replica.getLogTablet(); logTablet.updateLakeTableSnapshotId(lakeBucketOffset.getSnapshotId()); lakeBucketOffset @@ -1359,7 +1405,19 @@ public void notifyLakeTableOffset( lakeBucketOffset .getLogEndOffset() - .ifPresent(logTablet::updateLakeLogEndOffset); + .ifPresent( + lakeLogEndOffset -> { + logTablet.updateLakeLogEndOffset(lakeLogEndOffset); + if (replica.isHistoricalPartition() + && replica.isKvTable()) { + // Only an explicit log-end-offset notification can + // make historical cleanup eligible. + historicalPartitionManager.onLakeProgress( + replica, + lakeBucketOffset.getSnapshotId(), + lakeLogEndOffset); + } + }); lakeBucketOffset .getMaxTimestamp() @@ -1425,22 +1483,22 @@ private void updateWithLakeTableSnapshot(Replica replica) throws Exception { LakeTableSnapshot lakeTableSnapshot = optLakeTableSnapshot.get(); long snapshotId = optLakeTableSnapshot.get().getSnapshotId(); replica.getLogTablet().updateLakeTableSnapshotId(snapshotId); + lakeTableSnapshot + .getLogEndOffset(tb) + .ifPresent(replica.getLogTablet()::updateLakeLogEndOffset); if (replica.isHistoricalPartition()) { - // The historical overlay will be rebuilt from this snapshot's lake offset. + // Local historical KV state will be rebuilt from this snapshot's lake offset. // Refresh a cached lookuper before it becomes the fallback for data omitted - // from the rebuilt overlay. + // from the rebuilt local state. historicalPartitionManager.requireLakeSnapshot( replica.getTableBucket().getTableId(), snapshotId); } - lakeTableSnapshot - .getLogEndOffset(tb) - .ifPresent(replica.getLogTablet()::updateLakeLogEndOffset); } } catch (Exception e) { if (replica.isHistoricalPartition()) { // Historical recovery uses the lake offset as its durable base and replays the // retained WAL from that offset. Reject leader activation if the latest lake - // progress cannot be loaded, instead of rebuilding the overlay from stale state. + // progress cannot be loaded, instead of rebuilding local KV from stale state. throw e; } // Lake commit cleanup can race with this best-effort refresh and remove the diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java index 13ec002e0df..93d720ae6fe 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java @@ -19,9 +19,12 @@ import org.apache.fluss.annotation.Internal; import org.apache.fluss.annotation.VisibleForTesting; +import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; +import org.apache.fluss.exception.ConfigException; import org.apache.fluss.exception.HistoricalPartitionThrottledException; import org.apache.fluss.exception.InvalidPartitionException; +import org.apache.fluss.exception.KvStorageException; import org.apache.fluss.lake.lakestorage.LakeTableLookuper; import org.apache.fluss.metadata.ResolvedPartitionSpec; import org.apache.fluss.metadata.TableBucket; @@ -40,14 +43,20 @@ import org.apache.fluss.server.kv.historical.HistoricalValueLookup; import org.apache.fluss.server.log.LogAppendInfo; import org.apache.fluss.server.replica.Replica; +import org.apache.fluss.server.replica.Replica.HistoricalKvCleanupState; import org.apache.fluss.server.storage.LocalDiskManager; import org.apache.fluss.utils.ByteArraySlice; import org.apache.fluss.utils.ByteArrayWrapper; +import org.apache.fluss.utils.clock.Clock; import org.apache.fluss.utils.concurrent.Scheduler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import javax.annotation.Nullable; import java.io.File; +import java.time.Duration; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -56,14 +65,24 @@ import java.util.Map; import java.util.concurrent.CompletableFuture; +import static org.apache.fluss.utils.Preconditions.checkArgument; import static org.apache.fluss.utils.Preconditions.checkNotNull; /** Coordinates lookup, write, and lifecycle operations for historical partitions. */ @Internal public final class HistoricalPartitionManager implements AutoCloseable { + private static final Logger LOG = LoggerFactory.getLogger(HistoricalPartitionManager.class); + private static final long MAX_HISTORICAL_KV_SIZE_BYTES = 5L * 1024 * 1024 * 1024; + private final HistoricalPartitionTaskExecutor taskExecutor; private final HistoricalLakeLookupManager lakeLookupManager; + private final Clock clock; + private final @Nullable Scheduler cleanupScheduler; + private final long maxHistoricalKvSizeBytes; + + private volatile long cleanupIdleTimeMs; + private volatile boolean closed; /** Creates a historical partition manager from the tablet server dependencies. */ public HistoricalPartitionManager( @@ -72,8 +91,10 @@ public HistoricalPartitionManager( LocalDiskManager localDiskManager, File dataDir, long dataDirVolumeBytes, - Scheduler scheduler) { + Scheduler scheduler, + Clock clock) { this( + conf, new HistoricalPartitionTaskExecutor(conf), new HistoricalLakeLookupManager( conf, @@ -81,16 +102,35 @@ public HistoricalPartitionManager( localDiskManager, dataDir, dataDirVolumeBytes, - scheduler)); + scheduler), + clock, + MAX_HISTORICAL_KV_SIZE_BYTES, + scheduler); } @VisibleForTesting HistoricalPartitionManager( + Configuration conf, HistoricalPartitionTaskExecutor taskExecutor, - HistoricalLakeLookupManager lakeLookupManager) { + HistoricalLakeLookupManager lakeLookupManager, + Clock clock, + long maxHistoricalKvSizeBytes, + @Nullable Scheduler cleanupScheduler) { + Duration cleanupIdleTime = + conf.get(ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME); + checkArgument( + !cleanupIdleTime.isNegative(), + "%s must not be negative.", + ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME.key()); + checkArgument( + maxHistoricalKvSizeBytes > 0L, "maxHistoricalKvSizeBytes must be greater than 0."); this.taskExecutor = checkNotNull(taskExecutor, "taskExecutor must not be null"); this.lakeLookupManager = checkNotNull(lakeLookupManager, "lakeLookupManager must not be null"); + this.clock = checkNotNull(clock, "clock must not be null"); + this.cleanupScheduler = cleanupScheduler; + this.cleanupIdleTimeMs = cleanupIdleTime.toMillis(); + this.maxHistoricalKvSizeBytes = maxHistoricalKvSizeBytes; } /** Starts the resources used by historical partition operations. */ @@ -98,7 +138,25 @@ public void startup(Scheduler scheduler) { lakeLookupManager.startup(scheduler); } - /** Looks up historical keys from the local overlay and then lake storage. */ + /** Records new lake progress and schedules any cleanup that it makes eligible. */ + public void onLakeProgress(Replica replica, long lakeSnapshotId, long lakeLogEndOffset) { + if (!replica.isLeader() || !replica.isKvTable()) { + return; + } + int expectedLeaderEpoch = replica.getLeaderEpoch(); + long localLogEndOffset = replica.getLocalLogEndOffset(); + if (lakeLogEndOffset != localLogEndOffset) { + return; + } + HistoricalKvCleanupState cleanupState = replica.getHistoricalKvCleanupState(); + if (cleanupState == null) { + return; + } + cleanupState.updateCleanupCandidate(lakeSnapshotId, expectedLeaderEpoch, lakeLogEndOffset); + tryScheduleCleanup(replica, cleanupState); + } + + /** Looks up historical keys from local KV state and then lake storage. */ public CompletableFuture lookup( Replica replica, LookupDataForBucket lookupData, @@ -119,7 +177,8 @@ public CompletableFuture lookup( + tableBucket + " (original partition " + lookupData.originalPartitionName() - + ").")))); + + ") because the historical request " + + "queue is full.")))); } catch (RuntimeException e) { return CompletableFuture.completedFuture( new LookupResultForBucket( @@ -129,7 +188,7 @@ public CompletableFuture lookup( } } - /** Writes records to the local overlay of a historical partition. */ + /** Writes records to the local KV state of a historical partition. */ public CompletableFuture putKv( Replica replica, PutKvDataForBucket putData, @@ -141,6 +200,26 @@ public CompletableFuture putKv( checkNotNull( putData.originalPartitionName(), "originalPartitionName must not be null"); + HistoricalKvCleanupState cleanupState = replica.getHistoricalKvCleanupState(); + if (cleanupState == null) { + throw new KvStorageException( + "Local historical KV state is not ready for " + + replica.getTableBucket() + + " because its KV tablet is being initialized or rebuilt."); + } + if (cleanupState.maxSizeReached()) { + return CompletableFuture.completedFuture( + maxSizeThrottledResult( + putData, originalPartitionName, maxHistoricalKvSizeBytes)); + } + long liveSstSize = replica.logicalStorageKvSize(); + if (liveSstSize >= maxHistoricalKvSizeBytes) { + markMaxSizeReached(replica, cleanupState); + return CompletableFuture.completedFuture( + maxSizeThrottledResult( + putData, originalPartitionName, maxHistoricalKvSizeBytes)); + } + cleanupState.recordWrite(clock.milliseconds()); return taskExecutor.submitOrdered( putData.tableBucket(), () -> { @@ -163,17 +242,7 @@ public CompletableFuture putKv( originalPartitionName); } }, - () -> - PutKvResultForBucket.historicalFailure( - putData.tableBucket(), - ApiError.fromThrowable( - new HistoricalPartitionThrottledException( - "Historical write is throttled for " - + putData.tableBucket() - + " (original partition " - + originalPartitionName - + ").")), - originalPartitionName)); + () -> requestLimitThrottledResult(putData, originalPartitionName)); } catch (RuntimeException e) { return CompletableFuture.completedFuture( PutKvResultForBucket.historicalFailure( @@ -183,9 +252,33 @@ public CompletableFuture putKv( } } - /** Applies dynamic historical lookup configuration changes. */ + /** Validates dynamic historical partition configuration changes. */ + public void validate(Configuration newConf) throws ConfigException { + Duration newCleanupIdleTime = + newConf.get(ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME); + if (newCleanupIdleTime.isNegative()) { + throw new ConfigException( + String.format( + "Invalid configuration for %s, it must not be negative.", + ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME.key())); + } + } + + /** Applies dynamic historical partition configuration changes. */ public void reconfigure(Configuration newConf) { lakeLookupManager.reconfigure(newConf); + long newCleanupIdleTimeMs = + newConf.get(ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME) + .toMillis(); + if (newCleanupIdleTimeMs == cleanupIdleTimeMs) { + return; + } + long oldCleanupIdleTimeMs = cleanupIdleTimeMs; + cleanupIdleTimeMs = newCleanupIdleTimeMs; + LOG.info( + "Historical KV cleanup idle time reconfigured: {} ms -> {} ms.", + oldCleanupIdleTimeMs, + newCleanupIdleTimeMs); } /** Invalidates the cached lake lookuper for the given table. */ @@ -286,8 +379,194 @@ LogAppendInfo processPut( requiredAcks); } + private void markMaxSizeReached(Replica replica, HistoricalKvCleanupState cleanupState) { + if (cleanupState.markMaxSizeReached()) { + LOG.warn( + "Pausing historical writes for {} because its live SST size reached the " + + "maximum size {} bytes.", + replica.getTableBucket(), + maxHistoricalKvSizeBytes); + + tryScheduleCleanup(replica, cleanupState); + } + } + + private void tryScheduleCleanup(Replica replica, HistoricalKvCleanupState cleanupState) { + HistoricalKvCleanupState.CleanupCandidate candidate = cleanupState.cleanupCandidate(); + if (candidate == null) { + return; + } + scheduleCleanup( + replica, + cleanupState, + cleanupState.maxSizeReached(), + candidate.lakeSnapshotId(), + candidate.expectedLeaderEpoch(), + candidate.tieredLogEndOffset()); + } + + private void scheduleCleanup( + Replica replica, + HistoricalKvCleanupState cleanupState, + boolean maxSizeReached, + long lakeSnapshotId, + int expectedLeaderEpoch, + long tieredLogEndOffset) { + // Reject cleanup if the replica no longer matches the cleanup state, leader epoch, or + // tiered log end offset used when it was scheduled. + if (closed + || replica.getHistoricalKvCleanupState() != cleanupState + || expectedLeaderEpoch != replica.getLeaderEpoch() + || replica.getLocalLogEndOffset() != tieredLogEndOffset) { + return; + } + + if (!maxSizeReached + && deferIdleCleanupIfNeeded( + replica, + cleanupState, + lakeSnapshotId, + expectedLeaderEpoch, + tieredLogEndOffset)) { + return; + } + + CompletableFuture cleanupFuture; + cleanupFuture = + taskExecutor.submitOrderedMaintenance( + replica.getTableBucket(), + () -> + runCleanup( + replica, + cleanupState, + maxSizeReached, + lakeSnapshotId, + expectedLeaderEpoch, + tieredLogEndOffset)); + cleanupFuture.whenComplete( + (ignored, error) -> { + if (error != null) { + LOG.error( + "Historical KV cleanup failed for {}.", + replica.getTableBucket(), + error); + } + }); + } + + private void runCleanup( + Replica replica, + HistoricalKvCleanupState cleanupState, + boolean maxSizeReached, + long lakeSnapshotId, + int expectedLeaderEpoch, + long tieredLogEndOffset) { + if (replica.getHistoricalKvCleanupState() != cleanupState + || expectedLeaderEpoch != replica.getLeaderEpoch()) { + return; + } + if (!maxSizeReached + && deferIdleCleanupIfNeeded( + replica, + cleanupState, + lakeSnapshotId, + expectedLeaderEpoch, + tieredLogEndOffset)) { + return; + } + + if (replica.cleanupHistoricalKv( + expectedLeaderEpoch, + tieredLogEndOffset, + () -> requireLakeSnapshot(replica.getTableBucket().getTableId(), lakeSnapshotId))) { + LOG.info( + "Cleaned {} local historical KV state for {}.", + maxSizeReached ? "max-size-triggered" : "idle-triggered", + replica.getTableBucket()); + } + } + + /** + * Returns whether idle cleanup must stop now. If the idle window has not elapsed, schedules the + * next check at its deadline. + */ + private boolean deferIdleCleanupIfNeeded( + Replica replica, + HistoricalKvCleanupState cleanupState, + long lakeSnapshotId, + int expectedLeaderEpoch, + long tieredLogEndOffset) { + long idleTimeMs = cleanupIdleTimeMs; + if (idleTimeMs <= 0L) { + return true; + } + long delayMs = remainingIdleCleanupDelayMs(cleanupState, idleTimeMs); + if (delayMs <= 0L) { + return false; + } + checkNotNull(cleanupScheduler, "cleanupScheduler must not be null") + .scheduleOnce( + "historical-kv-idle-cleanup-" + replica.getTableBucket(), + () -> + scheduleCleanup( + replica, + cleanupState, + false, + lakeSnapshotId, + expectedLeaderEpoch, + tieredLogEndOffset), + delayMs); + return true; + } + + /** Returns the remaining delay before idle cleanup is eligible, or {@code 0} if it is due. */ + private long remainingIdleCleanupDelayMs( + HistoricalKvCleanupState cleanupState, long idleTimeMs) { + long now = clock.milliseconds(); + long lastWriteMs = cleanupState.lastWriteMs(); + if (now < lastWriteMs) { + // A backward clock jump restarts the idle window instead of cleaning prematurely. + return idleTimeMs; + } + return Math.max(0L, idleTimeMs - (now - lastWriteMs)); + } + + private static PutKvResultForBucket requestLimitThrottledResult( + PutKvDataForBucket putData, String originalPartitionName) { + return PutKvResultForBucket.historicalFailure( + putData.tableBucket(), + ApiError.fromThrowable( + new HistoricalPartitionThrottledException( + "Historical write is throttled for " + + putData.tableBucket() + + " (original partition " + + originalPartitionName + + ") because the historical request queue is full.")), + originalPartitionName); + } + + private static PutKvResultForBucket maxSizeThrottledResult( + PutKvDataForBucket putData, String originalPartitionName, long maxHistoricalKvSize) { + return PutKvResultForBucket.historicalFailure( + putData.tableBucket(), + ApiError.fromThrowable( + new HistoricalPartitionThrottledException( + "Historical write is throttled for " + + putData.tableBucket() + + " (original partition " + + originalPartitionName + + ") because its local historical KV state reached the live " + + "SST maximum size of " + + maxHistoricalKvSize + + " bytes. New writes are paused until lake tiering " + + "covers all previously accepted writes and cleanup of " + + "the local historical KV state completes.")), + originalPartitionName); + } + @Override public void close() { + closed = true; taskExecutor.close(); lakeLookupManager.close(); } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionTaskExecutor.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionTaskExecutor.java index 9bf66848fad..763f2cd1807 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionTaskExecutor.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionTaskExecutor.java @@ -141,24 +141,47 @@ public CompletableFuture submitOrdered( return CompletableFuture.completedFuture(throttledResult.get()); } - CompletableFuture future; - CompletableFuture tail; try { - synchronized (orderedTasksLock) { - CompletableFuture previousTail = orderedTaskTails.get(orderingKey); - if (previousTail == null) { - future = CompletableFuture.supplyAsync(task, executor); - } else { - future = previousTail.thenApplyAsync(ignored -> task.get(), executor); - } - // Convert success or failure into a normal completion used only for sequencing. - tail = future.handle((ignored, error) -> null); - orderedTaskTails.put(orderingKey, tail); - } + return enqueueOrdered(orderingKey, task, true); } catch (RuntimeException e) { requestPermits.release(); throw e; } + } + + /** + * Submits an internal maintenance task after all accepted tasks with the same ordering key. + * + *

Maintenance work is not a client request and therefore does not consume a request permit. + */ + public CompletableFuture submitOrderedMaintenance( + Object orderingKey, Runnable maintenanceTask) { + checkNotNull(orderingKey, "orderingKey must not be null."); + checkNotNull(maintenanceTask, "maintenanceTask must not be null."); + return enqueueOrdered( + orderingKey, + () -> { + maintenanceTask.run(); + return null; + }, + false); + } + + private CompletableFuture enqueueOrdered( + Object orderingKey, Supplier task, boolean releaseRequestPermit) { + CompletableFuture future; + CompletableFuture tail; + synchronized (orderedTasksLock) { + CompletableFuture previousTail = orderedTaskTails.get(orderingKey); + if (previousTail == null) { + future = CompletableFuture.supplyAsync(task, executor); + } else { + future = previousTail.thenApplyAsync(ignored -> task.get(), executor); + } + // Convert success or failure into a normal completion used only for sequencing. + tail = future.handle((ignored, error) -> null); + orderedTaskTails.put(orderingKey, tail); + } CompletableFuture currentTail = tail; tail.whenComplete( @@ -167,17 +190,24 @@ public CompletableFuture submitOrdered( orderedTaskTails.remove(orderingKey, currentTail); } }); - return trackAcceptedRequest(future); + return trackAcceptedRequest(future, releaseRequestPermit); } private CompletableFuture trackAcceptedRequest(CompletableFuture future) { + return trackAcceptedRequest(future, true); + } + + private CompletableFuture trackAcceptedRequest( + CompletableFuture future, boolean releaseRequestPermit) { pendingRequests.add(future); future.whenComplete( (ignored, error) -> { - // Release the permit exactly once when the accepted task reaches a terminal - // state, including exceptional completion and cancellation. pendingRequests.remove(future); - requestPermits.release(); + if (releaseRequestPermit) { + // Release the permit exactly once when the accepted request reaches a + // terminal state, including exceptional completion and cancellation. + requestPermits.release(); + } }); return future; } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java b/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java index bd3ef49b35a..c3468b7a941 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java @@ -36,6 +36,7 @@ import org.apache.fluss.rpc.entity.FetchLogResultForBucket; import org.apache.fluss.rpc.entity.LookupResultForBucket; import org.apache.fluss.rpc.entity.PrefixLookupResultForBucket; +import org.apache.fluss.rpc.entity.ProduceLogResultForBucket; import org.apache.fluss.rpc.entity.ResultForBucket; import org.apache.fluss.rpc.gateway.CoordinatorGateway; import org.apache.fluss.rpc.gateway.TabletServerGateway; @@ -92,6 +93,7 @@ import org.apache.fluss.server.entity.NotifyLakeTableOffsetData; import org.apache.fluss.server.entity.NotifyLeaderAndIsrData; import org.apache.fluss.server.entity.NotifyRemoteLogOffsetsData; +import org.apache.fluss.server.entity.ProduceLogDataForBucket; import org.apache.fluss.server.entity.PutKvDataForBucket; import org.apache.fluss.server.entity.StopReplicaData; import org.apache.fluss.server.entity.UserContext; @@ -125,6 +127,7 @@ import java.util.stream.Collectors; import static org.apache.fluss.rpc.util.CommonRpcMessageUtils.hasHistoricalLookup; +import static org.apache.fluss.rpc.util.CommonRpcMessageUtils.hasHistoricalProduce; import static org.apache.fluss.rpc.util.CommonRpcMessageUtils.hasHistoricalPut; import static org.apache.fluss.security.acl.OperationType.DESCRIBE; import static org.apache.fluss.security.acl.OperationType.READ; @@ -137,7 +140,6 @@ import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getNotifyLeaderAndIsrRequestData; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getNotifyRemoteLogOffsetsData; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getNotifySnapshotOffsetData; -import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getProduceLogData; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getStopReplicaData; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getTableFilterInfoMap; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getTableStatsRequestData; @@ -157,6 +159,7 @@ import static org.apache.fluss.server.utils.ServerRpcMessageUtils.toHistoricalLookupData; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.toLookupData; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.toPrefixLookupData; +import static org.apache.fluss.server.utils.ServerRpcMessageUtils.toProduceLogDataForBuckets; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.toPutKvDataForBuckets; /** An RPC Gateway service for tablet server. */ @@ -216,13 +219,29 @@ public void shutdown() {} public CompletableFuture produceLog(ProduceLogRequest request) { authorizeTable(WRITE, request.getTableId()); CompletableFuture response = new CompletableFuture<>(); - Map produceLogData = getProduceLogData(request); - replicaManager.appendRecordsToLog( - request.getTimeoutMs(), - request.getAcks(), - produceLogData, - new UserContext(currentSession().getPrincipal()), - bucketResponseMap -> response.complete(makeProduceLogResponse(bucketResponseMap))); + List produceLogData = toProduceLogDataForBuckets(request); + UserContext userContext = new UserContext(currentSession().getPrincipal()); + Consumer> responseCallback = + results -> response.complete(makeProduceLogResponse(results)); + if (hasHistoricalProduce(request)) { + replicaManager.appendHistoricalRecordsToLog( + request.getTimeoutMs(), + request.getAcks(), + produceLogData, + userContext, + responseCallback); + } else { + Map recordsByBucket = new HashMap<>(); + for (ProduceLogDataForBucket bucketData : produceLogData) { + recordsByBucket.put(bucketData.tableBucket(), bucketData.records()); + } + replicaManager.appendRecordsToLog( + request.getTimeoutMs(), + request.getAcks(), + recordsByBucket, + userContext, + responseCallback); + } return response; } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java index 02297ead2dd..58a35b392b0 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java @@ -187,6 +187,7 @@ import org.apache.fluss.server.entity.NotifyLeaderAndIsrData; import org.apache.fluss.server.entity.NotifyLeaderAndIsrResultForBucket; import org.apache.fluss.server.entity.NotifyRemoteLogOffsetsData; +import org.apache.fluss.server.entity.ProduceLogDataForBucket; import org.apache.fluss.server.entity.PutKvDataForBucket; import org.apache.fluss.server.entity.StopReplicaData; import org.apache.fluss.server.entity.StopReplicaResultForBucket; @@ -231,6 +232,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.apache.fluss.rpc.util.CommonRpcMessageUtils.hasHistoricalProduce; import static org.apache.fluss.rpc.util.CommonRpcMessageUtils.toByteBuffer; import static org.apache.fluss.rpc.util.CommonRpcMessageUtils.toPbAclInfo; import static org.apache.fluss.utils.Preconditions.checkNotNull; @@ -892,12 +894,20 @@ public static StopReplicaResponse makeStopReplicaResponse( return stopReplicaResponse; } - public static Map getProduceLogData( + /** Converts produce-log requests while preserving their historical partition context. */ + public static List toProduceLogDataForBuckets( ProduceLogRequest produceRequest) { long tableId = produceRequest.getTableId(); - Map produceEntryData = new HashMap<>(); + List produceLogData = + new ArrayList<>(produceRequest.getBucketsReqsCount()); + Map> originalPartitionsByBucket = new HashMap<>(); + boolean historicalWriteRequest = hasHistoricalProduce(produceRequest); for (PbProduceLogReqForBucket produceLogReqForBucket : produceRequest.getBucketsReqsList()) { + if (produceLogReqForBucket.hasOriginalPartitionName() != historicalWriteRequest) { + throw new IllegalArgumentException( + "Normal and historical writes cannot be mixed in the same request."); + } ByteBuffer recordBuffer = toByteBuffer(produceLogReqForBucket.getRecordsSlice()); MemoryLogRecords logRecords = MemoryLogRecords.pointToByteBuffer(recordBuffer); TableBucket tb = @@ -907,9 +917,23 @@ public static Map getProduceLogData( ? produceLogReqForBucket.getPartitionId() : null, produceLogReqForBucket.getBucketId()); - produceEntryData.put(tb, logRecords); + String originalPartitionName = + produceLogReqForBucket.hasOriginalPartitionName() + ? produceLogReqForBucket.getOriginalPartitionName() + : null; + Set originalPartitions = + originalPartitionsByBucket.computeIfAbsent(tb, ignored -> new HashSet<>()); + if (!originalPartitions.add(originalPartitionName)) { + throw new IllegalArgumentException( + "A ProduceLog request contains duplicate table bucket " + + tb + + " and original partition " + + originalPartitionName + + '.'); + } + produceLogData.add(new ProduceLogDataForBucket(tb, logRecords, originalPartitionName)); } - return produceEntryData; + return produceLogData; } public static ProduceLogResponse makeProduceLogResponse( @@ -923,6 +947,9 @@ public static ProduceLogResponse makeProduceLogResponse( if (tableBucket.getPartitionId() != null) { producedBucket.setPartitionId(tableBucket.getPartitionId()); } + if (bucketResult.getOriginalPartitionName() != null) { + producedBucket.setOriginalPartitionName(bucketResult.getOriginalPartitionName()); + } if (bucketResult.failed()) { producedBucket.setError( diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java index 8ba8204a54c..0bd9ab6f522 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java @@ -214,10 +214,6 @@ private static void checkHistoricalPartition( DataLakeFormat.PAIMON, dataLakeFormat.get())); } - if (!tableDescriptor.hasPrimaryKey()) { - unmetRequirements.add("the table must define a primary key"); - } - int partitionKeyCount = tableDescriptor.getPartitionKeys().size(); if (partitionKeyCount != 1) { unmetRequirements.add( diff --git a/fluss-server/src/test/java/org/apache/fluss/server/DynamicConfigChangeTest.java b/fluss-server/src/test/java/org/apache/fluss/server/DynamicConfigChangeTest.java index 5bc4a1fe8ce..6beccb61285 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/DynamicConfigChangeTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/DynamicConfigChangeTest.java @@ -191,6 +191,17 @@ void testAlterLakehouseConfigs() throws Exception { } } + @Test + void testAllowsHistoricalKvCleanupIdleTimeToChangeDynamically() { + assertThat( + new DynamicServerConfig(new Configuration()) + .isAllowedConfig( + ConfigOptions + .SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME + .key())) + .isTrue(); + } + @Test void testOverrideConfigs() throws Exception { Configuration configuration = new Configuration(); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerTest.java index db4a88bc085..3364aa61b08 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerTest.java @@ -67,6 +67,7 @@ import org.apache.fluss.server.entity.FetchReqInfo; import org.apache.fluss.server.entity.NotifyLeaderAndIsrData; import org.apache.fluss.server.entity.NotifyLeaderAndIsrResultForBucket; +import org.apache.fluss.server.entity.ProduceLogDataForBucket; import org.apache.fluss.server.entity.StopReplicaData; import org.apache.fluss.server.entity.StopReplicaResultForBucket; import org.apache.fluss.server.kv.KvTablet; @@ -238,6 +239,32 @@ tb, genMemoryLogRecordsByObject(DATA1)), "Unknown table or bucket: TableBucket{tableId=10001, bucket=0}"))); } + @Test + void testProduceHistoricalLogBatchesToSameTableBucket() throws Exception { + replicaManager.getDiskUsageMonitor().update(0.10); + TableBucket tableBucket = new TableBucket(DATA1_TABLE_ID, 1); + makeLogTableAsLeader(tableBucket.getBucket()); + + CompletableFuture> future = new CompletableFuture<>(); + replicaManager.appendHistoricalRecordsToLog( + 20_000, + 1, + Arrays.asList( + new ProduceLogDataForBucket( + tableBucket, genMemoryLogRecordsByObject(DATA1), "dt=2025-01-01"), + new ProduceLogDataForBucket( + tableBucket, genMemoryLogRecordsByObject(DATA1), "dt=2025-01-02")), + null, + future::complete); + + assertThat(future.get()) + .containsExactlyInAnyOrder( + ProduceLogResultForBucket.historicalSuccess( + tableBucket, 0L, 10L, "dt=2025-01-01"), + ProduceLogResultForBucket.historicalSuccess( + tableBucket, 10L, 20L, "dt=2025-01-02")); + } + @Test void testFetchLog() throws Exception { SchemaGetter schemaGetter = diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java index b18f96b3a7a..526623b050e 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java @@ -83,6 +83,7 @@ import com.github.benmanes.caffeine.cache.Scheduler; import com.github.benmanes.caffeine.cache.Ticker; import org.junit.jupiter.api.Test; +import org.rocksdb.FlushOptions; import javax.annotation.Nullable; @@ -95,6 +96,7 @@ import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -123,6 +125,10 @@ class HistoricalPartitionManagerTest extends ReplicaTestBase { private static final String ANOTHER_ORIGINAL_PARTITION = "20240108"; private static final String HISTORICAL_PARTITION = HISTORICAL_PARTITION_VALUE; private static final TableBucket TABLE_BUCKET = new TableBucket(TABLE_ID, PARTITION_ID, 0); + private static final RowType HISTORICAL_KEY_TYPE = + DataTypes.ROW( + new DataField("id", DataTypes.INT()), + new DataField("region", DataTypes.STRING())); @Test void testResolvesMultipleLakeMissesWithoutPrewriteRollback() throws Exception { @@ -133,7 +139,7 @@ void testResolvesMultipleLakeMissesWithoutPrewriteRollback() throws Exception { TestingHistoricalLakeLookupManager lakeLookupManager = new TestingHistoricalLakeLookupManager(lookupConfiguration()); HistoricalPartitionManager historicalPartitionManager = - new HistoricalPartitionManager( + createNonCleanupManager( new HistoricalPartitionTaskExecutor(lookupConfiguration()), lakeLookupManager); @@ -201,7 +207,7 @@ void testWalFullRowUpsertDoesNotLookupLake() throws Exception { TestingHistoricalLakeLookupManager lakeLookupManager = new TestingHistoricalLakeLookupManager(lookupConfiguration()); HistoricalPartitionManager historicalPartitionManager = - new HistoricalPartitionManager( + createNonCleanupManager( new HistoricalPartitionTaskExecutor(lookupConfiguration()), lakeLookupManager); @@ -250,7 +256,7 @@ void testHistoricalInsertUpdateAndDelete() throws Exception { TestingHistoricalLakeLookupManager lakeLookupManager = new TestingHistoricalLakeLookupManager(lookupConfiguration()); HistoricalPartitionManager historicalPartitionManager = - new HistoricalPartitionManager( + createNonCleanupManager( new HistoricalPartitionTaskExecutor(lookupConfiguration()), lakeLookupManager); @@ -262,7 +268,7 @@ void testHistoricalInsertUpdateAndDelete() throws Exception { byte[] primaryKey = new CompactedKeyEncoder(keyType).encodeKey(row(1, "us")); try { - // The first write misses both local state and lake, so it creates a local overlay. + // The first write misses both local KV state and lake, so it creates a local value. KvRecordBatch insertBatch = batch( keyType, @@ -321,7 +327,7 @@ void testHistoricalInsertUpdateAndDelete() throws Exception { row(1, "us", "20240108", "another")); assertThat(lakeLookupManager.lookupCount).hasValue(2); - // Exercise the ReplicaManager entry point; the update should reuse the local overlay. + // Exercise the ReplicaManager entry point; the update should reuse local KV state. KvRecordBatch updateBatch = batch( keyType, @@ -365,7 +371,7 @@ void testHistoricalInsertUpdateAndDelete() throws Exception { row(1, "us", "20240108", "another")); assertThat(lakeLookupManager.lookupCount).hasValue(2); - // Historical lookup should observe the updated value from the local overlay. + // Historical lookup should observe the updated value from local KV state. CompletableFuture> lookupResponse = new CompletableFuture<>(); replicaManager.historicalLookups( @@ -440,7 +446,7 @@ void testUpdateAndDeleteFromLakeFallback() throws Exception { TestingHistoricalLakeLookupManager lakeLookupManager = new TestingHistoricalLakeLookupManager(lookupConfiguration()); HistoricalPartitionManager historicalPartitionManager = - new HistoricalPartitionManager( + createNonCleanupManager( new HistoricalPartitionTaskExecutor(lookupConfiguration()), lakeLookupManager); @@ -541,7 +547,7 @@ void testLeaderChangeDuringLakeLookupFencesHistoricalWrite() throws Exception { TestingHistoricalLakeLookupManager lakeLookupManager = new TestingHistoricalLakeLookupManager(lookupConfiguration()); HistoricalPartitionManager historicalPartitionManager = - new HistoricalPartitionManager( + createNonCleanupManager( new HistoricalPartitionTaskExecutor(lookupConfiguration()), lakeLookupManager); CountDownLatch lakeLookupStarted = new CountDownLatch(1); @@ -592,13 +598,13 @@ void testLeaderChangeDuringLakeLookupFencesHistoricalWrite() throws Exception { } @Test - void testRecoversHistoricalOverlayFromLakeCommitOffset() throws Exception { + void testRecoversLocalHistoricalKvFromLakeCommitOffset() throws Exception { TableInfo tableInfo = registerHistoricalTableAndBecomeLeader(); Replica replica = replicaManager.getReplicaOrException(TABLE_BUCKET); TestingHistoricalLakeLookupManager lakeLookupManager = new TestingHistoricalLakeLookupManager(lookupConfiguration()); HistoricalPartitionManager historicalPartitionManager = - new HistoricalPartitionManager( + createNonCleanupManager( new HistoricalPartitionTaskExecutor(lookupConfiguration()), lakeLookupManager); @@ -679,7 +685,7 @@ void testRecoversHistoricalOverlayFromLakeCommitOffset() throws Exception { // Persist the exclusive end offset of the first write as the lake recovery point. The // replica has not received this offset locally, so becoming leader must load it before - // creating the historical overlay. + // creating the local historical KV state. long lakeCommitOffset = firstAppend.lastOffset() + 1; new LakeTableHelper(zkClient, DEFAULT_REMOTE_DATA_DIR) .registerLakeTableSnapshotV1( @@ -688,8 +694,8 @@ void testRecoversHistoricalOverlayFromLakeCommitOffset() throws Exception { 1L, Collections.singletonMap(TABLE_BUCKET, lakeCommitOffset))); assertThat(replica.getLakeLogEndOffset()).isEqualTo(-1L); - // Dropping and recreating the leader KV tablet forces the overlay to be rebuilt only - // from WAL after the lake commit offset. The recovered tombstone must remain + // Dropping and recreating the leader KV tablet rebuilds its state only from WAL after + // the lake commit offset. The recovered tombstone must remain // authoritative over lake fallback. assertThat(replica.makeFollower(followerState())).isTrue(); CompletableFuture> leaderFuture = @@ -728,6 +734,296 @@ void testRecoversHistoricalOverlayFromLakeCommitOffset() throws Exception { } } + @Test + void testCleansLocalHistoricalKvAfterTieringAndWriteIdleTime() throws Exception { + TableInfo tableInfo = registerHistoricalTableAndBecomeLeader(); + Replica replica = replicaManager.getReplicaOrException(TABLE_BUCKET); + KvTablet originalKvTablet = replica.getKvTablet(); + assertThat(originalKvTablet).isNotNull(); + + Configuration cleanupConf = lookupConfiguration(); + cleanupConf.set( + ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME, + Duration.ofMinutes(1)); + ManuallyTriggeredScheduledExecutorService executor = + new ManuallyTriggeredScheduledExecutorService(); + TestingHistoricalLakeLookupManager lakeLookupManager = + new TestingHistoricalLakeLookupManager(cleanupConf); + HistoricalPartitionManager historicalPartitionManager = + createCleanupManager(cleanupConf, executor, lakeLookupManager, Long.MAX_VALUE); + + byte[] primaryKey = new CompactedKeyEncoder(HISTORICAL_KEY_TYPE).encodeKey(row(1, "us")); + Object[] valueObjects = new Object[] {1, "us", ORIGINAL_PARTITION, "v1"}; + byte[] lakeValue = + ValueEncoder.encodeValue( + (short) tableInfo.getSchemaId(), + compactedRow(tableInfo.getRowType(), valueObjects)); + lakeLookupManager.putLakeValue(ORIGINAL_PARTITION, lakeValue); + + try { + CompletableFuture putFuture = + putHistoricalRecords( + historicalPartitionManager, + replica, + batch( + HISTORICAL_KEY_TYPE, + tableInfo.getRowType(), + Tuple2.of(new Object[] {1, "us"}, valueObjects))); + executor.triggerAll(); + assertThat(putFuture.get(10, TimeUnit.SECONDS).failed()).isFalse(); + flushAndWait(originalKvTablet, Long.MAX_VALUE); + + long tieredOffset = replica.getLocalLogEndOffset(); + historicalPartitionManager.onLakeProgress(replica, 9L, tieredOffset - 1); + assertThat(executor.numQueuedRunnables()).isZero(); + + replica.getLogTablet().updateLakeLogEndOffset(tieredOffset); + historicalPartitionManager.onLakeProgress(replica, 10L, tieredOffset); + // Lake normally catches up before local KV state becomes idle. Cleanup must wake at + // the write deadline even if no further lake progress notification arrives. + assertThat(executor.numQueuedRunnables()).isZero(); + assertThat(executor.getActiveNonPeriodicScheduledTask()).hasSize(1); + assertThat(replica.getKvTablet()).isSameAs(originalKvTablet); + manualClock.advanceTime(Duration.ofMinutes(1)); + executor.triggerNonPeriodicScheduledTasks(); + assertThat(executor.numQueuedRunnables()).isOne(); + assertThat(lakeLookupManager.requiredLakeSnapshotIds).isEmpty(); + executor.triggerAll(); + + KvTablet cleanedKvTablet = replica.getKvTablet(); + assertThat(cleanedKvTablet).isNotNull().isNotSameAs(originalKvTablet); + assertThat(lakeLookupManager.requiredLakeSnapshotIds).containsExactly(10L); + assertThat(cleanedKvTablet.getRocksDBKv().limitScan(10)).isEmpty(); + assertThat(cleanedKvTablet.lookupHistoricalLocal(ORIGINAL_PARTITION, primaryKey)) + .isEqualTo(KvStateLookupResult.notFound()); + + CompletableFuture lookupFuture = + historicalPartitionManager.lookup( + replica, + new LookupDataForBucket( + TABLE_BUCKET, + Collections.singletonList(primaryKey), + ORIGINAL_PARTITION), + (lookupTimeNanos, lookupFileDownloaded) -> {}); + executor.triggerAll(); + assertThat(lookupFuture.get(10, TimeUnit.SECONDS).lookupValues()) + .extracting(ByteArraySlice::toByteArray) + .containsExactly(lakeValue); + } finally { + historicalPartitionManager.close(); + } + } + + @Test + void testCleanupRequiresLakeAndLocalOffsetsToMatch() throws Exception { + TableInfo tableInfo = registerHistoricalTableAndBecomeLeader(); + Replica replica = replicaManager.getReplicaOrException(TABLE_BUCKET); + KvTablet originalKvTablet = replica.getKvTablet(); + assertThat(originalKvTablet).isNotNull(); + + Configuration cleanupConf = lookupConfiguration(); + cleanupConf.set( + ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME, + Duration.ofMinutes(1)); + ManuallyTriggeredScheduledExecutorService executor = + new ManuallyTriggeredScheduledExecutorService(); + TestingHistoricalLakeLookupManager lakeLookupManager = + new TestingHistoricalLakeLookupManager(cleanupConf); + HistoricalPartitionManager historicalPartitionManager = + createCleanupManager(cleanupConf, executor, lakeLookupManager, Long.MAX_VALUE); + + KvRecordBatch records = + batch( + HISTORICAL_KEY_TYPE, + tableInfo.getRowType(), + Tuple2.of( + new Object[] {1, "us"}, + new Object[] {1, "us", ORIGINAL_PARTITION, "v1"})); + + try { + CompletableFuture firstPut = + putHistoricalRecords(historicalPartitionManager, replica, records); + executor.triggerAll(); + assertThat(firstPut.get(10, TimeUnit.SECONDS).failed()).isFalse(); + + long firstTieredOffset = replica.getLocalLogEndOffset(); + replica.getLogTablet().updateLakeLogEndOffset(firstTieredOffset); + historicalPartitionManager.onLakeProgress(replica, 10L, firstTieredOffset); + executor.triggerAll(); + assertThat(replica.getKvTablet()).isSameAs(originalKvTablet); + assertThat(lakeLookupManager.requiredLakeSnapshotIds).isEmpty(); + + CompletableFuture secondPut = + putHistoricalRecords(historicalPartitionManager, replica, records); + manualClock.advanceTime(Duration.ofMinutes(1)); + historicalPartitionManager.onLakeProgress(replica, 10L, firstTieredOffset); + + // The second write runs before the cleanup that captured snapshot 10 / first offset. + executor.trigger(); + assertThat(secondPut.get(10, TimeUnit.SECONDS).failed()).isFalse(); + long secondTieredOffset = replica.getLocalLogEndOffset(); + assertThat(secondTieredOffset).isGreaterThan(firstTieredOffset); + + manualClock.advanceTime(Duration.ofMinutes(1)); + // A lake offset beyond the local end is inconsistent and must not schedule cleanup. + historicalPartitionManager.onLakeProgress(replica, 11L, secondTieredOffset + 1); + executor.trigger(); + + // Snapshot 10 does not cover the second write, so its queued cleanup must be skipped. + assertThat(replica.getKvTablet()).isSameAs(originalKvTablet); + assertThat(lakeLookupManager.requiredLakeSnapshotIds).isEmpty(); + assertThat(executor.numQueuedRunnables()).isZero(); + + replica.getLogTablet().updateLakeLogEndOffset(secondTieredOffset); + historicalPartitionManager.onLakeProgress(replica, 12L, secondTieredOffset); + executor.triggerAll(); + assertThat(replica.getKvTablet()).isNotNull().isNotSameAs(originalKvTablet); + assertThat(lakeLookupManager.requiredLakeSnapshotIds).containsExactly(12L); + } finally { + historicalPartitionManager.close(); + } + } + + @Test + void testMaxSizeBlocksWritesUntilAcceptedWritesAreTieredAndLocalKvIsCleaned() throws Exception { + TableInfo tableInfo = registerHistoricalTableAndBecomeLeader(); + Replica replica = replicaManager.getReplicaOrException(TABLE_BUCKET); + KvTablet originalKvTablet = replica.getKvTablet(); + assertThat(originalKvTablet).isNotNull(); + + Configuration cleanupConf = lookupConfiguration(); + cleanupConf.set( + ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME, Duration.ZERO); + ManuallyTriggeredScheduledExecutorService executor = + new ManuallyTriggeredScheduledExecutorService(); + HistoricalPartitionManager historicalPartitionManager = + createCleanupManager( + cleanupConf, + executor, + new TestingHistoricalLakeLookupManager(cleanupConf), + 1L); + + KvRecordBatch firstBatch = + batch( + HISTORICAL_KEY_TYPE, + tableInfo.getRowType(), + Tuple2.of( + new Object[] {1, "us"}, + new Object[] {1, "us", ORIGINAL_PARTITION, "v1"})); + KvRecordBatch secondBatch = + batch( + HISTORICAL_KEY_TYPE, + tableInfo.getRowType(), + Tuple2.of( + new Object[] {2, "eu"}, + new Object[] {2, "eu", ORIGINAL_PARTITION, "v2"})); + + try { + CompletableFuture firstPut = + putHistoricalRecords(historicalPartitionManager, replica, firstBatch); + executor.triggerAll(); + assertThat(firstPut.get(10, TimeUnit.SECONDS).failed()).isFalse(); + flushAndWait(originalKvTablet, Long.MAX_VALUE); + try (FlushOptions flushOptions = new FlushOptions().setWaitForFlush(true)) { + originalKvTablet.getRocksDBKv().getDb().flush(flushOptions); + } + assertThat(originalKvTablet.liveSstFilesSize()).isPositive(); + + PutKvResultForBucket blockedWrite = + putHistoricalRecords(historicalPartitionManager, replica, secondBatch) + .get(10, TimeUnit.SECONDS); + assertThat(blockedWrite.getError().error()) + .isEqualTo(Errors.HISTORICAL_PARTITION_THROTTLED); + assertThat(blockedWrite.getError().message()) + .contains( + "reached the live SST maximum size of 1 bytes", + "lake tiering covers all previously accepted writes"); + assertThat(executor.numQueuedRunnables()).isZero(); + + long tieredOffset = replica.getLocalLogEndOffset(); + replica.getLogTablet().updateLakeLogEndOffset(tieredOffset); + historicalPartitionManager.onLakeProgress(replica, 11L, tieredOffset); + executor.triggerAll(); + assertThat(replica.getKvTablet()).isNotNull().isNotSameAs(originalKvTablet); + + CompletableFuture resumedWrite = + putHistoricalRecords(historicalPartitionManager, replica, secondBatch); + executor.triggerAll(); + assertThat(resumedWrite.get(10, TimeUnit.SECONDS).failed()).isFalse(); + } finally { + historicalPartitionManager.close(); + } + } + + @Test + void testMaxSizeCleanupWhenLakeCaughtUpBeforeLimitIsObserved() throws Exception { + TableInfo tableInfo = registerHistoricalTableAndBecomeLeader(); + Replica replica = replicaManager.getReplicaOrException(TABLE_BUCKET); + KvTablet originalKvTablet = replica.getKvTablet(); + assertThat(originalKvTablet).isNotNull(); + + Configuration cleanupConf = lookupConfiguration(); + cleanupConf.set( + ConfigOptions.SERVER_HISTORICAL_PARTITION_KV_CLEANUP_IDLE_TIME, Duration.ZERO); + ManuallyTriggeredScheduledExecutorService executor = + new ManuallyTriggeredScheduledExecutorService(); + TestingHistoricalLakeLookupManager lakeLookupManager = + new TestingHistoricalLakeLookupManager(cleanupConf); + HistoricalPartitionManager historicalPartitionManager = + createCleanupManager(cleanupConf, executor, lakeLookupManager, 1L); + + KvRecordBatch firstBatch = + batch( + HISTORICAL_KEY_TYPE, + tableInfo.getRowType(), + Tuple2.of( + new Object[] {1, "us"}, + new Object[] {1, "us", ORIGINAL_PARTITION, "v1"})); + KvRecordBatch secondBatch = + batch( + HISTORICAL_KEY_TYPE, + tableInfo.getRowType(), + Tuple2.of( + new Object[] {2, "eu"}, + new Object[] {2, "eu", ORIGINAL_PARTITION, "v2"})); + + try { + CompletableFuture firstPut = + putHistoricalRecords(historicalPartitionManager, replica, firstBatch); + executor.triggerAll(); + assertThat(firstPut.get(10, TimeUnit.SECONDS).failed()).isFalse(); + flushAndWait(originalKvTablet, Long.MAX_VALUE); + try (FlushOptions flushOptions = new FlushOptions().setWaitForFlush(true)) { + originalKvTablet.getRocksDBKv().getDb().flush(flushOptions); + } + assertThat(originalKvTablet.liveSstFilesSize()).isPositive(); + + // Lake catches up before another request observes that the SST size reached the limit. + long tieredOffset = replica.getLocalLogEndOffset(); + replica.getLogTablet().updateLakeLogEndOffset(tieredOffset); + historicalPartitionManager.onLakeProgress(replica, 12L, tieredOffset); + assertThat(executor.numQueuedRunnables()).isZero(); + + PutKvResultForBucket blockedWrite = + putHistoricalRecords(historicalPartitionManager, replica, secondBatch) + .get(10, TimeUnit.SECONDS); + assertThat(blockedWrite.getError().error()) + .isEqualTo(Errors.HISTORICAL_PARTITION_THROTTLED); + assertThat(executor.numQueuedRunnables()).isOne(); + + executor.triggerAll(); + assertThat(replica.getKvTablet()).isNotNull().isNotSameAs(originalKvTablet); + assertThat(lakeLookupManager.requiredLakeSnapshotIds).containsExactly(12L); + + CompletableFuture resumedWrite = + putHistoricalRecords(historicalPartitionManager, replica, secondBatch); + executor.triggerAll(); + assertThat(resumedWrite.get(10, TimeUnit.SECONDS).failed()).isFalse(); + } finally { + historicalPartitionManager.close(); + } + } + @Test void testHistoricalLookupThrottledWhenPermitsExhausted() throws Exception { registerHistoricalTableAndBecomeLeader(); @@ -735,7 +1031,7 @@ void testHistoricalLookupThrottledWhenPermitsExhausted() throws Exception { ManuallyTriggeredScheduledExecutorService executor = new ManuallyTriggeredScheduledExecutorService(); HistoricalPartitionManager historicalPartitionManager = - new HistoricalPartitionManager( + createNonCleanupManager( new HistoricalPartitionTaskExecutor(lookupConfiguration(), executor), new TestingHistoricalLakeLookupManager(lookupConfiguration())); @@ -936,6 +1232,68 @@ private static void await(CountDownLatch latch) { } } + private HistoricalPartitionManager createCleanupManager( + Configuration configuration, + ManuallyTriggeredScheduledExecutorService executor, + TestingHistoricalLakeLookupManager lakeLookupManager, + long maxHistoricalKvSizeBytes) { + HistoricalPartitionManager manager = + new HistoricalPartitionManager( + configuration, + new HistoricalPartitionTaskExecutor(configuration, executor), + lakeLookupManager, + manualClock, + maxHistoricalKvSizeBytes, + new TestingCleanupScheduler(executor)); + return manager; + } + + private HistoricalPartitionManager createNonCleanupManager( + HistoricalPartitionTaskExecutor taskExecutor, + HistoricalLakeLookupManager lakeLookupManager) { + return new HistoricalPartitionManager( + new Configuration(), + taskExecutor, + lakeLookupManager, + manualClock, + Long.MAX_VALUE, + null); + } + + private static final class TestingCleanupScheduler + implements org.apache.fluss.utils.concurrent.Scheduler { + private final ManuallyTriggeredScheduledExecutorService executor; + + private TestingCleanupScheduler(ManuallyTriggeredScheduledExecutorService executor) { + this.executor = executor; + } + + @Override + public void startup() {} + + @Override + public void shutdown() {} + + @Override + public ScheduledFuture schedule( + String name, Runnable task, long delayMs, long periodMs) { + if (periodMs > 0L) { + return executor.scheduleAtFixedRate(task, delayMs, periodMs, TimeUnit.MILLISECONDS); + } + return executor.schedule(task, delayMs, TimeUnit.MILLISECONDS); + } + } + + private static CompletableFuture putHistoricalRecords( + HistoricalPartitionManager manager, Replica replica, KvRecordBatch records) { + return manager.putKv( + replica, + new PutKvDataForBucket(TABLE_BUCKET, records, ORIGINAL_PARTITION), + null, + MergeMode.DEFAULT, + 1); + } + @SafeVarargs private static KvRecordBatch batch( RowType keyType, RowType rowType, Tuple2... keyAndValues) @@ -971,6 +1329,7 @@ private final class TestingHistoricalLakeLookupManager extends HistoricalLakeLoo private final AtomicInteger lookupCount = new AtomicInteger(); private final AtomicInteger lookupBatchCount = new AtomicInteger(); private final Map lakeValuesByPartition = new HashMap<>(); + private final List requiredLakeSnapshotIds = new ArrayList<>(); private volatile @Nullable Runnable lookupHook; private TestingHistoricalLakeLookupManager(Configuration configuration) { @@ -992,6 +1351,12 @@ private void setLookupHook(Runnable lookupHook) { this.lookupHook = lookupHook; } + @Override + void requireLakeSnapshot(long tableId, long snapshotId) { + requiredLakeSnapshotIds.add(snapshotId); + super.requireLakeSnapshot(tableId, snapshotId); + } + @Override List lookup( LookupDataForBucket lookupData, diff --git a/fluss-server/src/test/java/org/apache/fluss/server/utils/HistoricalPartitionTableValidationTest.java b/fluss-server/src/test/java/org/apache/fluss/server/utils/HistoricalPartitionTableValidationTest.java index cb427c23ec8..9b6aa6913ea 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/utils/HistoricalPartitionTableValidationTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/utils/HistoricalPartitionTableValidationTest.java @@ -54,7 +54,6 @@ void testReportsAllUnmetHistoricalPartitionRequirements() { + "'table.datalake.enabled' must be set to true; " + "'table.datalake.format' must be set to 'paimon' " + "(currently not set); " - + "the table must define a primary key; " + "the table must define exactly one partition key (found 0)."); // Case 2: Aggregate requirements before related validators can report only one failure. @@ -80,7 +79,6 @@ void testReportsAllUnmetHistoricalPartitionRequirements() { "'table.datalake.historical-partition.enabled' has unmet requirements: " + "'table.datalake.format' must be set to 'paimon' " + "(currently 'iceberg'); " - + "the table must define a primary key; " + "the table must define exactly one partition key (found 0)."); } } diff --git a/fluss-server/src/test/java/org/apache/fluss/server/utils/ServerRpcMessageUtilsTest.java b/fluss-server/src/test/java/org/apache/fluss/server/utils/ServerRpcMessageUtilsTest.java index 9f318eded4c..c5ed0c48a6b 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/utils/ServerRpcMessageUtilsTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/utils/ServerRpcMessageUtilsTest.java @@ -21,12 +21,18 @@ import org.apache.fluss.record.KvRecordBatch; import org.apache.fluss.row.encode.KvValueLayout; import org.apache.fluss.rpc.entity.LookupResultForBucket; +import org.apache.fluss.rpc.entity.ProduceLogResultForBucket; import org.apache.fluss.rpc.entity.PutKvResultForBucket; import org.apache.fluss.rpc.messages.LookupResponse; +import org.apache.fluss.rpc.messages.PbProduceLogReqForBucket; +import org.apache.fluss.rpc.messages.PbProduceLogRespForBucket; import org.apache.fluss.rpc.messages.PbPutKvReqForBucket; import org.apache.fluss.rpc.messages.PbPutKvRespForBucket; +import org.apache.fluss.rpc.messages.ProduceLogRequest; +import org.apache.fluss.rpc.messages.ProduceLogResponse; import org.apache.fluss.rpc.messages.PutKvRequest; import org.apache.fluss.rpc.messages.PutKvResponse; +import org.apache.fluss.server.entity.ProduceLogDataForBucket; import org.apache.fluss.server.entity.PutKvDataForBucket; import org.junit.jupiter.api.Test; @@ -64,6 +70,52 @@ void testLookupResponseSupportsMixedValueLayouts() { .containsExactly(new byte[] {1, 0, 11, 12}, new byte[] {1, 0, 21, 22}); } + @Test + void testHistoricalProduceLogRequestAndResponsePreserveOriginalPartitions() { + long tableId = 1L; + long partitionId = 2L; + PbProduceLogReqForBucket firstBucketRequest = + new PbProduceLogReqForBucket() + .setPartitionId(partitionId) + .setBucketId(0) + .setRecords(new byte[0]) + .setOriginalPartitionName("dt=2025-01-01"); + ProduceLogRequest request = + new ProduceLogRequest().setTableId(tableId).setAcks(1).setTimeoutMs(10_000); + request.addAllBucketsReqs( + Arrays.asList( + firstBucketRequest, + new PbProduceLogReqForBucket() + .copyFrom(firstBucketRequest) + .setOriginalPartitionName("dt=2025-01-02"))); + + TableBucket tableBucket = new TableBucket(tableId, partitionId, 0); + List decoded = + ServerRpcMessageUtils.toProduceLogDataForBuckets(request); + assertThat(decoded) + .extracting(ProduceLogDataForBucket::tableBucket) + .containsOnly(tableBucket); + assertThat(decoded) + .extracting(ProduceLogDataForBucket::originalPartitionName) + .containsExactly("dt=2025-01-01", "dt=2025-01-02"); + + ProduceLogResponse response = + ServerRpcMessageUtils.makeProduceLogResponse( + Arrays.asList( + ProduceLogResultForBucket.historicalSuccess( + tableBucket, 0L, 1L, "dt=2025-01-01"), + ProduceLogResultForBucket.historicalSuccess( + tableBucket, 1L, 2L, "dt=2025-01-02"))); + assertThat(response.getBucketsRespsList()) + .extracting(PbProduceLogRespForBucket::getOriginalPartitionName) + .containsExactly("dt=2025-01-01", "dt=2025-01-02"); + + request.addAllBucketsReqs(Collections.singletonList(firstBucketRequest)); + assertThatThrownBy(() -> ServerRpcMessageUtils.toProduceLogDataForBuckets(request)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("duplicate table bucket"); + } + @Test void testHistoricalPutKvRequestAndResponsePreserveOriginalPartitions() throws Exception { long tableId = 1L;