From 34a12bde1a0ba3f38fafc03e15c5c3e0702fc772 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Fri, 11 Sep 2026 17:37:19 +0800 Subject: [PATCH 01/23] [core] Prune manifest blocks with row-id sidecar indexes Add optional bounded block indexes and Java/PyPaimon pruning. Publish explicit index references in manifest metadata and preserve them through serialization, rewrites, commit cleanup, snapshot retention, and orphan collection. --- .../java/org/apache/paimon/CoreOptions.java | 28 + .../org/apache/paimon/AbstractFileStore.java | 17 +- .../paimon/manifest/ManifestAvroReader.java | 13 + .../paimon/manifest/ManifestAvroWriter.java | 50 +- .../apache/paimon/manifest/ManifestFile.java | 93 +++- .../paimon/manifest/ManifestFileMeta.java | 54 +- .../manifest/ManifestFileMetaSerializer.java | 10 +- .../paimon/manifest/ManifestRowIdIndex.java | 502 ++++++++++++++++++ .../operation/AbstractFileStoreScan.java | 12 +- .../paimon/operation/ChangelogDeletion.java | 13 +- .../paimon/operation/FileDeletionBase.java | 17 +- .../paimon/operation/ManifestFileMerger.java | 2 +- .../paimon/operation/OrphanFilesClean.java | 3 + .../operation/commit/CommitCleaner.java | 6 +- .../ManifestFileMetaSerializerTest.java | 30 ++ .../paimon/manifest/ManifestFileTest.java | 436 ++++++++++++++- .../manifest/ManifestIndexTestUtils.java | 92 ++++ .../paimon/manifest/ManifestListTest.java | 6 +- .../manifest/ManifestRowIdIndexTest.java | 344 ++++++++++++ .../paimon/operation/ExpireSnapshotsTest.java | 57 ++ .../operation/LocalOrphanFilesCleanTest.java | 36 ++ .../resources/manifest-row-id-index-v2.txt | 19 + .../org/apache/avro/file/RawBlockReader.java | 84 ++- .../paimon/format/avro/AvroBlockReader.java | 13 + .../pypaimon/common/options/core_options.py | 24 + .../manifest/manifest_file_manager.py | 86 ++- .../pypaimon/manifest/manifest_file_merger.py | 6 +- .../manifest/manifest_list_manager.py | 2 + .../pypaimon/manifest/row_id_index.py | 303 +++++++++++ .../manifest/schema/manifest_file_meta.py | 2 + .../pypaimon/read/scanner/file_scanner.py | 1 + .../tests/manifest/row_id_index_test.py | 359 +++++++++++++ .../pypaimon/write/file_store_commit.py | 6 +- 33 files changed, 2645 insertions(+), 81 deletions(-) create mode 100644 paimon-core/src/main/java/org/apache/paimon/manifest/ManifestRowIdIndex.java create mode 100644 paimon-core/src/test/java/org/apache/paimon/manifest/ManifestIndexTestUtils.java create mode 100644 paimon-core/src/test/java/org/apache/paimon/manifest/ManifestRowIdIndexTest.java create mode 100644 paimon-core/src/test/resources/manifest-row-id-index-v2.txt create mode 100644 paimon-python/pypaimon/manifest/row_id_index.py create mode 100644 paimon-python/pypaimon/tests/manifest/row_id_index_test.py diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java index 7b0665c50296..5561d31abf5d 100644 --- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java @@ -510,6 +510,34 @@ public InlineElement getDescription() { + "in the previous file. This must not exceed " + "'variant.shredding.minFieldCardinalityRatio'."); + public static final ConfigOption MANIFEST_ROW_ID_INDEX_WRITE = + key("manifest.row-id-index.write") + .booleanType() + .defaultValue(false) + .withDescription( + "Write complete row-id block indexes for newly created manifests."); + + public static final ConfigOption MANIFEST_ROW_ID_INDEX_READ = + key("manifest.row-id-index.read") + .booleanType() + .defaultValue(false) + .withDescription( + "Read optional row-id sidecars after coarse manifest pruning. Missing or invalid indexes fall back to manifest reads."); + + public static final ConfigOption MANIFEST_ROW_ID_INDEX_MAX_RANGES = + key("manifest.row-id-index.max-ranges") + .intType() + .defaultValue(131072) + .withDescription( + "Maximum disjoint row-id intervals across all Avro blocks in a manifest. Exceeding the limit disables the entire index. Range: 1 to 1048576."); + + public static final ConfigOption MANIFEST_ROW_ID_INDEX_MAX_BYTES = + key("manifest.row-id-index.max-bytes") + .intType() + .defaultValue(8388608) + .withDescription( + "Maximum serialized row-id sidecar bytes, including header and checksum. Exceeding the limit disables the entire index. Range: 128 to 67108864."); + public static final ConfigOption MANIFEST_COMPRESSION = key("manifest.compression") .stringType() diff --git a/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java b/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java index 7399e057783c..df0cccb7d9bd 100644 --- a/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java +++ b/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java @@ -204,14 +204,15 @@ public ChangelogManager changelogManager() { @Override public ManifestFile.Factory manifestFileFactory() { return new ManifestFile.Factory( - fileIO, - schemaManager, - partitionType, - FileFormat.manifestFormat(options), - options.manifestCompression(), - pathFactory(), - options.manifestTargetSize().getBytes(), - readManifestCache); + fileIO, + schemaManager, + partitionType, + FileFormat.manifestFormat(options), + options.manifestCompression(), + pathFactory(), + options.manifestTargetSize().getBytes(), + readManifestCache) + .withRowIdIndexOptions(options.toConfiguration()); } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestAvroReader.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestAvroReader.java index b1863358779e..79500c97a60a 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestAvroReader.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestAvroReader.java @@ -70,6 +70,19 @@ public final class ManifestAvroReader implements AutoCloseable { } } + @Nullable + public byte[] headerBytes() { + return blockReader.headerBytes(); + } + + public long blockOffset() { + return blockReader.blockOffset(); + } + + public long blockLength() { + return blockReader.blockLength(); + } + /** Returns whether another raw Avro block is available. */ public boolean hasNext() throws IOException { return blockReader.hasNextBlock(); diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestAvroWriter.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestAvroWriter.java index e78f273e29f1..c0c7f456b764 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestAvroWriter.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestAvroWriter.java @@ -68,6 +68,7 @@ public final class ManifestAvroWriter implements AutoCloseable { private final String compression; private final PathFactory pathFactory; private final long targetFileSize; + private final ManifestRowIdIndex.Settings rowIdIndexSettings; private final List results = new ArrayList<>(); private final List completedPaths = new ArrayList<>(); @@ -83,7 +84,8 @@ public final class ManifestAvroWriter implements AutoCloseable { ObjectSerializer serializer, String compression, PathFactory pathFactory, - long targetFileSize) { + long targetFileSize, + ManifestRowIdIndex.Settings rowIdIndexSettings) { this.fileIO = fileIO; this.schemaManager = schemaManager; this.partitionType = partitionType; @@ -92,6 +94,7 @@ public final class ManifestAvroWriter implements AutoCloseable { this.compression = compression; this.pathFactory = pathFactory; this.targetFileSize = targetFileSize; + this.rowIdIndexSettings = rowIdIndexSettings; } public void write(ManifestEntry entry) throws IOException { @@ -218,6 +221,9 @@ private void closeCurrentWriter() throws IOException { currentWriter.close(); ManifestFileMeta result = currentWriter.result(); completedPaths.add(currentWriter.path); + if (currentWriter.sidecarCreated) { + completedPaths.add(ManifestRowIdIndex.path(currentWriter.path)); + } results.add(result); currentWriter = null; } @@ -403,6 +409,7 @@ private final class FileWriter { private @Nullable RowIdStats rowIdStats = new RowIdStats(); private boolean closed; private boolean aborted; + private boolean sidecarCreated; private FileWriter(Path path) { this.path = path; @@ -477,7 +484,7 @@ private void collectStats(ManifestEntry entry) { maxLevel = Math.max(maxLevel, entry.level()); if (rowIdStats != null) { Long firstRowId = entry.file().firstRowId(); - if (firstRowId == null) { + if (!validRowIdRange(firstRowId, entry.file().rowCount())) { rowIdStats = null; } else { rowIdStats.collect(firstRowId, entry.file().rowCount()); @@ -503,7 +510,7 @@ private void collectStats(EncodedEntry entry) { minLevel = Math.min(minLevel, entry.level); maxLevel = Math.max(maxLevel, entry.level); if (rowIdStats != null) { - if (!entry.hasRowId) { + if (!entry.hasRowId || !validRowIdRange(entry.firstRowId, entry.rowCount)) { rowIdStats = null; } else { rowIdStats.collect(entry.firstRowId, entry.rowCount); @@ -668,6 +675,14 @@ private Throwable abortCollecting(@Nullable Throwable primaryFailure, boolean de ExceptionUtils.firstOrSuppressed(cleanupFailure, primaryFailure); } } + if (sidecarCreated) { + try { + fileIO.deleteQuietly(ManifestRowIdIndex.path(path)); + } catch (Throwable cleanupFailure) { + primaryFailure = + ExceptionUtils.firstOrSuppressed(cleanupFailure, primaryFailure); + } + } return primaryFailure; } @@ -682,6 +697,7 @@ private void close() throws IOException { outputBytes = out.getPos(); out.close(); out = null; + writeRowIdIndex(); } catch (IOException | RuntimeException | Error failure) { abortCollecting(failure, true); throw failure; @@ -690,6 +706,27 @@ private void close() throws IOException { } } + private void writeRowIdIndex() throws IOException { + if (!rowIdIndexSettings.write) { + return; + } + byte[] bytes = + ManifestRowIdIndex.build( + fileIO, + path, + outputBytes, + Math.addExact(numAddedFiles, numDeletedFiles), + rowIdIndexSettings); + if (bytes != null) { + // Publish result() only after both immutable objects have closed. No rename. + try (PositionOutputStream indexOut = + fileIO.newOutputStream(ManifestRowIdIndex.path(path), false)) { + sidecarCreated = true; + indexOut.write(bytes); + } + } + } + private ManifestFileMeta result() { if (!closed || outputBytes == null) { throw new IllegalStateException( @@ -709,10 +746,15 @@ private ManifestFileMeta result() { levelStatsKnown ? minLevel : null, levelStatsKnown ? maxLevel : null, rowIdStats == null ? null : rowIdStats.minRowId, - rowIdStats == null ? null : rowIdStats.maxRowId); + rowIdStats == null ? null : rowIdStats.maxRowId, + sidecarCreated ? ManifestRowIdIndex.path(path).getName() : null); } } + private static boolean validRowIdRange(@Nullable Long first, long count) { + return first != null && first >= 0 && count > 0 && count - 1 <= Long.MAX_VALUE - first; + } + private static class RowIdStats { private long minRowId = Long.MAX_VALUE; diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java index 0dc99a047076..00848d0d35ea 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java @@ -27,6 +27,7 @@ import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.manifest.ProjectedManifestEntry.Projection; import org.apache.paimon.operation.metrics.CacheMetrics; +import org.apache.paimon.options.Options; import org.apache.paimon.partition.PartitionPredicate; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.types.RowType; @@ -36,6 +37,7 @@ import org.apache.paimon.utils.Filter; import org.apache.paimon.utils.ObjectsFile; import org.apache.paimon.utils.PathFactory; +import org.apache.paimon.utils.RowRangeIndex; import org.apache.paimon.utils.SegmentsCache; import javax.annotation.Nullable; @@ -59,6 +61,7 @@ public class ManifestFile extends ObjectsFile { private final RowType partitionType; private final AvroFileFormat avroFileFormat; private final long suggestedFileSize; + private final ManifestRowIdIndex.Settings rowIdIndexSettings; private ManifestFile( FileIO fileIO, @@ -69,7 +72,8 @@ private ManifestFile( String compression, PathFactory pathFactory, long suggestedFileSize, - @Nullable SegmentsCache cache) { + @Nullable SegmentsCache cache, + ManifestRowIdIndex.Settings rowIdIndexSettings) { super( fileIO, serializer, @@ -85,6 +89,7 @@ private ManifestFile( this.partitionType = partitionType; this.avroFileFormat = avroFileFormat; this.suggestedFileSize = suggestedFileSize; + this.rowIdIndexSettings = rowIdIndexSettings; } @Override @@ -136,9 +141,33 @@ public List read( Filter readFilter, Filter readTFilter, Function convertor) { + return read( + fileName, + fileSize, + partitionFilter, + bucketFilter, + readFilter, + readTFilter, + convertor, + null); + } + + public List read( + String fileName, + @Nullable Long fileSize, + @Nullable PartitionPredicate partitionFilter, + @Nullable BucketFilter bucketFilter, + Filter readFilter, + Filter readTFilter, + Function convertor, + @Nullable ManifestRowIdIndex.Selection selected) { + if (selected != null && selected.blocks().isEmpty()) { + return java.util.Collections.emptyList(); + } try { Path path = pathFactory.toPath(fileName); - if (cache != null) { + // A partial manifest must never enter the cache under the full manifest's key. + if (cache != null && selected == null) { ManifestEntryFilters filters = new ManifestEntryFilters( partitionFilter, bucketFilter, readFilter, readTFilter); @@ -151,7 +180,8 @@ public List read( path, ManifestEntry.MANIFEST_ROW_TYPE, partitionFilter, - bucketFilter); + bucketFilter, + selected); return readFromIterator(iterator, serializer, readFilter, readTFilter, convertor); } catch (IOException e) { throw new UncheckedIOException(e); @@ -205,8 +235,21 @@ private static CloseableIterator createManifestIterator( @Nullable PartitionPredicate partitionFilter, @Nullable BucketFilter bucketFilter) throws IOException { + return createManifestIterator( + fileIO, path, projectedType, partitionFilter, bucketFilter, null); + } + + private static CloseableIterator createManifestIterator( + FileIO fileIO, + Path path, + RowType projectedType, + @Nullable PartitionPredicate partitionFilter, + @Nullable BucketFilter bucketFilter, + @Nullable ManifestRowIdIndex.Selection selected) + throws IOException { try { - ManifestAvroReader reader = new ManifestAvroReader(fileIO.newInputStream(path)); + ManifestAvroReader reader = + new ManifestAvroReader(ManifestRowIdIndex.openManifest(fileIO, path, selected)); return reader.read(projectedType, partitionFilter, bucketFilter); } catch (IOException e) { FileUtils.checkExists(fileIO, path); @@ -301,7 +344,8 @@ public ManifestAvroWriter createAvroWriter() { serializer, compression, pathFactory, - suggestedFileSize); + suggestedFileSize, + rowIdIndexSettings); } /** Creates an Avro manifest writer for one explicit path. */ @@ -314,7 +358,8 @@ public ManifestAvroWriter createAvroWriter(Path manifestPath) { serializer, compression, singlePathFactory(manifestPath), - Long.MAX_VALUE); + Long.MAX_VALUE, + rowIdIndexSettings); } private PathFactory singlePathFactory(Path manifestPath) { @@ -339,6 +384,32 @@ public Path toPath(String fileName) { }; } + @Nullable + public ManifestRowIdIndex.Selection selectBlocks( + ManifestFileMeta manifest, @Nullable RowRangeIndex query) { + return !rowIdIndexSettings.read || query == null || manifest.indexFileName() == null + ? null + : ManifestRowIdIndex.read( + fileIO, + pathFactory.toPath(manifest.fileName()), + manifest, + query, + rowIdIndexSettings); + } + + public boolean mayContainRowIds(ManifestFileMeta manifest, @Nullable RowRangeIndex query) { + ManifestRowIdIndex.Selection selected = selectBlocks(manifest, query); + return selected == null || !selected.blocks().isEmpty(); + } + + /** Deletes an unreferenced manifest and its explicitly referenced sidecar. */ + public void delete(ManifestFileMeta manifest) { + delete(manifest.fileName()); + if (manifest.indexFileName() != null) { + delete(manifest.indexFileName()); + } + } + /** Creator of {@link ManifestFile}. */ public static class Factory { @@ -349,6 +420,8 @@ public static class Factory { private final String compression; private final FileStorePathFactory pathFactory; private final long suggestedFileSize; + private ManifestRowIdIndex.Settings rowIdIndexSettings = + new ManifestRowIdIndex.Settings(new Options()); @Nullable private final SegmentsCache cache; public Factory( @@ -370,6 +443,11 @@ public Factory( this.cache = cache; } + public Factory withRowIdIndexOptions(Options options) { + rowIdIndexSettings = new ManifestRowIdIndex.Settings(options); + return this; + } + public boolean isCacheEnabled() { return cache != null; } @@ -384,7 +462,8 @@ public ManifestFile create() { compression, pathFactory.manifestFileFactory(), suggestedFileSize, - cache); + cache, + rowIdIndexSettings); } } } diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFileMeta.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFileMeta.java index 4a66a3fb0d10..44367e04982e 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFileMeta.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFileMeta.java @@ -57,7 +57,11 @@ public class ManifestFileMeta { new DataField(8, "_MIN_LEVEL", new IntType(true)), new DataField(9, "_MAX_LEVEL", new IntType(true)), new DataField(10, "_MIN_ROW_ID", new BigIntType(true)), - new DataField(11, "_MAX_ROW_ID", new BigIntType(true)))); + new DataField(11, "_MAX_ROW_ID", new BigIntType(true)), + new DataField( + 12, + "_INDEX_FILE_NAME", + new VarCharType(true, Integer.MAX_VALUE)))); private final String fileName; private final long fileSize; @@ -71,6 +75,7 @@ public class ManifestFileMeta { private final @Nullable Integer maxLevel; private final @Nullable Long minRowId; private final @Nullable Long maxRowId; + private final @Nullable String indexFileName; public ManifestFileMeta( String fileName, @@ -85,6 +90,36 @@ public ManifestFileMeta( @Nullable Integer maxLevel, @Nullable Long minRowId, @Nullable Long maxRowId) { + this( + fileName, + fileSize, + numAddedFiles, + numDeletedFiles, + partitionStats, + schemaId, + minBucket, + maxBucket, + minLevel, + maxLevel, + minRowId, + maxRowId, + null); + } + + public ManifestFileMeta( + String fileName, + long fileSize, + long numAddedFiles, + long numDeletedFiles, + SimpleStats partitionStats, + long schemaId, + @Nullable Integer minBucket, + @Nullable Integer maxBucket, + @Nullable Integer minLevel, + @Nullable Integer maxLevel, + @Nullable Long minRowId, + @Nullable Long maxRowId, + @Nullable String indexFileName) { this.fileName = fileName; this.fileSize = fileSize; this.numAddedFiles = numAddedFiles; @@ -97,6 +132,7 @@ public ManifestFileMeta( this.maxLevel = maxLevel; this.minRowId = minRowId; this.maxRowId = maxRowId; + this.indexFileName = indexFileName; } public String fileName() { @@ -147,6 +183,11 @@ public long schemaId() { return maxRowId; } + /** Name of the published sidecar in the manifest directory; null means no index. */ + public @Nullable String indexFileName() { + return indexFileName; + } + @Override public boolean equals(Object o) { if (!(o instanceof ManifestFileMeta)) { @@ -164,7 +205,8 @@ public boolean equals(Object o) { && Objects.equals(minLevel, that.minLevel) && Objects.equals(maxLevel, that.maxLevel) && Objects.equals(minRowId, that.minRowId) - && Objects.equals(maxRowId, that.maxRowId); + && Objects.equals(maxRowId, that.maxRowId) + && Objects.equals(indexFileName, that.indexFileName); } @Override @@ -181,13 +223,14 @@ public int hashCode() { minLevel, maxLevel, minRowId, - maxRowId); + maxRowId, + indexFileName); } @Override public String toString() { return String.format( - "{%s, %d, %d, %d, %s, %d, %s, %s, %s, %s, %s, %s}", + "{%s, %d, %d, %d, %s, %d, %s, %s, %s, %s, %s, %s, %s}", fileName, fileSize, numAddedFiles, @@ -199,7 +242,8 @@ public String toString() { minLevel, maxLevel, minRowId, - maxRowId); + maxRowId, + indexFileName); } // ----------------------- Serialization ----------------------------- diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFileMetaSerializer.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFileMetaSerializer.java index 4c0749324231..5c240651ad9b 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFileMetaSerializer.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFileMetaSerializer.java @@ -56,7 +56,10 @@ public InternalRow toRow(ManifestFileMeta meta) { meta.minLevel(), meta.maxLevel(), meta.minRowId(), - meta.maxRowId()); + meta.maxRowId(), + meta.indexFileName() == null + ? null + : BinaryString.fromString(meta.indexFileName())); } @Override @@ -90,6 +93,9 @@ private ManifestFileMeta fromDataRow(InternalRow row) { row.isNullAt(8) ? null : row.getInt(8), row.isNullAt(9) ? null : row.getInt(9), row.isNullAt(10) ? null : row.getLong(10), - row.isNullAt(11) ? null : row.getLong(11)); + row.isNullAt(11) ? null : row.getLong(11), + row.getFieldCount() <= 12 || row.isNullAt(12) + ? null + : row.getString(12).toString()); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestRowIdIndex.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestRowIdIndex.java new file mode 100644 index 000000000000..bd55946c8478 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestRowIdIndex.java @@ -0,0 +1,502 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.manifest; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.SeekableInputStream; +import org.apache.paimon.options.Options; +import org.apache.paimon.utils.RowRangeIndex; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.io.InterruptedIOException; +import java.io.UncheckedIOException; +import java.net.SocketTimeoutException; +import java.nio.ByteBuffer; +import java.nio.channels.ClosedByInterruptException; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.CancellationException; + +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** Complete row-id interval unions and physical locations of a manifest's Avro blocks. */ +public final class ManifestRowIdIndex { + public static final String SUFFIX = ".row-id-index"; + private static final Logger LOG = LoggerFactory.getLogger(ManifestRowIdIndex.class); + private static final long MAGIC = 0x5041494d52494458L; + private static final int HEADER_BYTES = 68; + private static final int DIGEST_BYTES = 32; + private static final int MAX_AVRO_HEADER = 1024 * 1024; + + private ManifestRowIdIndex() {} + + public static Path path(Path manifest) { + return new Path(manifest.toString() + SUFFIX); + } + + /** Independent read/write switches and construction/serialization bounds. */ + public static final class Settings { + public final boolean write; + public final boolean read; + public final int maxRanges; + public final int maxBytes; + + public Settings(Options options) { + write = options.get(CoreOptions.MANIFEST_ROW_ID_INDEX_WRITE); + read = options.get(CoreOptions.MANIFEST_ROW_ID_INDEX_READ); + maxRanges = options.get(CoreOptions.MANIFEST_ROW_ID_INDEX_MAX_RANGES); + maxBytes = options.get(CoreOptions.MANIFEST_ROW_ID_INDEX_MAX_BYTES); + checkArgument( + maxRanges > 0 && maxRanges <= 1048576, + "manifest.row-id-index.max-ranges must be in [1, 1048576]"); + checkArgument( + maxBytes >= 128 && maxBytes <= 64 * 1024 * 1024, + "manifest.row-id-index.max-bytes must be in [128, 67108864]"); + } + } + + /** Original file offset/length and zero-based manifest entry ordinal, not table row id. */ + public static final class Block { + public final long offset; + public final long length; + public final long firstRecord; + public final long recordCount; + + public Block(long offset, long length, long firstRecord, long recordCount) { + this.offset = offset; + this.length = length; + this.firstRecord = firstRecord; + this.recordCount = recordCount; + } + } + + /** Selected blocks in original file order. Empty means the manifest can be excluded. */ + public static final class Selection { + private final byte[] header; + private final List blocks; + + private Selection(byte[] header, List blocks) { + this.header = header; + this.blocks = Collections.unmodifiableList(blocks); + } + + public List blocks() { + return blocks; + } + } + + /** Bounded range union. No row-id enumeration, even for a range ending at Long.MAX_VALUE. */ + public static final class Builder { + private final Settings settings; + private final TreeMap ranges = new TreeMap<>(); + private final ByteArrayOutputStream payload = new ByteArrayOutputStream(); + private final DataOutputStream out = new DataOutputStream(payload); + private final int countPosition; + private boolean complete; + private long nextOffset; + private long nextRecord; + private Block current; + private long entriesInBlock; + private int blocks; + private int rangeCount; + + public Builder(Settings settings, @Nullable byte[] header) throws IOException { + this.settings = settings; + this.complete = + header != null + && header.length <= MAX_AVRO_HEADER + && header.length + HEADER_BYTES + DIGEST_BYTES + 8 <= settings.maxBytes; + countPosition = complete ? 4 + header.length : 0; + if (complete) { + out.writeInt(header.length); + out.write(header); + out.writeInt(0); + nextOffset = header.length; + } + } + + public boolean complete() { + return complete; + } + + private void disable(String reason) { + complete = false; + ranges.clear(); + payload.reset(); + LOG.debug("Omitting manifest row-id block index: {}", reason); + } + + public void beginBlock(long offset, long length, long records) throws IOException { + if (!complete) { + return; + } + require(current == null && offset == nextOffset && length > 0 && records > 0); + current = new Block(offset, length, nextRecord, records); + entriesInBlock = 0; + } + + public void add(@Nullable Long first, long count) { + if (!complete) { + return; + } + if (current == null) { + throw new IllegalStateException("No current Avro block"); + } + entriesInBlock++; + if (first == null || first < 0 || count <= 0 || count - 1 > Long.MAX_VALUE - first) { + disable("unknown or invalid row-id coverage"); + return; + } + long start = first; + long end = first + (count - 1); + Map.Entry before = ranges.floorEntry(start); + if (before != null && before.getValue() >= start - 1) { + start = before.getKey(); + end = Math.max(end, before.getValue()); + ranges.remove(before.getKey()); + } + Map.Entry next; + while ((next = ranges.ceilingEntry(start)) != null + && (next.getKey() <= end || next.getKey() - end == 1)) { + end = Math.max(end, next.getValue()); + ranges.remove(next.getKey()); + } + if (rangeCount + ranges.size() >= settings.maxRanges) { + disable("range budget exceeded"); + return; + } + ranges.put(start, end); + } + + public void endBlock() throws IOException { + if (!complete) { + return; + } + require(current != null && entriesInBlock == current.recordCount && !ranges.isEmpty()); + if (HEADER_BYTES + DIGEST_BYTES + (long) payload.size() + 36 + 16L * ranges.size() + > settings.maxBytes) { + disable("serialized byte budget exceeded"); + return; + } + out.writeLong(current.offset); + out.writeLong(current.length); + out.writeLong(current.firstRecord); + out.writeLong(current.recordCount); + out.writeInt(ranges.size()); + for (Map.Entry range : ranges.entrySet()) { + out.writeLong(range.getKey()); + out.writeLong(range.getValue()); + } + nextOffset = Math.addExact(current.offset, current.length); + nextRecord = Math.addExact(current.firstRecord, current.recordCount); + rangeCount += ranges.size(); + blocks++; + ranges.clear(); + current = null; + } + + @Nullable + public byte[] serialize(String name, long fileSize, long entryCount) throws IOException { + if (!complete) { + return null; + } + require(current == null && nextOffset == fileSize && nextRecord == entryCount); + byte[] body = payload.toByteArray(); + ByteBuffer.wrap(body).putInt(countPosition, blocks); + ByteArrayOutputStream buffer = + new ByteArrayOutputStream(HEADER_BYTES + body.length + DIGEST_BYTES); + DataOutputStream envelope = new DataOutputStream(buffer); + envelope.writeLong(MAGIC); + envelope.writeShort(2); + envelope.writeShort(2); // sorted inclusive interval unions per Avro block + envelope.writeInt(1); // COMPLETE; all other bits reserved + envelope.write(digest(name.getBytes(StandardCharsets.UTF_8))); + envelope.writeLong(fileSize); + envelope.writeLong(entryCount); + envelope.writeInt(body.length); + envelope.write(body); + envelope.write(digest(buffer.toByteArray())); + return buffer.toByteArray(); + } + } + + /** Rebuild from the final physical blocks, including raw-copy and encoded rewrite paths. */ + @Nullable + public static byte[] build(FileIO io, Path path, long size, long records, Settings settings) + throws IOException { + try (ManifestAvroReader reader = new ManifestAvroReader(io.newInputStream(path))) { + Builder builder = new Builder(settings, reader.headerBytes()); + ProjectedManifestEntry.Projection projection = + ProjectedManifestEntry.ROW_RANGE_PROJECTION; + ProjectedManifestEntry entry = projection.createEntry(); + while (builder.complete() && reader.hasNext()) { + ManifestAvroReader.RawBlock block = reader.next(); + builder.beginBlock(reader.blockOffset(), reader.blockLength(), block.recordCount()); + ManifestAvroReader.RowIterator rows = block.toRows(projection.projectedType()); + while (builder.complete() && rows.hasNext()) { + entry.replace(rows.next()); + builder.add(entry.file().firstRowId(), entry.file().rowCount()); + } + builder.endBlock(); + } + return builder.serialize(path.getName(), size, records); + } + } + + /** Validate the complete index before allowing any negative decision. */ + public static Selection select( + byte[] data, ManifestFileMeta manifest, RowRangeIndex query, Settings settings) + throws IOException { + require(data.length >= 128 && data.length <= settings.maxBytes); + int checksumOffset = data.length - DIGEST_BYTES; + require( + MessageDigest.isEqual( + digest(Arrays.copyOf(data, checksumOffset)), + Arrays.copyOfRange(data, checksumOffset, data.length))); + DataInputStream in = new DataInputStream(new ByteArrayInputStream(data, 0, checksumOffset)); + require( + in.readLong() == MAGIC + && in.readUnsignedShort() == 2 + && in.readUnsignedShort() == 2 + && in.readInt() == 1); + byte[] nameHash = new byte[DIGEST_BYTES]; + in.readFully(nameHash); + require( + MessageDigest.isEqual( + nameHash, digest(manifest.fileName().getBytes(StandardCharsets.UTF_8)))); + require(in.readLong() == manifest.fileSize()); + long entries = Math.addExact(manifest.numAddedFiles(), manifest.numDeletedFiles()); + require(in.readLong() == entries && in.readInt() == checksumOffset - HEADER_BYTES); + int headerLength = in.readInt(); + require( + headerLength >= 21 + && headerLength <= MAX_AVRO_HEADER + && headerLength <= in.available() - 4); + byte[] header = new byte[headerLength]; + in.readFully(header); + require(header[0] == 'O' && header[1] == 'b' && header[2] == 'j' && header[3] == 1); + int blocks = in.readInt(); + require(blocks >= 0 && blocks <= in.available() / 52); + long nextOffset = headerLength; + long nextRecord = 0; + int totalRanges = 0; + List selected = new ArrayList<>(); + ByteBuffer view = ByteBuffer.wrap(data); + for (int i = 0; i < blocks; i++) { + long offset = in.readLong(); + long length = in.readLong(); + long first = in.readLong(); + long count = in.readLong(); + int ranges = in.readInt(); + require(offset == nextOffset && length > 0 && length <= manifest.fileSize() - offset); + require(first == nextRecord && count > 0 && count <= entries - first); + require( + ranges > 0 + && ranges <= settings.maxRanges - totalRanges + && ranges <= in.available() / 16); + totalRanges += ranges; + int rangesEnd = checksumOffset - in.available() + 16 * ranges; + long minRowId = in.readLong(); + long firstEnd = in.readLong(); + // Sorted intervals already encode the envelope. Peek at the final endpoint without + // adding redundant fields to the format or materializing the interval list. + long maxRowId = ranges == 1 ? firstEnd : view.getLong(rangesEnd - Long.BYTES); + require(minRowId >= 0 && firstEnd >= minRowId && maxRowId >= firstEnd); + boolean candidate = query.intersects(minRowId, maxRowId); + boolean hit = candidate && (ranges == 1 || query.intersects(minRowId, firstEnd)); + long previousEnd = firstEnd; + for (int j = 1; j < ranges; j++) { + long start = in.readLong(); + long end = in.readLong(); + // Validate even rejected blocks: a checksummed but malformed interval list must + // still cause a conservative fallback, not a false negative from its envelope. + require(start >= 0 && end >= start && start > previousEnd); + previousEnd = end; + if (candidate && !hit) { + hit = query.intersects(start, end); + } + } + if (hit) { + selected.add(new Block(offset, length, first, count)); + } + nextOffset = offset + length; + nextRecord = first + count; + } + require(in.available() == 0 && nextOffset == manifest.fileSize() && nextRecord == entries); + return new Selection(header, selected); + } + + /** One bounded GET attempt, without a preceding HEAD. Null means read the original manifest. */ + @Nullable + public static Selection read( + FileIO io, + Path path, + ManifestFileMeta manifest, + RowRangeIndex query, + Settings settings) { + if (manifest.indexFileName() == null) { + return null; + } + try { + byte[] data; + try (InputStream in = + io.newInputStream(new Path(path.getParent(), manifest.indexFileName()))) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buffer = new byte[8192]; + int n; + while ((n = + in.read( + buffer, + 0, + Math.min( + buffer.length, settings.maxBytes + 1 - out.size()))) + != -1) { + out.write(buffer, 0, n); + require(out.size() <= settings.maxBytes); + } + data = out.toByteArray(); + } + return select(data, manifest, query, settings); + } catch (CancellationException failure) { + throw failure; + } catch (IOException | RuntimeException failure) { + for (Throwable cause = failure; cause != null; cause = cause.getCause()) { + if (cause instanceof CancellationException) { + throw (CancellationException) cause; + } + if (cause instanceof InterruptedException + || cause instanceof ClosedByInterruptException + || (cause instanceof InterruptedIOException + && !(cause instanceof SocketTimeoutException))) { + Thread.currentThread().interrupt(); + throw interrupted(failure); + } + } + if (Thread.currentThread().isInterrupted()) { + throw interrupted(failure); + } + LOG.debug("Cannot use row-id block index for {}; reading manifest", path, failure); + return null; + } + } + + private static UncheckedIOException interrupted(Throwable failure) { + InterruptedIOException interrupted = + new InterruptedIOException("Interrupted reading row-id index"); + interrupted.initCause(failure); + return new UncheckedIOException(interrupted); + } + + static InputStream openManifest(FileIO io, Path path, @Nullable Selection selected) + throws IOException { + SeekableInputStream input = io.newInputStream(path); + return selected == null ? input : new SelectedBlockInput(input, selected); + } + + /** An OCF stream comprising the original header and selected complete compressed blocks. */ + private static final class SelectedBlockInput extends InputStream { + private final SeekableInputStream input; + private final Selection selected; + private int headerPosition; + private int blockPosition; + private long remaining; + private long previousEnd = -1; + + private SelectedBlockInput(SeekableInputStream input, Selection selected) { + this.input = input; + this.selected = selected; + } + + @Override + public int read() throws IOException { + byte[] one = new byte[1]; + return read(one, 0, 1) < 0 ? -1 : one[0] & 255; + } + + @Override + public int read(byte[] bytes, int offset, int length) throws IOException { + if (length == 0) { + return 0; + } + if (headerPosition < selected.header.length) { + int n = Math.min(length, selected.header.length - headerPosition); + System.arraycopy(selected.header, headerPosition, bytes, offset, n); + headerPosition += n; + return n; + } + if (remaining == 0) { + if (blockPosition == selected.blocks.size()) { + return -1; + } + Block block = selected.blocks.get(blockPosition++); + if (block.offset != previousEnd) { + input.seek(block.offset); + } + previousEnd = block.offset + block.length; + remaining = block.length; + } + int n = input.read(bytes, offset, (int) Math.min(length, remaining)); + if (n < 0) { + throw new EOFException("Truncated manifest block"); + } + remaining -= n; + return n; + } + + @Override + public void close() throws IOException { + input.close(); + } + } + + private static void require(boolean valid) throws IOException { + if (!valid) { + throw new IOException( + "Invalid, unsupported, mismatched or over-budget manifest row-id block index"); + } + } + + private static byte[] digest(byte[] bytes) { + try { + return MessageDigest.getInstance("SHA-256").digest(bytes); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException(e); + } + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreScan.java b/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreScan.java index a9ef5902ec9e..934c6151e5cb 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreScan.java @@ -29,6 +29,7 @@ import org.apache.paimon.manifest.ManifestEntrySerializer; import org.apache.paimon.manifest.ManifestFile; import org.apache.paimon.manifest.ManifestFileMeta; +import org.apache.paimon.manifest.ManifestRowIdIndex; import org.apache.paimon.manifest.PartitionEntry; import org.apache.paimon.manifest.SimpleFileEntry; import org.apache.paimon.operation.metrics.ScanMetrics; @@ -496,13 +497,17 @@ private List readManifest( @Nullable Filter additionalFilter, @Nullable Filter additionalTFilter) { + ManifestFile manifestFile = manifestFileFactory.create(); + ManifestRowIdIndex.Selection selected = manifestFile.selectBlocks(manifest, rowRangeIndex); + if (selected != null && selected.blocks().isEmpty()) { + return Collections.emptyList(); + } Filter entryRowFilter = createEntryRowFilter(); Function finalConverter = dropStats ? e -> converter.apply(dropStats(e)) : converter; List entries = - manifestFileFactory - .create() + manifestFile .withCacheMetrics( scanMetrics != null ? scanMetrics.getCacheMetrics() : null) .read( @@ -516,7 +521,8 @@ private List readManifest( && (manifestEntryFilter == null || manifestEntryFilter.test(entry)) && filterByStats(entry), - finalConverter); + finalConverter, + selected); LOG.info("Read {} manifest entries from {}", entries.size(), manifest.fileName()); return entries; } diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/ChangelogDeletion.java b/paimon-core/src/main/java/org/apache/paimon/operation/ChangelogDeletion.java index 9689f272e2eb..eec6090a3635 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/ChangelogDeletion.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/ChangelogDeletion.java @@ -26,7 +26,6 @@ import org.apache.paimon.manifest.ExpireFileEntry; import org.apache.paimon.manifest.IndexManifestEntry; import org.apache.paimon.manifest.ManifestFile; -import org.apache.paimon.manifest.ManifestFileMeta; import org.apache.paimon.manifest.ManifestList; import org.apache.paimon.stats.StatsFileHandler; import org.apache.paimon.utils.FileStorePathFactory; @@ -100,17 +99,17 @@ public Set manifestSkippingSet(List skippingSnapshots) { // base manifests if (manifestList.exists(skippingSnapshot.baseManifestList())) { skippingSet.add(skippingSnapshot.baseManifestList()); - manifestList.read(skippingSnapshot.baseManifestList()).stream() - .map(ManifestFileMeta::fileName) - .forEach(skippingSet::add); + manifestList + .read(skippingSnapshot.baseManifestList()) + .forEach(manifest -> addManifestToSkippingSet(skippingSet, manifest)); } // delta manifests if (manifestList.exists(skippingSnapshot.deltaManifestList())) { skippingSet.add(skippingSnapshot.deltaManifestList()); - manifestList.read(skippingSnapshot.deltaManifestList()).stream() - .map(ManifestFileMeta::fileName) - .forEach(skippingSet::add); + manifestList + .read(skippingSnapshot.deltaManifestList()) + .forEach(manifest -> addManifestToSkippingSet(skippingSet, manifest)); } // index manifests diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/FileDeletionBase.java b/paimon-core/src/main/java/org/apache/paimon/operation/FileDeletionBase.java index a0545c87e484..56afbdcdb3de 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/FileDeletionBase.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/FileDeletionBase.java @@ -326,6 +326,9 @@ protected void collectUnusedManifestList( String fileName = manifest.fileName(); if (skippingSet.add(fileName)) { manifests.add(fileName); + if (manifest.indexFileName() != null && skippingSet.add(manifest.indexFileName())) { + manifests.add(manifest.indexFileName()); + } } } if (skippingSet.add(manifestName)) { @@ -486,9 +489,9 @@ private Set manifestSkippingSet(Snapshot skippingSnapshot) { // data manifests skippingSet.add(skippingSnapshot.baseManifestList()); skippingSet.add(skippingSnapshot.deltaManifestList()); - manifestList.readDataManifests(skippingSnapshot).stream() - .map(ManifestFileMeta::fileName) - .forEach(skippingSet::add); + manifestList + .readDataManifests(skippingSnapshot) + .forEach(manifest -> addManifestToSkippingSet(skippingSet, manifest)); // index manifests String indexManifest = skippingSnapshot.indexManifest(); @@ -508,6 +511,14 @@ private Set manifestSkippingSet(Snapshot skippingSnapshot) { return skippingSet; } + protected static void addManifestToSkippingSet( + Set skippingSet, ManifestFileMeta manifest) { + skippingSet.add(manifest.fileName()); + if (manifest.indexFileName() != null) { + skippingSet.add(manifest.indexFileName()); + } + } + private boolean tryDeleteEmptyDirectory(Path path) { try { fileIO.delete(path, false); diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileMerger.java b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileMerger.java index e3f8c7af7671..fb6e7aec89ec 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileMerger.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileMerger.java @@ -79,7 +79,7 @@ public static List merge( // exception occurs, clean up and rethrow for (ManifestFileMeta manifest : newFilesForAbort) { try { - manifestFile.delete(manifest.fileName()); + manifestFile.delete(manifest); } catch (Throwable cleanupFailure) { primaryFailure = ExceptionUtils.firstOrSuppressed(cleanupFailure, primaryFailure); diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/OrphanFilesClean.java b/paimon-core/src/main/java/org/apache/paimon/operation/OrphanFilesClean.java index 4245460225ae..a90e63cc9e5e 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/OrphanFilesClean.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/OrphanFilesClean.java @@ -309,6 +309,9 @@ protected void collectWithoutDataFileWithManifestFlag( // collect manifests for (ManifestFileMeta manifest : manifestFileMetas) { usedFileWithFlagConsumer.accept(Pair.of(manifest.fileName(), true)); + if (manifest.indexFileName() != null) { + usedFileWithFlagConsumer.accept(Pair.of(manifest.indexFileName(), false)); + } } // index files diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/commit/CommitCleaner.java b/paimon-core/src/main/java/org/apache/paimon/operation/commit/CommitCleaner.java index a24b5c4c6e9b..735a706937cb 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/commit/CommitCleaner.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/commit/CommitCleaner.java @@ -52,14 +52,14 @@ public void cleanUpReuseTmpManifests( String newIndexManifest) { if (deltaManifestList != null) { for (ManifestFileMeta manifest : manifestList.read(deltaManifestList.getKey())) { - manifestFile.delete(manifest.fileName()); + manifestFile.delete(manifest); } manifestList.delete(deltaManifestList.getKey()); } if (changelogManifestList != null) { for (ManifestFileMeta manifest : manifestList.read(changelogManifestList.getKey())) { - manifestFile.delete(manifest.fileName()); + manifestFile.delete(manifest); } manifestList.delete(changelogManifestList.getKey()); } @@ -80,7 +80,7 @@ public void cleanUpNoReuseTmpManifests( .collect(Collectors.toSet()); for (ManifestFileMeta suspect : mergeAfterManifests) { if (!oldMetaSet.contains(suspect.fileName())) { - manifestFile.delete(suspect.fileName()); + manifestFile.delete(suspect); } } } diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaSerializerTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaSerializerTest.java index 57b5a08ed0fc..b0d23d0a26f0 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaSerializerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaSerializerTest.java @@ -18,6 +18,7 @@ package org.apache.paimon.manifest; +import org.apache.paimon.data.GenericRow; import org.apache.paimon.utils.ObjectSerializer; import org.apache.paimon.utils.ObjectSerializerTestBase; @@ -26,6 +27,7 @@ import java.util.ArrayList; import java.util.List; +import static org.apache.paimon.manifest.ManifestIndexTestUtils.withIndexFileName; import static org.assertj.core.api.Assertions.assertThat; /** Tests for {@link ManifestFileMetaSerializer}. */ @@ -40,6 +42,34 @@ void testFormatIdentifier() { assertThat(new ManifestFileMetaSerializer().toRow(object()).getInt(0)).isEqualTo(2); } + @Test + void testIndexReferenceRoundTripAndEquality() throws Exception { + ManifestFileMeta meta = object(); + ManifestFileMeta indexed = withIndexFileName(meta, "independent-index"); + ManifestFileMetaSerializer serializer = new ManifestFileMetaSerializer(); + assertThat(serializer.fromRow(serializer.toRow(indexed))).isEqualTo(indexed); + assertThat(serializer.deserializeFromBytes(serializer.serializeToBytes(indexed))) + .isEqualTo(indexed); + assertThat(indexed).isNotEqualTo(meta); + assertThat(indexed.hashCode()) + .isEqualTo(withIndexFileName(meta, "independent-index").hashCode()); + assertThat(indexed.toString()).contains("independent-index"); + assertThat(serializer.fromRow(serializer.toRow(meta)).indexFileName()).isNull(); + } + + @Test + void testOldRowWithoutIndexField() { + ManifestFileMeta meta = object(); + ManifestFileMetaSerializer serializer = new ManifestFileMetaSerializer(); + GenericRow current = (GenericRow) serializer.toRow(meta); + GenericRow legacy = new GenericRow(current.getFieldCount() - 1); + for (int i = 0; i < legacy.getFieldCount(); i++) { + legacy.setField(i, current.getField(i)); + } + assertThat(serializer.fromRow(legacy)).isEqualTo(meta); + assertThat(serializer.fromRow(legacy).indexFileName()).isNull(); + } + @Override protected ObjectSerializer serializer() { return new ManifestFileMetaSerializer(); diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java index c3a50f4ef1de..18284bb4cd94 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java @@ -28,13 +28,19 @@ import org.apache.paimon.fs.FileIOFinder; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.PositionOutputStream; +import org.apache.paimon.fs.SeekableInputStream; +import org.apache.paimon.fs.SeekableInputStreamWrapper; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.io.DataFileMetaWriteColsLegacySerializer; +import org.apache.paimon.operation.AppendOnlyFileStoreScan; +import org.apache.paimon.operation.ManifestsReader; +import org.apache.paimon.operation.commit.CommitCleaner; import org.apache.paimon.options.MemorySize; import org.apache.paimon.options.Options; import org.apache.paimon.partition.PartitionPredicate; import org.apache.paimon.schema.FileSystemSchemaManager; +import org.apache.paimon.schema.TableSchema; import org.apache.paimon.stats.StatsTestUtils; import org.apache.paimon.types.DataField; import org.apache.paimon.types.RowType; @@ -42,6 +48,9 @@ import org.apache.paimon.utils.FailingFileIO; import org.apache.paimon.utils.FileStorePathFactory; import org.apache.paimon.utils.Filter; +import org.apache.paimon.utils.Pair; +import org.apache.paimon.utils.Range; +import org.apache.paimon.utils.RowRangeIndex; import org.apache.paimon.utils.SegmentsCache; import org.junit.jupiter.api.RepeatedTest; @@ -63,13 +72,18 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; import java.util.stream.Stream; import static org.apache.paimon.TestKeyValueGenerator.DEFAULT_PART_TYPE; +import static org.apache.paimon.manifest.ManifestIndexTestUtils.withIndexFileName; import static org.apache.paimon.stats.StatsTestUtils.convertWithoutSchemaEvolution; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; /** Tests for {@link ManifestFile}. */ public class ManifestFileTest { @@ -1224,16 +1238,433 @@ private static int indexOf(byte[] bytes, byte[] target, int from, int limit) { return -1; } + @Test + void testRowIdSidecarRollingRawRewriteAndDelete() throws Exception { + Options options = new Options(); + options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_WRITE, true); + options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_READ, true); + ManifestFile manifests = createManifestFile(tempDir.toString(), 1, options); + List entries = new ArrayList<>(); + for (int i = 0; i < 2200; i++) { + ManifestEntry source = gen.next(); + entries.add( + ManifestEntry.create( + i % 2 == 0 ? FileKind.ADD : FileKind.DELETE, + source.partition(), + source.bucket(), + source.totalBuckets(), + source.file().newFirstRowId(i * 100000000L))); + } + List metas = manifests.write(entries); + assertThat(metas.size()).isGreaterThan(1); + + for (ManifestFileMeta meta : metas) { + assertThat(meta.indexFileName()).isEqualTo(meta.fileName() + ManifestRowIdIndex.SUFFIX); + List actual = manifests.read(meta.fileName()); + for (ManifestEntry entry : + Arrays.asList(actual.get(0), actual.get(actual.size() - 1))) { + assertThat( + manifests.mayContainRowIds( + meta, + RowRangeIndex.create( + Collections.singletonList( + new Range( + entry.file().firstRowId(), + entry.file().firstRowId()))))) + .isTrue(); + } + long gap = actual.get(0).file().firstRowId() + actual.get(0).file().rowCount(); + assertThat( + manifests.mayContainRowIds( + meta, + RowRangeIndex.create( + Collections.singletonList(new Range(gap, gap))))) + .isFalse(); + assertThat( + java.nio.file.Files.exists( + tempDir.resolve("manifest") + .resolve(meta.fileName() + ManifestRowIdIndex.SUFFIX))) + .isTrue(); + } + ManifestFileMeta source = metas.get(0); + ManifestAvroWriter rewrite = manifests.createAvroWriter(); + try (ManifestAvroReader reader = + manifests.scanAvroBlocks(source.fileName(), source.fileSize())) { + rewrite.writeEncodedManifest(reader, source); + } + rewrite.close(); + ManifestFileMeta rewritten = rewrite.result().get(0); + assertThat(rewritten.indexFileName()) + .isEqualTo(rewritten.fileName() + ManifestRowIdIndex.SUFFIX); + assertThat(manifests.read(rewritten.fileName())) + .isEqualTo(manifests.read(source.fileName())); + long outside = metas.get(metas.size() - 1).maxRowId(); + assertThat( + manifests.mayContainRowIds( + rewritten, + RowRangeIndex.create( + Collections.singletonList(new Range(outside, outside))))) + .isFalse(); + rewrite.abort(); + assertThat( + java.nio.file.Files.exists( + tempDir.resolve("manifest") + .resolve(rewritten.fileName() + ManifestRowIdIndex.SUFFIX))) + .isFalse(); + for (ManifestFileMeta meta : metas) { + manifests.delete(meta); + assertThat( + java.nio.file.Files.exists( + tempDir.resolve("manifest") + .resolve(meta.fileName() + ManifestRowIdIndex.SUFFIX))) + .isFalse(); + } + } + + @Test + void testReadsOnlySelectedBlocksAndPreservesPhysicalOrdinals() throws Exception { + Options options = new Options(); + options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_WRITE, true); + options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_READ, true); + RecordingFileIO fileIO = new RecordingFileIO(); + ManifestFile.Factory factory = + createManifestFileFactory(tempDir.toString(), Long.MAX_VALUE, options, fileIO); + ManifestFile manifests = factory.create(); + List entries = new ArrayList<>(); + for (int i = 0; i < 4000; i++) { + ManifestEntry entry = gen.next(); + entries.add( + ManifestEntry.create( + FileKind.ADD, + entry.partition(), + entry.bucket(), + entry.totalBuckets(), + entry.file().newFirstRowId(i * 1000000L))); + } + ManifestFileMeta meta = manifests.write(entries).get(0); + RowRangeIndex query = + RowRangeIndex.create( + Arrays.asList( + new Range(1000000000L, 1000000000L), + new Range(3000000000L, 3000000000L))); + + ManifestRowIdIndex.Selection selected = manifests.selectBlocks(meta, query); + assertThat(selected.blocks()).hasSize(2); + + fileIO.reset(); + List actual = + factory.create() + .read( + meta.fileName(), + meta.fileSize(), + null, + null, + row -> true, + entry -> true, + java.util.function.Function.identity(), + selected); + List expected = new ArrayList<>(); + for (ManifestRowIdIndex.Block block : selected.blocks()) { + expected.addAll( + entries.subList( + (int) block.firstRecord, + (int) (block.firstRecord + block.recordCount))); + } + assertThat(actual).containsExactlyElementsOf(expected); + assertThat(actual).contains(entries.get(1000), entries.get(3000)); + assertThat(fileIO.bytes.get()).isLessThan(meta.fileSize() / 4); + assertThat(fileIO.seeks) + .containsExactlyElementsOf( + selected.blocks().stream() + .map(block -> block.offset) + .collect(Collectors.toList())); + assertThat(fileIO.opened) + .containsExactly(new Path(tempDir.toString(), "manifest/" + meta.fileName())); + // Block selections cannot populate the ordinary full-manifest read cache. + assertThat(manifests.read(meta.fileName())).containsExactlyElementsOf(entries); + } + + @Test + void testScannerPreservesDeletesAndColumnGroups() { + Options options = new Options(); + options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_WRITE, true); + options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_READ, true); + RecordingFileIO fileIO = new RecordingFileIO(); + ManifestFile.Factory factory = + createManifestFileFactory(tempDir.toString(), Long.MAX_VALUE, options, fileIO); + ManifestFile manifests = factory.create(); + ManifestEntry entry = gen.next(); + ManifestEntry add = + ManifestEntry.create( + FileKind.ADD, + entry.partition(), + entry.bucket(), + entry.totalBuckets(), + entry.file().newFirstRowId(100L)); + ManifestEntry delete = + ManifestEntry.create( + FileKind.DELETE, + entry.partition(), + entry.bucket(), + entry.totalBuckets(), + add.file()); + ManifestEntry other = gen.next(); + ManifestEntry live = + ManifestEntry.create( + FileKind.ADD, + other.partition(), + other.bucket(), + other.totalBuckets(), + other.file().newFirstRowId(100L)); + List metas = new ArrayList<>(); + metas.addAll(manifests.write(Arrays.asList(add, live))); + metas.addAll(manifests.write(Collections.singletonList(delete))); + metas.addAll( + manifests.write( + Collections.singletonList( + ManifestEntry.create( + FileKind.ADD, + entry.partition(), + entry.bucket(), + entry.totalBuckets(), + entry.file().newFirstRowId(0L))))); + AppendOnlyFileStoreScan scan = + new AppendOnlyFileStoreScan( + mock(ManifestsReader.class), + null, + null, + null, + mock(TableSchema.class), + factory, + 2, + false, + false, + false); + scan.withRowRanges(Collections.singletonList(new Range(100, 100))); + fileIO.reset(); + List result = new ArrayList<>(); + scan.readManifestEntries(metas, false).forEachRemaining(result::add); + assertThat(result).containsExactly(live); + assertThat( + fileIO.opened.stream() + .filter(path -> !path.getName().endsWith(ManifestRowIdIndex.SUFFIX)) + .map(Path::getName)) + .containsExactlyInAnyOrder(metas.get(0).fileName(), metas.get(1).fileName()); + } + + @Test + void testSidecarWriteFailureAbortsAllRollingOutputs() { + Options options = new Options(); + options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_WRITE, true); + AtomicInteger indexes = new AtomicInteger(); + FileIO failing = + new LocalFileIO() { + @Override + public PositionOutputStream newOutputStream(Path path, boolean overwrite) + throws IOException { + if (path.getName().endsWith(ManifestRowIdIndex.SUFFIX) + && indexes.incrementAndGet() == 2) { + throw new IOException("sidecar write failed"); + } + return super.newOutputStream(path, overwrite); + } + }; + ManifestFile manifests = + createManifestFileFactory(tempDir.toString(), 1, options, failing).create(); + List entries = new ArrayList<>(); + for (int i = 0; i < 2200; i++) { + ManifestEntry entry = gen.next(); + entries.add( + ManifestEntry.create( + FileKind.ADD, + entry.partition(), + entry.bucket(), + entry.totalBuckets(), + entry.file().newFirstRowId(0L))); + } + assertThatThrownBy(() -> manifests.write(entries)) + .hasRootCauseMessage("sidecar write failed"); + try (java.util.stream.Stream files = + java.nio.file.Files.list(tempDir.resolve("manifest"))) { + assertThat(files).isEmpty(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Test + void testUnknownRowIdDisablesIndexAndNoQueryDoesNotReadSidecar() { + Options options = new Options(); + options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_WRITE, true); + options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_READ, true); + RecordingFileIO fileIO = new RecordingFileIO(); + ManifestFile manifests = + createManifestFileFactory(tempDir.toString(), Long.MAX_VALUE, options, fileIO) + .create(); + ManifestFileMeta meta = manifests.write(Collections.singletonList(gen.next())).get(0); + assertThat(meta.indexFileName()).isNull(); + assertThat( + java.nio.file.Files.exists( + tempDir.resolve("manifest") + .resolve(meta.fileName() + ManifestRowIdIndex.SUFFIX))) + .isFalse(); + + fileIO.reset(); + assertThat(manifests.mayContainRowIds(meta, null)).isTrue(); + assertThat(fileIO.opened).isEmpty(); + } + + @Test + void testExplicitIndexReferenceAndNullDoesNotProbe() throws Exception { + Options options = new Options(); + options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_WRITE, true); + options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_READ, true); + RecordingFileIO io = new RecordingFileIO(); + ManifestFile manifests = + createManifestFileFactory(tempDir.toString(), Long.MAX_VALUE, options, io).create(); + ManifestEntry original = gen.next(); + ManifestEntry entry = + ManifestEntry.create( + FileKind.ADD, + original.partition(), + original.bucket(), + original.totalBuckets(), + original.file().newFirstRowId(100L)); + ManifestFileMeta written = manifests.write(Collections.singletonList(entry)).get(0); + assertThat(written.indexFileName()).isNotNull(); + RowRangeIndex query = RowRangeIndex.create(Collections.singletonList(new Range(0, 0))); + io.reset(); + ManifestFileMeta unindexed = withIndexFileName(written, null); + assertThat(manifests.selectBlocks(unindexed, query)).isNull(); + assertThat(io.opened).isEmpty(); + // An existing suffix-named object must not be inferred as a reference. + assertThat( + java.nio.file.Files.exists( + tempDir.resolve("manifest").resolve(written.indexFileName()))) + .isTrue(); + String explicitName = "custom-index-name"; + java.nio.file.Files.move( + tempDir.resolve("manifest").resolve(written.indexFileName()), + tempDir.resolve("manifest").resolve(explicitName)); + ManifestFileMeta indexed = withIndexFileName(written, explicitName); + assertThat(manifests.selectBlocks(indexed, query).blocks()).isEmpty(); + assertThat(io.opened) + .containsExactly(new Path(tempDir.toString(), "manifest/" + explicitName)); + manifests.delete(indexed); + assertThat(java.nio.file.Files.exists(tempDir.resolve("manifest").resolve(explicitName))) + .isFalse(); + assertThat( + java.nio.file.Files.exists( + tempDir.resolve("manifest").resolve(written.fileName()))) + .isFalse(); + } + + @Test + void testCommitCleanerDeletesExplicitIndexReferences() throws Exception { + ManifestFile manifests = createManifestFile(tempDir.toString(), Long.MAX_VALUE); + ManifestList lists = mock(ManifestList.class); + CommitCleaner cleaner = new CommitCleaner(lists, manifests, mock(IndexManifestFile.class)); + for (int mode = 0; mode < 2; mode++) { + ManifestFileMeta meta = manifests.write(Collections.singletonList(gen.next())).get(0); + String indexName = "commit-index-" + mode; + Path indexPath = new Path(tempDir.toString(), "manifest/" + indexName); + LocalFileIO.create().newOutputStream(indexPath, false).close(); + ManifestFileMeta indexed = withIndexFileName(meta, indexName); + if (mode == 0) { + when(lists.read("delta-list")).thenReturn(Collections.singletonList(indexed)); + cleaner.cleanUpReuseTmpManifests(Pair.of("delta-list", 1L), null, null, null); + } else { + cleaner.cleanUpNoReuseTmpManifests( + null, Collections.emptyList(), Collections.singletonList(indexed)); + } + assertThat(LocalFileIO.create().exists(indexPath)).isFalse(); + assertThat( + java.nio.file.Files.exists( + tempDir.resolve("manifest").resolve(meta.fileName()))) + .isFalse(); + } + } + + /** Observes actual file access without adding counters to production readers. */ + private static final class RecordingFileIO extends LocalFileIO { + private final List opened = Collections.synchronizedList(new ArrayList<>()); + private final List seeks = Collections.synchronizedList(new ArrayList<>()); + private final AtomicLong bytes = new AtomicLong(); + + private void reset() { + opened.clear(); + seeks.clear(); + bytes.set(0); + } + + @Override + public SeekableInputStream newInputStream(Path path) throws IOException { + opened.add(path); + return new SeekableInputStreamWrapper(super.newInputStream(path)) { + @Override + public void seek(long desired) throws IOException { + seeks.add(desired); + super.seek(desired); + } + + @Override + public int read() throws IOException { + int value = super.read(); + if (value >= 0) { + bytes.incrementAndGet(); + } + return value; + } + + @Override + public int read(byte[] buffer, int offset, int length) throws IOException { + int n = super.read(buffer, offset, length); + if (n > 0) { + bytes.addAndGet(n); + } + return n; + } + }; + } + } + private ManifestFile createManifestFile(String pathStr) { return createManifestFile(pathStr, ThreadLocalRandom.current().nextInt(8192) + 1024); } private ManifestFile createManifestFile(String pathStr, long suggestedFileSize) { - return createManifestFile(pathStr, suggestedFileSize, null); + return createManifestFile(pathStr, suggestedFileSize, new Options()); } private ManifestFile createManifestFile( String pathStr, long suggestedFileSize, @Nullable SegmentsCache cache) { + return createManifestFileFactory( + pathStr, + suggestedFileSize, + new Options(), + FileIOFinder.find(new Path(pathStr)), + cache) + .create(); + } + + private ManifestFile createManifestFile( + String pathStr, long suggestedFileSize, Options options) { + return createManifestFileFactory( + pathStr, suggestedFileSize, options, FileIOFinder.find(new Path(pathStr))) + .create(); + } + + private ManifestFile.Factory createManifestFileFactory( + String pathStr, long suggestedFileSize, Options options, FileIO fileIO) { + return createManifestFileFactory(pathStr, suggestedFileSize, options, fileIO, null); + } + + private ManifestFile.Factory createManifestFileFactory( + String pathStr, + long suggestedFileSize, + Options options, + FileIO fileIO, + @Nullable SegmentsCache cache) { Path path = new Path(pathStr); FileStorePathFactory pathFactory = new FileStorePathFactory( @@ -1252,7 +1683,6 @@ private ManifestFile createManifestFile( null, false, null); - FileIO fileIO = FileIOFinder.find(path); return new ManifestFile.Factory( fileIO, new FileSystemSchemaManager(fileIO, path), @@ -1262,7 +1692,7 @@ private ManifestFile createManifestFile( pathFactory, suggestedFileSize, cache) - .create(); + .withRowIdIndexOptions(options); } @Test diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestIndexTestUtils.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestIndexTestUtils.java new file mode 100644 index 000000000000..6159fcce6452 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestIndexTestUtils.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.manifest; + +import org.apache.paimon.FileStore; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; +import org.apache.paimon.utils.JsonSerdeUtil; +import org.apache.paimon.utils.Pair; +import org.apache.paimon.utils.SnapshotManager; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** Synthetic index references for manifest serialization and lifecycle tests. */ +public final class ManifestIndexTestUtils { + private ManifestIndexTestUtils() {} + + public static ManifestFileMeta withIndexFileName(ManifestFileMeta meta, String indexFileName) { + return new ManifestFileMeta( + meta.fileName(), + meta.fileSize(), + meta.numAddedFiles(), + meta.numDeletedFiles(), + meta.partitionStats(), + meta.schemaId(), + meta.minBucket(), + meta.maxBucket(), + meta.minLevel(), + meta.maxLevel(), + meta.minRowId(), + meta.maxRowId(), + indexFileName); + } + + /** Replaces only synthetic snapshot fixtures, using newly written manifest lists. */ + public static void registerIndexReferences(FileStore store, long snapshotId) + throws IOException { + SnapshotManager manager = store.snapshotManager(); + FileIO io = manager.fileIO(); + Path snapshotPath = manager.snapshotPath(snapshotId); + ObjectNode snapshot = + (ObjectNode) + JsonSerdeUtil.OBJECT_MAPPER_INSTANCE.readTree( + io.readFileUtf8(snapshotPath)); + ManifestList lists = store.manifestListFactory().create(); + for (String field : + new String[] {"baseManifestList", "deltaManifestList", "changelogManifestList"}) { + JsonNode value = snapshot.get(field); + if (value == null || value.isNull()) { + continue; + } + List indexed = new ArrayList<>(); + for (ManifestFileMeta meta : lists.read(value.asText())) { + // Deliberately use a name which cannot be derived by appending the sidecar suffix. + String name = "index-for-" + meta.fileName(); + Path index = store.pathFactory().toManifestFilePath(name); + if (!io.exists(index)) { + // GC treats index bytes as opaque; unsupported/partial files are still owned. + io.newOutputStream(index, false).close(); + } + indexed.add(withIndexFileName(meta, name)); + } + Pair replacement = lists.write(indexed); + snapshot.put(field, replacement.getLeft()); + snapshot.put(field + "Size", replacement.getRight()); + } + io.overwriteFileUtf8( + snapshotPath, JsonSerdeUtil.OBJECT_MAPPER_INSTANCE.writeValueAsString(snapshot)); + manager.invalidateCache(); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestListTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestListTest.java index 8442be28c640..98cfbcb98954 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestListTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestListTest.java @@ -156,7 +156,11 @@ private List generateData() { for (int j = random.nextInt(10) + 1; j > 0; j--) { entries.add(gen.next()); } - metas.add(gen.createManifestFileMeta(entries)); + ManifestFileMeta meta = gen.createManifestFileMeta(entries); + metas.add( + i % 2 == 0 + ? ManifestIndexTestUtils.withIndexFileName(meta, "index-" + i) + : meta); } return metas; } diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestRowIdIndexTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestRowIdIndexTest.java new file mode 100644 index 000000000000..35da1891e562 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestRowIdIndexTest.java @@ -0,0 +1,344 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.manifest; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.options.Options; +import org.apache.paimon.utils.Range; +import org.apache.paimon.utils.RowRangeIndex; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.security.MessageDigest; +import java.util.Arrays; +import java.util.Base64; +import java.util.Collections; +import java.util.Properties; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** Cross-language format, physical block positions, completeness and allocation bounds. */ +class ManifestRowIdIndexTest { + @TempDir java.nio.file.Path temp; + private final ManifestRowIdIndex.Settings settings = + new ManifestRowIdIndex.Settings(new Options()); + + static ManifestFileMeta meta(String name, long size, long entries) { + ManifestFileMeta meta = mock(ManifestFileMeta.class); + when(meta.fileName()).thenReturn(name); + when(meta.fileSize()).thenReturn(size); + when(meta.indexFileName()).thenReturn(name + ManifestRowIdIndex.SUFFIX); + when(meta.numAddedFiles()).thenReturn(entries); + return meta; + } + + private Properties fixture() throws IOException { + Properties properties = new Properties(); + try (java.io.InputStream input = + getClass().getResourceAsStream("/manifest-row-id-index-v2.txt")) { + properties.load(input); + } + return properties; + } + + private byte[] header() throws IOException { + return Base64.getDecoder().decode(fixture().getProperty("avroHeader")); + } + + private byte[] golden() throws IOException { + return Base64.getDecoder().decode(fixture().getProperty("index")); + } + + private ManifestFileMeta goldenMeta() throws IOException { + return meta("manifest-golden", header().length + 400, 7); + } + + @Test + void crossLanguageFormatAndBlockOrdinals() throws Exception { + byte[] header = header(); + ManifestRowIdIndex.Builder builder = new ManifestRowIdIndex.Builder(settings, header); + builder.beginBlock(header.length, 100, 3); + builder.add(0L, 10); + builder.add(5L, 5); + builder.add(20L, 5); + builder.endBlock(); + builder.beginBlock(header.length + 100, 200, 2); + builder.add((1L << 32) - 2, 5); + builder.add(8254058425445L, 1); + builder.endBlock(); + builder.beginBlock(header.length + 300, 100, 2); + builder.add(20L, 5); + builder.add(Long.MAX_VALUE, 1); + builder.endBlock(); + byte[] data = builder.serialize("manifest-golden", header.length + 400, 7); + assertThat(data).isEqualTo(golden()); + ManifestFileMeta meta = goldenMeta(); + for (long point : + new long[] { + 0, + 9, + 20, + 24, + (1L << 32) - 2, + 1L << 32, + (1L << 32) + 2, + 8254058425445L, + Long.MAX_VALUE + }) { + assertThat(select(data, meta, point).blocks()).as("row %s", point).isNotEmpty(); + } + for (long point : + new long[] { + 10, 19, 25, (1L << 32) - 3, (1L << 32) + 3, 8254058425444L, Long.MAX_VALUE - 1 + }) { + assertThat(select(data, meta, point).blocks()).as("row %s", point).isEmpty(); + } + ManifestRowIdIndex.Selection selected = select(data, meta, 20); + assertThat(selected.blocks()).extracting(b -> b.firstRecord).containsExactly(0L, 5L); + assertThat(selected.blocks()) + .extracting(b -> b.offset) + .containsExactly((long) header.length, header.length + 300L); + assertThat(selected.blocks()).extracting(b -> b.length).containsExactly(100L, 100L); + + ManifestRowIdIndex.Selection gap = select(data, meta, 16); + + assertThat(gap.blocks()).isEmpty(); + RowRangeIndex query = + RowRangeIndex.create(Arrays.asList(new Range(10, 19), new Range(25, 40))); + assertThat(ManifestRowIdIndex.select(data, meta, query, settings).blocks()).isEmpty(); + assertThat(query.ranges()).containsExactly(new Range(10, 19), new Range(25, 40)); + } + + @Test + void minMaxSkipsExactIntersectionChecksAndHandlesOneInterval() throws Exception { + byte[] header = header(); + ManifestRowIdIndex.Builder builder = new ManifestRowIdIndex.Builder(settings, header); + builder.beginBlock(header.length, 100, 2); + builder.add(0L, 10); + builder.add(20L, 10); + builder.endBlock(); + builder.beginBlock(header.length + 100, 100, 2); + builder.add(100L, 10); + builder.add(200L, 10); + builder.endBlock(); + builder.beginBlock(header.length + 200, 100, 1); + builder.add(1L << 32, 10); + builder.endBlock(); + byte[] data = builder.serialize("m", header.length + 300, 5); + ManifestFileMeta meta = meta("m", header.length + 300, 5); + RowRangeIndex outside = + spy(RowRangeIndex.create(Collections.singletonList(new Range(50, 59)))); + ManifestRowIdIndex.Selection none = + ManifestRowIdIndex.select(data, meta, outside, settings); + assertThat(none.blocks()).isEmpty(); + + // Only the three envelopes are tested; no individual interval intersection is evaluated. + verify(outside, times(3)).intersects(anyLong(), anyLong()); + verify(outside).intersects(0, 29); + verify(outside).intersects(100, 209); + verify(outside).intersects(1L << 32, (1L << 32) + 9); + + RowRangeIndex one = + spy( + RowRangeIndex.create( + Collections.singletonList( + new Range((1L << 32) + 9, (1L << 32) + 9)))); + ManifestRowIdIndex.Selection hit = ManifestRowIdIndex.select(data, meta, one, settings); + assertThat(hit.blocks()).extracting(b -> b.firstRecord).containsExactly(4L); + + // A one-interval block needs no second intersection check after its envelope matches. + verify(one, times(3)).intersects(anyLong(), anyLong()); + } + + @Test + void malformedIntervalsStillFallbackAfterMinMaxRejectionOrAnEarlyHit() throws Exception { + byte[] data = golden(); + int firstBlockIntervals = 68 + 4 + header().length + 4 + 36; + // Make the second interval overlap the first, keeping the envelope unchanged. + ByteBuffer.wrap(data).putLong(firstBlockIntervals + 16, 9L); + byte[] hash = + MessageDigest.getInstance("SHA-256").digest(Arrays.copyOf(data, data.length - 32)); + System.arraycopy(hash, 0, data, data.length - 32, 32); + Files.write(temp.resolve("manifest-golden" + ManifestRowIdIndex.SUFFIX), data); + ManifestFileMeta meta = goldenMeta(); + + for (long point : new long[] {30, 0}) { + RowRangeIndex query = + RowRangeIndex.create(Collections.singletonList(new Range(point, point))); + assertThat( + ManifestRowIdIndex.read( + LocalFileIO.create(), + new Path(temp.toString(), "manifest-golden"), + meta, + query, + settings)) + .isNull(); + } + } + + @Test + void hugeRangesAreNotExpandedAndInvalidCoverageDisablesIndex() throws Exception { + byte[] header = header(); + ManifestRowIdIndex.Builder builder = new ManifestRowIdIndex.Builder(settings, header); + builder.beginBlock(header.length, 100, 2); + builder.add(0L, Long.MAX_VALUE); + builder.add(Long.MAX_VALUE, 1); + builder.endBlock(); + byte[] data = builder.serialize("m", header.length + 100, 2); + assertThat(data.length).isLessThan(512); + assertThat(select(data, meta("m", header.length + 100, 2), Long.MAX_VALUE).blocks()) + .hasSize(1); + for (Long first : Arrays.asList(null, -1L, Long.MAX_VALUE)) { + builder = new ManifestRowIdIndex.Builder(settings, header); + builder.beginBlock(header.length, 100, 1); + builder.add(first, 2); + builder.endBlock(); + assertThat(builder.serialize("m", header.length + 100, 1)).isNull(); + } + for (long count : new long[] {0, -1}) { + builder = new ManifestRowIdIndex.Builder(settings, header); + builder.beginBlock(header.length, 100, 1); + builder.add(0L, count); + assertThat(builder.serialize("m", 1, 1)).isNull(); + } + Options options = new Options(); + options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_MAX_RANGES, 1); + builder = new ManifestRowIdIndex.Builder(new ManifestRowIdIndex.Settings(options), header); + builder.beginBlock(header.length, 100, 2); + builder.add(0L, 1); + builder.add(10L, 1); + assertThat(builder.serialize("m", 1, 2)).isNull(); + options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_MAX_BYTES, 128); + builder = new ManifestRowIdIndex.Builder(new ManifestRowIdIndex.Settings(options), header); + assertThat(builder.serialize("m", 1, 2)).isNull(); + } + + @Test + void corruptMissingIncompleteAndMismatchedIndexesFallback() throws Exception { + Path manifest = new Path(temp.toString(), "manifest-golden"); + java.nio.file.Path index = temp.resolve("manifest-golden" + ManifestRowIdIndex.SUFFIX); + ManifestFileMeta meta = goldenMeta(); + RowRangeIndex query = RowRangeIndex.create(Collections.singletonList(new Range(11, 11))); + + assertThat(ManifestRowIdIndex.read(LocalFileIO.create(), manifest, meta, query, settings)) + .isNull(); + byte[] good = golden(); + for (int position : new int[] {0, 9, 11, 15, 16, 55, 63, 67, 75, good.length - 1}) { + byte[] bad = good.clone(); + bad[position] ^= 2; + Files.write(index, bad); + assertThat( + ManifestRowIdIndex.read( + LocalFileIO.create(), manifest, meta, query, settings)) + .isNull(); + } + // Self-consistent checksum cannot turn an unsupported or incomplete envelope into an index. + for (int position : new int[] {9, 11, 15}) { + byte[] bad = good.clone(); + bad[position] = 0; + byte[] hash = + MessageDigest.getInstance("SHA-256") + .digest(Arrays.copyOf(bad, bad.length - 32)); + System.arraycopy(hash, 0, bad, bad.length - 32, 32); + assertThatThrownBy(() -> ManifestRowIdIndex.select(bad, meta, query, settings)) + .isInstanceOf(IOException.class); + } + Files.write(index, Arrays.copyOf(good, good.length - 1)); + assertThat(ManifestRowIdIndex.read(LocalFileIO.create(), manifest, meta, query, settings)) + .isNull(); + Files.write(index, good); + assertThat( + ManifestRowIdIndex.read( + LocalFileIO.create(), manifest, meta, query, settings) + .blocks()) + .isEmpty(); + assertThatThrownBy( + () -> + ManifestRowIdIndex.select( + good, meta("other", meta.fileSize(), 7), query, settings)) + .isInstanceOf(IOException.class); + } + + @Test + void ioTimeoutFallsBackButInterruptionAndFatalErrorsPropagate() { + ManifestFileMeta manifest = meta("m", 1, 1); + RowRangeIndex query = RowRangeIndex.create(Collections.singletonList(new Range(1, 1))); + Path path = new Path(temp.toString(), "m"); + + LocalFileIO timedOut = + new LocalFileIO() { + @Override + public org.apache.paimon.fs.SeekableInputStream newInputStream(Path path) + throws IOException { + throw new java.net.SocketTimeoutException("timeout"); + } + }; + assertThat(ManifestRowIdIndex.read(timedOut, path, manifest, query, settings)).isNull(); + assertThat(Thread.currentThread().isInterrupted()).isFalse(); + LocalFileIO interrupted = + new LocalFileIO() { + @Override + public org.apache.paimon.fs.SeekableInputStream newInputStream(Path path) + throws IOException { + throw new java.io.InterruptedIOException("stop"); + } + }; + try { + assertThatThrownBy( + () -> + ManifestRowIdIndex.read( + interrupted, path, manifest, query, settings)) + .isInstanceOf(java.io.UncheckedIOException.class); + assertThat(Thread.currentThread().isInterrupted()).isTrue(); + } finally { + Thread.interrupted(); + } + LocalFileIO failed = + new LocalFileIO() { + @Override + public org.apache.paimon.fs.SeekableInputStream newInputStream(Path path) { + throw new AssertionError("fatal"); + } + }; + assertThatThrownBy(() -> ManifestRowIdIndex.read(failed, path, manifest, query, settings)) + .isInstanceOf(AssertionError.class); + } + + private ManifestRowIdIndex.Selection select(byte[] data, ManifestFileMeta meta, long point) + throws IOException { + return ManifestRowIdIndex.select( + data, + meta, + RowRangeIndex.create(Collections.singletonList(new Range(point, point))), + settings); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/ExpireSnapshotsTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/ExpireSnapshotsTest.java index c5a21e57c9d9..abbbbde9a2f7 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/ExpireSnapshotsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/ExpireSnapshotsTest.java @@ -38,6 +38,7 @@ import org.apache.paimon.manifest.FileSource; import org.apache.paimon.manifest.ManifestEntry; import org.apache.paimon.manifest.ManifestFileMeta; +import org.apache.paimon.manifest.ManifestIndexTestUtils; import org.apache.paimon.mergetree.compact.DeduplicateMergeFunction; import org.apache.paimon.options.ExpireConfig; import org.apache.paimon.schema.FileSystemSchemaManager; @@ -741,6 +742,62 @@ public void testExpirePlansManifestsConcurrentlyWithSkippingSet() throws Excepti store.assertCleaned(); } + @Test + void testSidecarsFollowSnapshotAndTagRetention() throws Exception { + store.options().toConfiguration().set(CoreOptions.MANIFEST_MERGE_MIN_COUNT, 2); + List allData = new ArrayList<>(); + List snapshotPositions = new ArrayList<>(); + commit(8, allData, snapshotPositions); + int latest = requireNonNull(snapshotManager.latestSnapshotId()).intValue(); + Set manifests = new HashSet<>(); + for (int i = 1; i <= latest; i++) { + rewriteSnapshotTime(i, 0); + ManifestIndexTestUtils.registerIndexReferences(store, i); + snapshotManager.invalidateCache(); + store.manifestListFactory() + .create() + .readDataManifests(snapshotManager.snapshot(i)) + .forEach( + meta -> + manifests.add( + store.pathFactory() + .toManifestFilePath(meta.fileName()))); + } + store.newTagManager() + .createTag( + snapshotManager.snapshot(3), + "keep-sidecars", + store.options().tagDefaultTimeRetained(), + Collections.emptyList(), + false); + ExpireSnapshotsImpl expire = + (ExpireSnapshotsImpl) store.newExpire(expireAllButLatestConfig()); + expire.setCurrentTimeMillis(() -> 1000L); + expire.expire(); + boolean reclaimed = false; + for (Path manifest : manifests) { + boolean retained = fileIO.exists(manifest); + assertThat( + fileIO.exists( + new Path( + manifest.getParent(), + "index-for-" + manifest.getName()))) + .isEqualTo(retained); + reclaimed |= !retained; + } + assertThat(reclaimed).isTrue(); + for (ManifestFileMeta meta : + store.manifestListFactory() + .create() + .readDataManifests( + store.newTagManager() + .getOrThrow("keep-sidecars") + .trimToSnapshot())) { + assertThat(fileIO.exists(store.pathFactory().toManifestFilePath(meta.indexFileName()))) + .isTrue(); + } + } + @Test public void testExpireWithTagsAndConcurrentPlanningKeepsTaggedSnapshotsReadable() throws Exception { diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/LocalOrphanFilesCleanTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/LocalOrphanFilesCleanTest.java index bfe3f6d76ca8..86f8c07971a6 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/LocalOrphanFilesCleanTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/LocalOrphanFilesCleanTest.java @@ -31,7 +31,10 @@ import org.apache.paimon.fs.FileStatus; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.manifest.ManifestFileMeta; +import org.apache.paimon.manifest.ManifestIndexTestUtils; import org.apache.paimon.manifest.ManifestList; +import org.apache.paimon.manifest.ManifestRowIdIndex; import org.apache.paimon.mergetree.compact.ConcatRecordReader; import org.apache.paimon.options.Options; import org.apache.paimon.reader.ReaderSupplier; @@ -153,6 +156,39 @@ public void testNormallyRemoving() throws Throwable { normallyRemoving(tablePath); } + @Test + void testOrphanCleanupProtectsReferencedSidecars() throws Exception { + commit(Collections.singletonList(TestPojo.next())); + ManifestIndexTestUtils.registerIndexReferences( + table.store(), table.snapshotManager().latestSnapshotId()); + table.snapshotManager().invalidateCache(); + table.createTag("sidecar-tag", table.snapshotManager().latestSnapshotId()); + List sidecars = new ArrayList<>(); + List unreferenced = new ArrayList<>(); + for (ManifestFileMeta meta : + table.store() + .manifestListFactory() + .create() + .readDataManifests(table.snapshotManager().latestSnapshot())) { + Path sidecar = new Path(manifestDir, meta.indexFileName()); + sidecars.add(sidecar); + Path guessed = new Path(manifestDir, meta.fileName() + ManifestRowIdIndex.SUFFIX); + fileIO.newOutputStream(guessed, false).close(); + unreferenced.add(guessed); + } + Path orphan = new Path(manifestDir, "manifest-orphan" + ManifestRowIdIndex.SUFFIX); + fileIO.newOutputStream(orphan, false).close(); + new LocalOrphanFilesClean(table, System.currentTimeMillis() + 2000).clean(); + assertThat(fileIO.exists(orphan)).isFalse(); + assertThat(sidecars).isNotEmpty(); + for (Path sidecar : sidecars) { + assertThat(fileIO.exists(sidecar)).isTrue(); + } + for (Path guessed : unreferenced) { + assertThat(fileIO.exists(guessed)).isFalse(); + } + } + @Test public void testKeepManagedBlobPack() throws Exception { commit(Collections.singletonList(TestPojo.next())); diff --git a/paimon-core/src/test/resources/manifest-row-id-index-v2.txt b/paimon-core/src/test/resources/manifest-row-id-index-v2.txt new file mode 100644 index 000000000000..c58cf599b276 --- /dev/null +++ b/paimon-core/src/test/resources/manifest-row-id-index-v2.txt @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +avroHeader=T2JqAQQUYXZyby5jb2RlYwhudWxsFmF2cm8uc2NoZW1hDCJsb25nIgAAAAAAAAAAAAAAAAAAAAAA +index=UEFJTVJJRFgAAgACAAAAAS9Hlrm5B3S+oqDXSD494xrafUwJPN8G7QLJnPsSVcDPAAAAAAAAAckAAAAAAAAABwAAAQ0AAAA5T2JqAQQUYXZyby5jb2RlYwhudWxsFmF2cm8uc2NoZW1hDCJsb25nIgAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAA5AAAAAAAAAGQAAAAAAAAAAAAAAAAAAAADAAAAAgAAAAAAAAAAAAAAAAAAAAkAAAAAAAAAFAAAAAAAAAAYAAAAAAAAAJ0AAAAAAAAAyAAAAAAAAAADAAAAAAAAAAIAAAACAAAAAP////4AAAABAAAAAgAAB4HMOGxlAAAHgcw4bGUAAAAAAAABZQAAAAAAAABkAAAAAAAAAAUAAAAAAAAAAgAAAAIAAAAAAAAAFAAAAAAAAAAYf/////////9//////////xekCXjgYJlWPiPlQ80IyyKSmIPH5z5iMhyRa8BhBph5 diff --git a/paimon-format/src/main/java/org/apache/avro/file/RawBlockReader.java b/paimon-format/src/main/java/org/apache/avro/file/RawBlockReader.java index 43a68c54a44d..382f55f93d60 100644 --- a/paimon-format/src/main/java/org/apache/avro/file/RawBlockReader.java +++ b/paimon-format/src/main/java/org/apache/avro/file/RawBlockReader.java @@ -22,27 +22,107 @@ import org.apache.avro.io.DatumReader; import org.apache.avro.io.Decoder; +import java.io.ByteArrayOutputStream; +import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; +import java.util.Arrays; /** Package bridge exposing Avro's compressed blocks without reflection. */ public final class RawBlockReader extends DataFileStream { + private final CountingInput input; + private final byte[] headerBytes; + private long blockOffset; + private long blockLength; + private boolean pending; + public RawBlockReader(InputStream input) throws IOException { + this(new CountingInput(input)); + } + + private RawBlockReader(CountingInput input) throws IOException { super(input, new NoOpDatumReader()); + this.input = input; + long length = position(); + this.headerBytes = + length <= CountingInput.MAX_HEADER + ? Arrays.copyOf(input.prefix.toByteArray(), (int) length) + : null; + input.prefix = null; + } + + public byte[] headerBytes() { + return headerBytes == null ? null : headerBytes.clone(); + } + + public long blockOffset() { + return blockOffset; + } + + public long blockLength() { + return blockLength; } - public boolean hasNextRawBlock() { - return super.hasNextBlock(); + private long position() throws IOException { + // This is the same read-ahead adjustment used by DataFileReader.blockFinished(). + return input.position - vin.inputStream().available(); + } + + public boolean hasNextRawBlock() throws IOException { + if (!pending) { + blockOffset = position(); + pending = super.hasNextBlock(); + } + return pending; } public RawBlock nextRawBlock(RawBlock reuse) throws IOException { + if (!hasNextRawBlock()) { + throw new java.util.NoSuchElementException(); + } DataBlock raw = super.nextRawBlock(reuse == null ? null : reuse.dataBlock()); + blockLength = position() - blockOffset; + pending = false; return reuse == null ? new RawBlock(raw, resolveCodec(), getSchema()) : reuse.replace(raw, resolveCodec(), getSchema()); } + private static final class CountingInput extends FilterInputStream { + private static final int MAX_HEADER = 1024 * 1024; + private long position; + private ByteArrayOutputStream prefix = new ByteArrayOutputStream(); + + private CountingInput(InputStream input) { + super(input); + } + + @Override + public int read() throws IOException { + int value = in.read(); + if (value >= 0) { + position++; + if (prefix != null && prefix.size() < MAX_HEADER) { + prefix.write(value); + } + } + return value; + } + + @Override + public int read(byte[] bytes, int offset, int length) throws IOException { + int n = in.read(bytes, offset, length); + if (n > 0) { + position += n; + if (prefix != null && prefix.size() < MAX_HEADER) { + prefix.write(bytes, offset, Math.min(n, MAX_HEADER - prefix.size())); + } + } + return n; + } + } + private static final class NoOpDatumReader implements DatumReader { @Override diff --git a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroBlockReader.java b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroBlockReader.java index c4359afcda43..eeca62bb18c8 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroBlockReader.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroBlockReader.java @@ -54,6 +54,19 @@ public AvroBlockReader(InputStream input) throws IOException { } } + @Nullable + public byte[] headerBytes() { + return reader.headerBytes(); + } + + public long blockOffset() { + return reader.blockOffset(); + } + + public long blockLength() { + return reader.blockLength(); + } + /** Creates a record decoder from the writer schema stored in the Avro file header. */ public AvroRecordDecoder createRecordDecoder() { return new AvroRecordDecoder(reader.getSchema()); diff --git a/paimon-python/pypaimon/common/options/core_options.py b/paimon-python/pypaimon/common/options/core_options.py index 903697a94ff2..3d500fd7a5a7 100644 --- a/paimon-python/pypaimon/common/options/core_options.py +++ b/paimon-python/pypaimon/common/options/core_options.py @@ -297,6 +297,30 @@ class CoreOptions: .with_description("The parallelism for scanning manifest files.") ) + MANIFEST_ROW_ID_INDEX_WRITE: ConfigOption[bool] = ( + ConfigOptions.key("manifest.row-id-index.write") + .boolean_type() + .default_value(False) + ) + + MANIFEST_ROW_ID_INDEX_READ: ConfigOption[bool] = ( + ConfigOptions.key("manifest.row-id-index.read") + .boolean_type() + .default_value(False) + ) + + MANIFEST_ROW_ID_INDEX_MAX_RANGES: ConfigOption[int] = ( + ConfigOptions.key("manifest.row-id-index.max-ranges") + .int_type() + .default_value(131072) + ) + + MANIFEST_ROW_ID_INDEX_MAX_BYTES: ConfigOption[int] = ( + ConfigOptions.key("manifest.row-id-index.max-bytes") + .int_type() + .default_value(8388608) + ) + MANIFEST_COMPRESSION: ConfigOption[str] = ( ConfigOptions.key("manifest.compression") .string_type() diff --git a/paimon-python/pypaimon/manifest/manifest_file_manager.py b/paimon-python/pypaimon/manifest/manifest_file_manager.py index a3ac5cfe7434..0b948f3726a6 100644 --- a/paimon-python/pypaimon/manifest/manifest_file_manager.py +++ b/paimon-python/pypaimon/manifest/manifest_file_manager.py @@ -19,6 +19,10 @@ from io import BytesIO from typing import Callable, List, Optional +from pypaimon.manifest.row_id_index import ( + Settings, SUFFIX, Query, build_from_entries, read_index, read_selected_bytes, +) + import fastavro from datetime import datetime @@ -57,13 +61,25 @@ def read_entries_parallel(self, manifest_files: List[ManifestFileMeta], manifest early_entry_filter: Optional[Callable[[int, int], bool]] = None, early_record_filter: Optional[Callable[[dict], bool]] = None, partition_filter=None, + row_ranges=None, ) -> List[ManifestEntry]: - def _process_single_manifest(manifest_file: ManifestFileMeta) -> List[ManifestEntry]: - return self.read(manifest_file.file_name, manifest_entry_filter, drop_stats, - early_entry_filter=early_entry_filter, - early_record_filter=early_record_filter, - partition_filter=partition_filter) + settings = Settings.from_options(self.table.options) + query = Query(row_ranges) if settings.read and row_ranges is not None else None + + def _process_single_manifest(manifest_file: ManifestFileMeta): + path = f"{self.manifest_path}/{manifest_file.file_name}" + selected = None + if query is not None and manifest_file.index_file_name is not None: + selected = read_index(self.file_io, path, manifest_file, query, settings) + if selected is not None and not selected.blocks: + return [] + return self.read( + manifest_file.file_name, manifest_entry_filter, drop_stats, + early_entry_filter=early_entry_filter, + early_record_filter=early_record_filter, + partition_filter=partition_filter, + selected_blocks=selected) def _entry_identifier(e: ManifestEntry) -> tuple: return ( @@ -97,6 +113,7 @@ def read(self, manifest_file_name: str, manifest_entry_filter=None, drop_stats=T early_entry_filter: Optional[Callable[[int, int], bool]] = None, early_record_filter: Optional[Callable[[dict], bool]] = None, partition_filter=None, + selected_blocks=None, ) -> List[ManifestEntry]: """ early_entry_filter: ``(bucket, total_buckets) -> bool``, skip before deserializing _FILE. @@ -110,8 +127,11 @@ def read(self, manifest_file_name: str, manifest_entry_filter=None, drop_stats=T manifest_file_path = f"{self.manifest_path}/{manifest_file_name}" entries = [] - with self.file_io.new_input_stream(manifest_file_path) as input_stream: - avro_bytes = input_stream.read() + if selected_blocks is not None: + avro_bytes = read_selected_bytes(self.file_io, manifest_file_path, selected_blocks) + else: + with self.file_io.new_input_stream(manifest_file_path) as input_stream: + avro_bytes = input_stream.read() buffer = BytesIO(avro_bytes) reader = fastavro.reader(buffer) @@ -244,7 +264,7 @@ def write(self, file_name, entries: List[ManifestEntry]): fastavro.writer( buf, MANIFEST_ENTRY_SCHEMA, self._to_avro_records(entries), codec=self._codec) - self._flush(file_name, buf.getvalue()) + return self._flush(file_name, buf.getvalue(), entries) def rolling_write(self, entries: List[ManifestEntry], suggested_file_size: int, @@ -268,10 +288,9 @@ def rolling_write(self, entries: List[ManifestEntry], writer.flush() avro_bytes = buf.getvalue() file_name = f"{name_prefix}-{len(result)}" - self._flush(file_name, avro_bytes) - written_files.append(file_name) - result.append(self._build_meta( - file_name, entries[chunk_start:i + 1], len(avro_bytes))) + meta = self._flush(file_name, avro_bytes, entries[chunk_start:i + 1]) + written_files.append(meta) + result.append(meta) chunk_start = i + 1 buf = BytesIO() writer = Writer( @@ -282,13 +301,12 @@ def rolling_write(self, entries: List[ManifestEntry], writer.flush() avro_bytes = buf.getvalue() file_name = f"{name_prefix}-{len(result)}" - self._flush(file_name, avro_bytes) - written_files.append(file_name) - result.append(self._build_meta( - file_name, entries[chunk_start:], len(avro_bytes))) - except Exception: - for fname in written_files: - self.file_io.delete_quietly(f"{self.manifest_path}/{fname}") + meta = self._flush(file_name, avro_bytes, entries[chunk_start:]) + written_files.append(meta) + result.append(meta) + except BaseException: + for meta in written_files: + self.delete(meta) raise return result @@ -335,17 +353,36 @@ def _to_avro_record(entry: ManifestEntry) -> dict: def _to_avro_records(self, entries: List[ManifestEntry]) -> List[dict]: return [self._to_avro_record(e) for e in entries] - def _flush(self, file_name: str, avro_bytes: bytes): + def delete(self, manifest: ManifestFileMeta): + self.file_io.delete_quietly(f"{self.manifest_path}/{manifest.file_name}") + if manifest.index_file_name is not None: + self.file_io.delete_quietly(f"{self.manifest_path}/{manifest.index_file_name}") + + def _flush(self, file_name: str, avro_bytes: bytes, entries) -> ManifestFileMeta: manifest_path = f"{self.manifest_path}/{file_name}" + index_file_name = None try: with self.file_io.new_output_stream(manifest_path) as output_stream: output_stream.write(avro_bytes) - except Exception as e: + settings = Settings.from_options(self.table.options) + if settings.write: + data = build_from_entries(avro_bytes, entries, file_name, settings) + if data is not None: + index_file_name = file_name + SUFFIX + with self.file_io.new_output_stream(f"{self.manifest_path}/{index_file_name}") as output_stream: + output_stream.write(data) + # Publish the reference only after both objects close successfully. + return self._build_meta(file_name, entries, len(avro_bytes), index_file_name) + except BaseException as e: self.file_io.delete_quietly(manifest_path) + if index_file_name is not None: + self.file_io.delete_quietly(f"{self.manifest_path}/{index_file_name}") + if not isinstance(e, Exception) or isinstance(e, InterruptedError): + raise raise RuntimeError(f"Failed to write manifest file: {e}") from e def _build_meta(self, file_name: str, entries: List[ManifestEntry], - file_size: int = None) -> ManifestFileMeta: + file_size: int = None, index_file_name: Optional[str] = None) -> ManifestFileMeta: added_file_count = 0 deleted_file_count = 0 schema_id = None @@ -370,7 +407,9 @@ def _build_meta(self, file_name: str, entries: List[ManifestEntry], min_row_id = None max_row_id = None for entry in entries: - if entry.file.first_row_id is None: + if (entry.file.first_row_id is None or entry.file.first_row_id < 0 + or entry.file.row_count <= 0 + or entry.file.row_count - 1 > (1 << 63) - 1 - entry.file.first_row_id): min_row_id = None max_row_id = None break @@ -402,4 +441,5 @@ def _build_meta(self, file_name: str, entries: List[ManifestEntry], schema_id=schema_id, min_row_id=min_row_id, max_row_id=max_row_id, + index_file_name=index_file_name, ) diff --git a/paimon-python/pypaimon/manifest/manifest_file_merger.py b/paimon-python/pypaimon/manifest/manifest_file_merger.py index 821b12aef5e9..2f14f44e7736 100644 --- a/paimon-python/pypaimon/manifest/manifest_file_merger.py +++ b/paimon-python/pypaimon/manifest/manifest_file_merger.py @@ -93,8 +93,4 @@ def _merge_candidates(self, candidates: List[ManifestFileMeta], def _delete_manifests(self, manifests: List[ManifestFileMeta]): for manifest in manifests: - manifest_path = "{}/{}".format( - self.manifest_file_manager.manifest_path, - manifest.file_name, - ) - self.manifest_file_manager.file_io.delete_quietly(manifest_path) + self.manifest_file_manager.delete(manifest) diff --git a/paimon-python/pypaimon/manifest/manifest_list_manager.py b/paimon-python/pypaimon/manifest/manifest_list_manager.py index 3a0e606ef5c4..0b500c769ae6 100644 --- a/paimon-python/pypaimon/manifest/manifest_list_manager.py +++ b/paimon-python/pypaimon/manifest/manifest_list_manager.py @@ -98,6 +98,7 @@ def _read_from_storage(self, manifest_list_name: str) -> List[ManifestFileMeta]: schema_id=record['_SCHEMA_ID'], min_row_id=record.get('_MIN_ROW_ID'), max_row_id=record.get('_MAX_ROW_ID'), + index_file_name=record.get('_INDEX_FILE_NAME'), ) manifest_files.append(manifest_file_meta) @@ -120,6 +121,7 @@ def write(self, file_name, manifest_file_metas: List[ManifestFileMeta]): "_SCHEMA_ID": meta.schema_id, "_MIN_ROW_ID": meta.min_row_id, "_MAX_ROW_ID": meta.max_row_id, + "_INDEX_FILE_NAME": meta.index_file_name, } avro_records.append(avro_record) diff --git a/paimon-python/pypaimon/manifest/row_id_index.py b/paimon-python/pypaimon/manifest/row_id_index.py new file mode 100644 index 000000000000..98bfe25c71a6 --- /dev/null +++ b/paimon-python/pypaimon/manifest/row_id_index.py @@ -0,0 +1,303 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Complete row-id interval unions with Avro block offsets and entry ordinals. + +Version 2 uses fixed-width big-endian integers; no library-specific bitmap encoding. +""" + +import hashlib +import logging +import struct +from bisect import bisect_left +from concurrent.futures import CancelledError +from dataclasses import dataclass +from io import BytesIO +from typing import Tuple + +from pypaimon.common.options.core_options import CoreOptions +from pypaimon.utils.range import Range + +LOG = logging.getLogger(__name__) +SUFFIX = '.row-id-index' +MAGIC = b'PAIMRIDX' +MAX_ROW_ID = (1 << 63) - 1 +MAX_AVRO_HEADER = 1024 * 1024 +HEADER = struct.Struct('>8sHHI32sqqI') +BLOCK = struct.Struct('>qqqqI') +PAIR = struct.Struct('>qq') +LONG = struct.Struct('>q') + + +@dataclass +class Settings: + write: bool = False + read: bool = False + max_ranges: int = 131072 + max_bytes: int = 8 * 1024 * 1024 + + def __post_init__(self): + if not 1 <= self.max_ranges <= 1048576: + raise ValueError('manifest.row-id-index.max-ranges must be in [1, 1048576]') + if not 128 <= self.max_bytes <= 64 * 1024 * 1024: + raise ValueError('manifest.row-id-index.max-bytes must be in [128, 67108864]') + + @classmethod + def from_options(cls, options): + return cls( + options.options.get(CoreOptions.MANIFEST_ROW_ID_INDEX_WRITE), + options.options.get(CoreOptions.MANIFEST_ROW_ID_INDEX_READ), + options.options.get(CoreOptions.MANIFEST_ROW_ID_INDEX_MAX_RANGES), + options.options.get(CoreOptions.MANIFEST_ROW_ID_INDEX_MAX_BYTES)) + + +@dataclass(frozen=True) +class Block: + offset: int + length: int + first_record: int + record_count: int + + +@dataclass(frozen=True) +class Selection: + header: bytes + blocks: Tuple[Block, ...] + + +class Query: + def __init__(self, ranges): + normalized = Range.sort_and_merge_overlap(list(ranges), True) + self.starts = [r.from_ for r in normalized] + self.ends = [r.to for r in normalized] + + def intersects(self, first, last): + candidate = bisect_left(self.ends, first) + return candidate < len(self.starts) and self.starts[candidate] <= last + + +class Builder: + def __init__(self, settings, header): + self.settings = settings + self.complete = (header is not None and len(header) <= MAX_AVRO_HEADER + and len(header) + HEADER.size + 40 <= settings.max_bytes) + self.payload = bytearray() + self.ranges = [] + self.count_position = 4 + len(header) if self.complete else 0 + self.next_offset = len(header) if self.complete else 0 + self.next_record = 0 + self.range_count = 0 + self.blocks = 0 + self.current = None + self.entries_in_block = 0 + if self.complete: + self.payload.extend(struct.pack('>I', len(header))) + self.payload.extend(header) + self.payload.extend(struct.pack('>I', 0)) + + def _disable(self, reason): + self.complete = False + self.payload.clear() + self.ranges.clear() + LOG.debug('Omitting manifest row-id block index: %s', reason) + + def begin_block(self, offset, length, records): + if not self.complete: + return + _require(self.current is None and offset == self.next_offset and length > 0 and records > 0) + self.current = Block(offset, length, self.next_record, records) + self.entries_in_block = 0 + + def add(self, first, count): + if not self.complete: + return + _require(self.current is not None) + self.entries_in_block += 1 + if (first is None or first < 0 or count <= 0 + or first > MAX_ROW_ID or count - 1 > MAX_ROW_ID - first): + self._disable('unknown or invalid row-id coverage') + return + end = first + count - 1 + left = bisect_left(self.ranges, (first, -1)) + if left and self.ranges[left - 1][1] >= first - 1: + left -= 1 + right = left + while right < len(self.ranges) and self.ranges[right][0] <= end + 1: + first = min(first, self.ranges[right][0]) + end = max(end, self.ranges[right][1]) + right += 1 + if self.range_count + len(self.ranges) - (right - left) >= self.settings.max_ranges: + self._disable('range budget exceeded') + return + self.ranges[left:right] = [(first, end)] + + def end_block(self): + if not self.complete: + return + block = self.current + _require(block is not None and self.entries_in_block == block.record_count and self.ranges) + if HEADER.size + 32 + len(self.payload) + BLOCK.size + 16 * len(self.ranges) > self.settings.max_bytes: + self._disable('serialized byte budget exceeded') + return + self.payload.extend(BLOCK.pack(block.offset, block.length, block.first_record, + block.record_count, len(self.ranges))) + for first, end in self.ranges: + self.payload.extend(PAIR.pack(first, end)) + self.next_offset = block.offset + block.length + self.next_record = block.first_record + block.record_count + self.range_count += len(self.ranges) + self.blocks += 1 + self.ranges.clear() + self.current = None + + def serialize(self, name, file_size, entry_count): + if not self.complete: + return None + _require(self.current is None and self.next_offset == file_size and self.next_record == entry_count) + struct.pack_into('>I', self.payload, self.count_position, self.blocks) + header = HEADER.pack(MAGIC, 2, 2, 1, hashlib.sha256(name.encode('utf-8')).digest(), + file_size, entry_count, len(self.payload)) + data = header + self.payload + return data + hashlib.sha256(data).digest() + + +def build_from_entries(avro_bytes, entries, name, settings): + import fastavro + blocks = iter(fastavro.block_reader(BytesIO(avro_bytes))) + first_block = next(blocks, None) + header = avro_bytes[:first_block.offset] if first_block else avro_bytes + builder = Builder(settings, header) + position = 0 + block = first_block + while block is not None and builder.complete: + builder.begin_block(block.offset, block.size, block.num_records) + end = position + block.num_records + _require(end <= len(entries)) + for i in range(position, end): + entry = entries[i] + builder.add(entry.file.first_row_id, entry.file.row_count) + if not builder.complete: + break + builder.end_block() + position = end + block = next(blocks, None) if builder.complete else None + return builder.serialize(name, len(avro_bytes), len(entries)) + + +def _require(condition): + if not condition: + raise ValueError('Invalid, unsupported, mismatched or over-budget manifest row-id block index') + + +def select(data, manifest, query, settings): + if not isinstance(query, Query): + query = Query(query) + _require(128 <= len(data) <= settings.max_bytes) + _require(hashlib.sha256(data[:-32]).digest() == data[-32:]) + magic, version, codec, flags, name_hash, size, entries, length = HEADER.unpack_from(data) + _require((magic, version, codec, flags) == (MAGIC, 2, 2, 1)) + _require(name_hash == hashlib.sha256(manifest.file_name.encode('utf-8')).digest()) + _require(size == manifest.file_size and entries == manifest.num_added_files + manifest.num_deleted_files) + _require(length == len(data) - HEADER.size - 32) + offset = HEADER.size + header_length, = struct.unpack_from('>I', data, offset) + offset += 4 + _require(21 <= header_length <= MAX_AVRO_HEADER and header_length <= len(data) - offset - 36) + header = bytes(data[offset:offset + header_length]) + _require(header[:4] == b'Obj\x01') + offset += header_length + blocks, = struct.unpack_from('>I', data, offset) + offset += 4 + _require(blocks <= (len(data) - 32 - offset) // 52) + next_offset = header_length + next_record = 0 + total_ranges = 0 + selected = [] + for _ in range(blocks): + file_offset, block_length, first, count, ranges = BLOCK.unpack_from(data, offset) + offset += BLOCK.size + _require(file_offset == next_offset and 0 < block_length <= size - file_offset) + _require(first == next_record and 0 < count <= entries - first) + _require(0 < ranges <= settings.max_ranges - total_ranges and ranges <= (len(data) - 32 - offset) // 16) + total_ranges += ranges + ranges_end = offset + PAIR.size * ranges + min_row_id, first_end = PAIR.unpack_from(data, offset) + offset += PAIR.size + # The sorted interval list already contains min/max; no format change or extra fields. + max_row_id = first_end if ranges == 1 else LONG.unpack_from(data, ranges_end - LONG.size)[0] + _require(min_row_id >= 0 and first_end >= min_row_id and max_row_id >= first_end) + candidate = query.intersects(min_row_id, max_row_id) + hit = candidate and (ranges == 1 or query.intersects(min_row_id, first_end)) + previous_end = first_end + for _ in range(1, ranges): + start, end = PAIR.unpack_from(data, offset) + offset += PAIR.size + # Retain validation even when min/max rejects the block or an earlier interval hit. + _require(start >= 0 and end >= start and start > previous_end) + previous_end = end + if candidate and not hit: + hit = query.intersects(start, end) + if hit: + selected.append(Block(file_offset, block_length, first, count)) + next_offset = file_offset + block_length + next_record = first + count + _require(offset == len(data) - 32 and next_offset == size and next_record == entries) + return Selection(header, tuple(selected)) + + +def read_index(file_io, manifest_path, manifest, query, settings): + if manifest.index_file_name is None: + return None + index_path = manifest_path.rsplit('/', 1)[0] + '/' + manifest.index_file_name + try: + with file_io.new_input_stream(index_path) as stream: + data = bytearray() + while True: + chunk = stream.read(min(8192, settings.max_bytes + 1 - len(data))) + if not chunk: + break + data.extend(chunk) + _require(len(data) <= settings.max_bytes) + return select(data, manifest, query, settings) + except (InterruptedError, CancelledError, MemoryError, RecursionError): + raise + except Exception as error: + LOG.debug('Cannot use row-id block index for %s; reading manifest: %s', manifest_path, error) + return None + + +def read_selected_bytes(file_io, manifest_path, selected): + """Read complete selected blocks with seek; adjacent blocks share one contiguous span. + + The concatenated original header and blocks form a valid Avro OCF. Partial entries + must not be stored in a cache keyed by the complete manifest. + """ + data = bytearray(selected.header) + with file_io.new_input_stream(manifest_path) as stream: + previous_end = -1 + for block in selected.blocks: + if block.offset != previous_end: + stream.seek(block.offset) + remaining = block.length + while remaining: + chunk = stream.read(min(remaining, 1024 * 1024)) + if not chunk: + raise EOFError('Truncated manifest block') + data.extend(chunk) + remaining -= len(chunk) + previous_end = block.offset + block.length + return bytes(data) diff --git a/paimon-python/pypaimon/manifest/schema/manifest_file_meta.py b/paimon-python/pypaimon/manifest/schema/manifest_file_meta.py index 3c45b716950c..3e4fea3b69b5 100644 --- a/paimon-python/pypaimon/manifest/schema/manifest_file_meta.py +++ b/paimon-python/pypaimon/manifest/schema/manifest_file_meta.py @@ -33,6 +33,7 @@ class ManifestFileMeta: min_row_id: Optional[int] = None max_row_id: Optional[int] = None + index_file_name: Optional[str] = None MANIFEST_FILE_META_SCHEMA = { "type": "record", @@ -47,5 +48,6 @@ class ManifestFileMeta: {"name": "_SCHEMA_ID", "type": "long"}, {"name": "_MIN_ROW_ID", "type": ["null", "long"], "default": None}, {"name": "_MAX_ROW_ID", "type": ["null", "long"], "default": None}, + {"name": "_INDEX_FILE_NAME", "type": ["null", "string"], "default": None}, ] } diff --git a/paimon-python/pypaimon/read/scanner/file_scanner.py b/paimon-python/pypaimon/read/scanner/file_scanner.py index c114fd656e1c..477d0f2c0772 100755 --- a/paimon-python/pypaimon/read/scanner/file_scanner.py +++ b/paimon-python/pypaimon/read/scanner/file_scanner.py @@ -597,6 +597,7 @@ def read_manifest_entries(self, manifest_files: List[ManifestFileMeta], early_entry_filter=self._build_early_bucket_filter(), early_record_filter=early_row_filter, partition_filter=partition_filter, + row_ranges=row_ranges, ) def _build_early_bucket_filter(self): diff --git a/paimon-python/pypaimon/tests/manifest/row_id_index_test.py b/paimon-python/pypaimon/tests/manifest/row_id_index_test.py new file mode 100644 index 000000000000..a3a208540986 --- /dev/null +++ b/paimon-python/pypaimon/tests/manifest/row_id_index_test.py @@ -0,0 +1,359 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import base64 +import hashlib +import os +import struct +import unittest +from copy import deepcopy +from io import BytesIO + +import fastavro +from dataclasses import replace +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +from pypaimon.common.options.core_options import CoreOptions +from pypaimon.globalindex.global_index_result import GlobalIndexResult +from pypaimon.manifest.row_id_index import ( + Builder, Settings, SUFFIX, MAX_ROW_ID, Query, select, read_index, + read_selected_bytes, +) +from pypaimon.manifest.schema.manifest_entry import ManifestEntry +from pypaimon.manifest.manifest_list_manager import ManifestListManager +from pypaimon.manifest.schema.manifest_file_meta import MANIFEST_FILE_META_SCHEMA +from pypaimon.read.scanner.file_scanner import FileScanner +from pypaimon.tests.manifest import manifest_entry_identifier_test as existing +from pypaimon.utils.range import Range + + +def fixture(): + path = (Path(__file__).resolve().parents[4] / 'paimon-core/src/test/resources' / + 'manifest-row-id-index-v2.txt') + return dict(line.split('=', 1) for line in path.read_text().splitlines() + if line.startswith(('index=', 'avroHeader='))) + + +def golden(): + return base64.b64decode(fixture()['index']) + + +def avro_header(): + return base64.b64decode(fixture()['avroHeader']) + + +def golden_meta(): + return SimpleNamespace(file_name='manifest-golden', file_size=len(avro_header()) + 400, + num_added_files=7, num_deleted_files=0) + + +def intersects(data, meta, ranges, settings): + return bool(select(data, meta, ranges, settings).blocks) + + +class RowIdIndexFormatTest(unittest.TestCase): + def test_cross_language_and_block_ordinals(self): + data, meta, header = golden(), golden_meta(), avro_header() + for point in (0, 9, 20, 24, (1 << 32) - 2, 1 << 32, (1 << 32) + 2, + 8254058425445, MAX_ROW_ID): + self.assertTrue(intersects(data, meta, [Range(point, point)], Settings())) + for point in (10, 19, 25, (1 << 32) - 3, (1 << 32) + 3, 8254058425444, MAX_ROW_ID - 1): + self.assertFalse(intersects(data, meta, [Range(point, point)], Settings())) + selected = select(data, meta, [Range(20, 20)], Settings()) + self.assertEqual([b.first_record for b in selected.blocks], [0, 5]) + self.assertEqual([b.offset for b in selected.blocks], [len(header), len(header) + 300]) + self.assertEqual([b.length for b in selected.blocks], [100, 100]) + + gap = select(data, meta, [Range(16, 16)], Settings()) + + self.assertFalse(gap.blocks) + ranges = [Range(10, 19), Range(25, 40)] + self.assertFalse(intersects(data, meta, ranges, Settings())) + self.assertEqual(ranges, [Range(10, 19), Range(25, 40)]) + b = Builder(Settings(), header) + for offset, length, values in [ + (len(header), 100, [(0, 10), (5, 5), (20, 5)]), + (len(header) + 100, 200, [((1 << 32) - 2, 5), (8254058425445, 1)]), + (len(header) + 300, 100, [(20, 5), (MAX_ROW_ID, 1)])]: + b.begin_block(offset, length, len(values)) + for first, count in values: + b.add(first, count) + b.end_block() + self.assertEqual(b.serialize(meta.file_name, meta.file_size, 7), golden()) + + def test_minmax_skips_exact_checks_and_one_interval_is_already_exact(self): + header = avro_header() + builder = Builder(Settings(), header) + for offset, values in [(0, [(0, 10), (20, 10)]), + (100, [(100, 10), (200, 10)]), + (200, [(1 << 32, 10)])]: + builder.begin_block(len(header) + offset, 100, len(values)) + for first, count in values: + builder.add(first, count) + builder.end_block() + data = builder.serialize('m', len(header) + 300, 5) + meta = SimpleNamespace(file_name='m', file_size=len(header) + 300, + num_added_files=5, num_deleted_files=0) + for point, expected in [(50, 0), ((1 << 32) + 9, 1)]: + query = Query([Range(point, point)]) + with patch.object(query, 'intersects', wraps=query.intersects) as check: + selected = select(data, meta, query, Settings()) + + self.assertEqual(len(selected.blocks), expected) + self.assertEqual(check.call_count, 3) + check.assert_any_call(0, 29) + check.assert_any_call(100, 209) + check.assert_any_call(1 << 32, (1 << 32) + 9) + + def test_rejected_and_early_hit_blocks_still_validate_every_interval(self): + data = bytearray(golden()) + first_block_intervals = 68 + 4 + len(avro_header()) + 4 + 36 + struct.pack_into('>q', data, first_block_intervals + 16, 9) + data[-32:] = hashlib.sha256(data[:-32]).digest() + for point in (30, 0): + with self.assertRaises(ValueError): + select(data, golden_meta(), [Range(point, point)], Settings()) + + def test_coverage_and_budgets(self): + header = avro_header() + for first, count in [(None, 1), (-1, 1), (10, 0), (10, -1), (MAX_ROW_ID, 2)]: + b = Builder(Settings(), header) + b.begin_block(len(header), 100, 1) + b.add(first, count) + self.assertIsNone(b.serialize('m', 1, 1)) + b = Builder(Settings(), header) + b.begin_block(len(header), 100, 2) + b.add(0, MAX_ROW_ID) + b.add(MAX_ROW_ID, 1) + b.end_block() + self.assertLess(len(b.serialize('m', len(header) + 100, 2)), 512) + b = Builder(Settings(max_ranges=1), header) + b.begin_block(len(header), 100, 2) + b.add(1, 1) + b.add(1 << 32, 1) + self.assertIsNone(b.serialize('m', 1, 2)) + b = Builder(Settings(max_bytes=128), header) + self.assertIsNone(b.serialize('m', 1, 1)) + + def test_invalid_envelopes(self): + meta, data = golden_meta(), golden() + for index in (0, 9, 11, 15, 16, 55, 63, 67, 75, len(data) - 1): + bad = bytearray(data) + bad[index] ^= 2 + with self.assertRaises(ValueError): + select(bad, meta, [Range(10, 10)], Settings()) + for index in (9, 11, 15): + bad = bytearray(data[:-32]) + bad[index] = 0 + bad.extend(hashlib.sha256(bad).digest()) + with self.assertRaises(ValueError): + select(bad, meta, [Range(10, 10)], Settings()) + with self.assertRaises(ValueError): + select(data[:-1], meta, [Range(10, 10)], Settings()) + meta.file_name = 'mismatch' + with self.assertRaises(ValueError): + select(data, meta, [Range(10, 10)], Settings()) + + +class RowIdIndexScanTest(existing.ManifestEntryIdentifierTest): + def setUp(self): + super().setUp() + self.table.options.options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_WRITE, True) + self.table.options.options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_READ, True) + + def entry(self, name, first, count=10, kind=0): + return ManifestEntry(kind, self._create_file_meta('unused').min_key, 0, 1, + replace(self._create_file_meta(name), first_row_id=first, row_count=count)) + + def write_meta(self, name, entries): + manager = self.manifest_file_manager + return manager.write(name, entries) + + def test_explicit_reference_and_null_does_not_probe(self): + manager = self.manifest_file_manager + written = self.write_meta('explicit', [self.entry('data.parquet', 100)]) + self.assertEqual(written.index_file_name, written.file_name + SUFFIX) + index_path = Path(manager.manifest_path, written.index_file_name) + explicit_path = index_path.with_name('independent-index') + index_path.rename(explicit_path) + indexed = replace(written, index_file_name=explicit_path.name) + with patch.object(self.table.file_io, 'new_input_stream', + wraps=self.table.file_io.new_input_stream) as opened: + self.assertEqual(manager.read_entries_parallel([indexed], row_ranges=[Range(0, 0)]), []) + self.assertEqual([call[0][0] for call in opened.call_args_list], [str(explicit_path)]) + + unindexed = replace(indexed, index_file_name=None) + with patch.object(self.table.file_io, 'new_input_stream', + wraps=self.table.file_io.new_input_stream) as opened: + actual = manager.read_entries_parallel([unindexed], row_ranges=[Range(0, 0)]) + self.assertEqual(len(actual), 1) + self.assertEqual([call[0][0] for call in opened.call_args_list], + [str(Path(manager.manifest_path, written.file_name))]) + manager.delete(indexed) + self.assertFalse(explicit_path.exists()) + self.assertFalse(Path(manager.manifest_path, written.file_name).exists()) + + def test_manifest_list_index_reference_compatibility(self): + indexed = self.write_meta('indexed', [self.entry('data.parquet', 100)]) + unindexed = self.write_meta('legacy-entry', [self.entry('old.parquet', None)]) + self.assertIsNone(unindexed.index_file_name) + lists = ManifestListManager(self.table) + lists.write('references', [indexed, unindexed]) + actual = lists.read('references') + self.assertEqual([meta.index_file_name for meta in actual], [indexed.index_file_name, None]) + self.assertEqual([meta.file_name for meta in actual], [indexed.file_name, unindexed.file_name]) + + data = Path(lists.manifest_path, 'references').read_bytes() + legacy_schema = deepcopy(MANIFEST_FILE_META_SCHEMA) + legacy_schema['fields'] = [field for field in legacy_schema['fields'] + if field['name'] != '_INDEX_FILE_NAME'] + legacy_records = list(fastavro.reader(BytesIO(data), reader_schema=legacy_schema)) + self.assertTrue(all('_INDEX_FILE_NAME' not in record for record in legacy_records)) + self.assertEqual([record['_FILE_NAME'] for record in legacy_records], + [indexed.file_name, unindexed.file_name]) + with self.table.file_io.new_output_stream(str(Path(lists.manifest_path, 'old-list'))) as stream: + fastavro.writer(stream, legacy_schema, legacy_records) + self.assertTrue(all(meta.index_file_name is None for meta in lists.read('old-list'))) + + def test_skips_blocks_inside_a_matching_manifest(self): + entries = [self.entry('file-%d.parquet' % i, i * 1000) for i in range(4000)] + meta = self.write_meta('many-blocks', entries) + outputs = [] + reader = fastavro.reader + for enabled in (False, True): + self.table.options.options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_READ, enabled) + scanner = FileScanner(self.table, lambda: ([meta], None)) + scanner.with_global_index_result(GlobalIndexResult.from_ranges([Range(2000005, 2000005)])) + decoded = [] + + def observed_reader(stream): + for record in reader(stream): + decoded.append(record) + yield record + + with patch('pypaimon.manifest.manifest_file_manager.fastavro.reader', side_effect=observed_reader), \ + patch('pypaimon.manifest.manifest_file_manager.read_selected_bytes', + wraps=read_selected_bytes) as selected_read: + actual, _ = scanner._create_data_evolution_split_generator() + outputs.append([e.file.file_name for e in actual]) + if enabled: + self.assertEqual(selected_read.call_count, 1) + selected = selected_read.call_args[0][2] + self.assertEqual(len(selected.blocks), 1) + self.assertLess(sum(block.length for block in selected.blocks), meta.file_size // 10) + self.assertLess(len(decoded), 200) + self.assertEqual(len(decoded), selected.blocks[0].record_count) + else: + self.assertEqual(selected_read.call_count, 0) + self.assertEqual(len(decoded), 4000) + self.assertEqual(outputs, [['file-2000.parquet']] * 2) + + def test_actual_global_index_scanner_72_to_2(self): + metas = [] + for i in range(72): + entries = [self.entry('a%d.parquet' % i, 0), self.entry('b%d.parquet' % i, 100)] + if i < 2: + entries.append(self.entry('hit.' + ('parquet' if i == 0 else 'blob'), 45)) + metas.append(self.write_meta('manifest-%d' % i, entries)) + results = [] + for enabled in (False, True): + self.table.options.options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_READ, enabled) + scanner = FileScanner(self.table, lambda: (metas, None)) + scanner.with_global_index_result(GlobalIndexResult.from_ranges([Range(50, 50)])) + manager = scanner.manifest_file_manager + with patch.object(manager, 'read', wraps=manager.read) as read_body, \ + patch('pypaimon.manifest.manifest_file_manager.read_index', wraps=read_index) as read_sidecar: + entries, _ = scanner._create_data_evolution_split_generator() + results.append(sorted(e.file.file_name for e in entries)) + self.assertEqual(len(read_body.call_args_list), 2 if enabled else 72) + self.assertEqual(len(read_sidecar.call_args_list), 72 if enabled else 0) + self.assertEqual(results, [['hit.blob', 'hit.parquet']] * 2) + + def test_delete_union_no_resurrection_and_no_query_no_index_io(self): + add = self.entry('data.parquet', 45) + blob = self.entry('data.blob', 45) + metas = [self.write_meta('add', [add, blob]), + self.write_meta('delete', [replace(add, kind=1), replace(blob, kind=1)]), + self.write_meta('gap', [self.entry('lo', 0), self.entry('hi', 100)])] + manager = self.manifest_file_manager + for enabled in (False, True): + self.table.options.options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_READ, enabled) + with patch.object(manager, 'read', wraps=manager.read) as read_body: + entries = manager.read_entries_parallel(metas[:2], row_ranges=[Range(50, 50)]) + self.assertEqual(entries, []) + self.assertEqual(len(read_body.call_args_list), 2) + with patch.object(self.table.file_io, 'new_input_stream', + wraps=self.table.file_io.new_input_stream) as opened: + manager.read_entries_parallel(metas) + self.assertTrue(all(not call[0][0].endswith(SUFFIX) for call in opened.call_args_list)) + # Missing and corrupt objects retain their manifests and read the full body. + path = manager.manifest_path + '/gap' + SUFFIX + for bad in (None, b'partial'): + if bad is None: + os.unlink(path) + else: + Path(path).write_bytes(bad) + with patch.object(manager, 'read', wraps=manager.read) as read_body: + entries = manager.read_entries_parallel(metas[2:], row_ranges=[Range(50, 50)]) + self.assertEqual(len(entries), 2) + self.assertEqual(read_body.call_count, 1) + self.assertIsNone(read_body.call_args[1]['selected_blocks']) + with patch.object(self.table.file_io, 'new_input_stream', side_effect=InterruptedError('stop')): + with self.assertRaises(InterruptedError): + read_index(self.table.file_io, path, metas[0], [Range(0, 0)], Settings()) + + def test_rolling_merge_limits_and_abort_cleanup(self): + entries = [self.entry('file-%d' % i, i * 1000) for i in range(300)] + manager = self.manifest_file_manager + metas = manager.rolling_write(entries, 300, 'rolling') + self.assertGreater(len(metas), 1) + for meta in metas: + actual = manager.read(meta.file_name) + data = Path(manager.manifest_path, meta.file_name + SUFFIX).read_bytes() + for e in actual: + self.assertTrue(intersects(data, meta, [Range(e.file.first_row_id, e.file.first_row_id)], Settings())) + gap = actual[0].file.first_row_id + 10 + self.assertFalse(intersects(data, meta, [Range(gap, gap)], Settings())) + from pypaimon.manifest.manifest_file_merger import ManifestFileMerger + merger = ManifestFileMerger(manager, 1000000, 2) + merged = merger.merge(metas) + # Merger returns both the final manifest list and newly written outputs. + outputs = merged[0] if isinstance(merged, tuple) else merged + for meta in outputs: + self.assertTrue(Path(manager.manifest_path, meta.file_name + SUFFIX).exists()) + for meta in metas: + self.assertIsNotNone(meta.index_file_name) + manager.delete(meta) + self.assertFalse(Path(manager.manifest_path, meta.file_name + SUFFIX).exists()) + original = self.table.file_io.new_output_stream + + def fail(path): + if path.endswith(SUFFIX): + raise OSError('sidecar write failed') + return original(path) + with patch.object(self.table.file_io, 'new_output_stream', side_effect=fail): + with self.assertRaises(RuntimeError): + manager.write('failed', entries[:1]) + self.assertFalse(Path(manager.manifest_path, 'failed').exists()) + self.assertFalse(Path(manager.manifest_path, 'failed' + SUFFIX).exists()) + manager.write('unknown', [self.entry('legacy', None)]) + self.assertFalse(Path(manager.manifest_path, 'unknown' + SUFFIX).exists()) + self.table.options.options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_MAX_RANGES, 1) + manager.write('huge', [self.entry('one', 0, 10), self.entry('two', 100, 10)]) + self.assertFalse(Path(manager.manifest_path, 'huge' + SUFFIX).exists()) diff --git a/paimon-python/pypaimon/write/file_store_commit.py b/paimon-python/pypaimon/write/file_store_commit.py index 66e88f2f5a68..2fd81fda2cc7 100644 --- a/paimon-python/pypaimon/write/file_store_commit.py +++ b/paimon-python/pypaimon/write/file_store_commit.py @@ -1140,8 +1140,7 @@ def _clean_up_reuse_tmp_manifests( if ml_name: try: for meta in self.manifest_list_manager.read(ml_name): - self.table.file_io.delete_quietly( - f"{self.manifest_file_manager.manifest_path}/{meta.file_name}") + self.manifest_file_manager.delete(meta) except Exception: pass self.table.file_io.delete_quietly(f"{manifest_path}/{ml_name}") @@ -1160,8 +1159,7 @@ def _clean_up_no_reuse_tmp_manifests( if base_manifest_list: self.table.file_io.delete_quietly(f"{manifest_path}/{base_manifest_list}") for meta in merge_new_files: - self.table.file_io.delete_quietly( - f"{self.manifest_file_manager.manifest_path}/{meta.file_name}") + self.manifest_file_manager.delete(meta) def abort(self, commit_messages: List[CommitMessage]): """Abort commit and delete files. Uses external_path if available to ensure proper scheme handling.""" From e9796aee09d4b6d0f242411e7dd95cefc074531d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Fri, 11 Sep 2026 20:34:16 +0800 Subject: [PATCH 02/23] [core] Coalesce manifest row-id index and block reads Use bounded 1 MiB read requests in Java and Python to avoid object-store request amplification. Merge adjacent selected blocks into spans and buffer Java reads independently of the Avro consumer read size. Add regression tests for request counts, skipped gaps, short reads, size budgets, stream closure and truncated inputs. --- .../paimon/manifest/ManifestRowIdIndex.java | 59 +++-- .../manifest/ManifestRowIdIndexTest.java | 240 ++++++++++++++++++ .../pypaimon/manifest/row_id_index.py | 22 +- .../tests/manifest/row_id_index_test.py | 117 ++++++++- 4 files changed, 414 insertions(+), 24 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestRowIdIndex.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestRowIdIndex.java index bd55946c8478..f97279bbdc49 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestRowIdIndex.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestRowIdIndex.java @@ -63,6 +63,7 @@ public final class ManifestRowIdIndex { private static final int HEADER_BYTES = 68; private static final int DIGEST_BYTES = 32; private static final int MAX_AVRO_HEADER = 1024 * 1024; + private static final int READ_BUFFER_BYTES = 1024 * 1024; private ManifestRowIdIndex() {} @@ -362,7 +363,7 @@ public static Selection select( return new Selection(header, selected); } - /** One bounded GET attempt, without a preceding HEAD. Null means read the original manifest. */ + /** Bounded, bulk index reads. Null means read the original manifest. */ @Nullable public static Selection read( FileIO io, @@ -378,7 +379,7 @@ public static Selection read( try (InputStream in = io.newInputStream(new Path(path.getParent(), manifest.indexFileName()))) { ByteArrayOutputStream out = new ByteArrayOutputStream(); - byte[] buffer = new byte[8192]; + byte[] buffer = new byte[Math.min(READ_BUFFER_BYTES, settings.maxBytes + 1)]; int n; while ((n = in.read( @@ -436,7 +437,9 @@ private static final class SelectedBlockInput extends InputStream { private int headerPosition; private int blockPosition; private long remaining; - private long previousEnd = -1; + private byte[] buffer; + private int bufferPosition; + private int bufferLimit; private SelectedBlockInput(SeekableInputStream input, Selection selected) { this.input = input; @@ -445,8 +448,10 @@ private SelectedBlockInput(SeekableInputStream input, Selection selected) { @Override public int read() throws IOException { - byte[] one = new byte[1]; - return read(one, 0, 1) < 0 ? -1 : one[0] & 255; + if (headerPosition < selected.header.length) { + return selected.header[headerPosition++] & 255; + } + return fillBuffer() ? buffer[bufferPosition++] & 255 : -1; } @Override @@ -460,23 +465,47 @@ public int read(byte[] bytes, int offset, int length) throws IOException { headerPosition += n; return n; } + if (!fillBuffer()) { + return -1; + } + int copied = Math.min(length, bufferLimit - bufferPosition); + System.arraycopy(buffer, bufferPosition, bytes, offset, copied); + bufferPosition += copied; + return copied; + } + + private boolean fillBuffer() throws IOException { + if (bufferPosition < bufferLimit) { + return true; + } if (remaining == 0) { if (blockPosition == selected.blocks.size()) { - return -1; + return false; } Block block = selected.blocks.get(blockPosition++); - if (block.offset != previousEnd) { - input.seek(block.offset); + long end = block.offset + block.length; + while (blockPosition < selected.blocks.size() + && selected.blocks.get(blockPosition).offset == end) { + end += selected.blocks.get(blockPosition++).length; } - previousEnd = block.offset + block.length; - remaining = block.length; + input.seek(block.offset); + remaining = end - block.offset; } - int n = input.read(bytes, offset, (int) Math.min(length, remaining)); - if (n < 0) { - throw new EOFException("Truncated manifest block"); + int requested = (int) Math.min(READ_BUFFER_BYTES, remaining); + if (buffer == null || buffer.length < requested) { + buffer = new byte[requested]; + } + bufferPosition = 0; + bufferLimit = 0; + while (bufferLimit < requested) { + int count = input.read(buffer, bufferLimit, requested - bufferLimit); + if (count < 0) { + throw new EOFException("Truncated manifest block"); + } + bufferLimit += count; } - remaining -= n; - return n; + remaining -= requested; + return true; } @Override diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestRowIdIndexTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestRowIdIndexTest.java index 35da1891e562..526fe42cbaeb 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestRowIdIndexTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestRowIdIndexTest.java @@ -19,22 +19,30 @@ package org.apache.paimon.manifest; import org.apache.paimon.CoreOptions; +import org.apache.paimon.fs.ByteArraySeekableStream; +import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.options.Options; +import org.apache.paimon.utils.IOUtils; import org.apache.paimon.utils.Range; import org.apache.paimon.utils.RowRangeIndex; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import java.io.ByteArrayOutputStream; +import java.io.EOFException; import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; import java.nio.file.Files; import java.security.MessageDigest; +import java.util.ArrayList; import java.util.Arrays; import java.util.Base64; import java.util.Collections; +import java.util.List; import java.util.Properties; import static org.assertj.core.api.Assertions.assertThat; @@ -333,6 +341,238 @@ public org.apache.paimon.fs.SeekableInputStream newInputStream(Path path) { .isInstanceOf(AssertionError.class); } + @Test + void indexReadsUseBoundedBulkRequests() throws Exception { + byte[] header = header(); + for (int blockCount : new int[] {5000, 25000}) { + ManifestRowIdIndex.Builder builder = new ManifestRowIdIndex.Builder(settings, header); + for (int blockNumber = 0; blockNumber < blockCount; blockNumber++) { + builder.beginBlock(header.length + blockNumber * 100L, 100, 1); + builder.add((long) blockNumber, 1); + builder.endBlock(); + } + long size = header.length + blockCount * 100L; + byte[] data = builder.serialize("manifest-large", size, blockCount); + ManifestFileMeta meta = meta("manifest-large", size, blockCount); + CountingInput stream = new CountingInput(data, Integer.MAX_VALUE); + Path path = new Path(temp.toString(), meta.fileName()); + FileIO io = mock(FileIO.class); + when(io.newInputStream(ManifestRowIdIndex.path(path))).thenReturn(stream); + ManifestRowIdIndex.Selection actual = + ManifestRowIdIndex.read( + io, + path, + meta, + RowRangeIndex.create(Collections.singletonList(new Range(0, 0))), + settings); + assertThat(actual.blocks()).hasSize(1); + assertThat(actual.blocks().get(0).offset).isEqualTo(header.length); + assertThat(stream.readLengths).hasSize((data.length + (1 << 20) - 1) / (1 << 20)); + assertThat(stream.requests).allMatch(request -> request <= 1 << 20); + assertThat(stream.closed).isTrue(); + } + } + + @Test + void indexShortReadsAndExactBudget() throws Exception { + byte[] data = golden(); + Options options = new Options(); + options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_MAX_BYTES, data.length); + Path path = new Path(temp.toString(), "manifest-golden"); + for (int maxRead : new int[] {Integer.MAX_VALUE, 7}) { + CountingInput stream = new CountingInput(data, maxRead); + FileIO io = mock(FileIO.class); + when(io.newInputStream(ManifestRowIdIndex.path(path))).thenReturn(stream); + ManifestRowIdIndex.Selection actual = + ManifestRowIdIndex.read( + io, + path, + goldenMeta(), + RowRangeIndex.create(Collections.singletonList(new Range(20, 20))), + new ManifestRowIdIndex.Settings(options)); + assertThat(actual.blocks()) + .extracting(block -> block.firstRecord) + .containsExactly(0L, 5L); + assertThat(stream.closed).isTrue(); + } + } + + @Test + void indexOverBudgetStopsAfterOneExtraByte() throws Exception { + Options options = new Options(); + options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_MAX_BYTES, 128); + Path path = new Path(temp.toString(), "manifest-golden"); + CountingInput stream = new CountingInput(golden(), Integer.MAX_VALUE); + FileIO io = mock(FileIO.class); + when(io.newInputStream(ManifestRowIdIndex.path(path))).thenReturn(stream); + assertThat( + ManifestRowIdIndex.read( + io, + path, + goldenMeta(), + RowRangeIndex.create(Collections.singletonList(new Range(20, 20))), + new ManifestRowIdIndex.Settings(options))) + .isNull(); + assertThat(stream.readLengths).containsExactly(129); + assertThat(stream.closed).isTrue(); + } + + @Test + void adjacentBlocksShareReadsForSingleByteConsumers() throws Exception { + byte[] header = header(); + byte[] body = new byte[400]; + for (int position = 0; position < body.length; position++) { + body[position] = (byte) position; + } + ManifestRowIdIndex.Selection selected = + ManifestRowIdIndex.select( + golden(), + goldenMeta(), + RowRangeIndex.create( + Arrays.asList( + new Range(0, 0), + new Range(8254058425445L, 8254058425445L))), + settings); + byte[] manifest = Arrays.copyOf(header, header.length + body.length); + System.arraycopy(body, 0, manifest, header.length, body.length); + CountingInput stream = new CountingInput(manifest, Integer.MAX_VALUE); + Path path = new Path(temp.toString(), "manifest-golden"); + FileIO io = mock(FileIO.class); + when(io.newInputStream(path)).thenReturn(stream); + ByteArrayOutputStream actual = new ByteArrayOutputStream(); + try (InputStream input = ManifestRowIdIndex.openManifest(io, path, selected)) { + int value; + while ((value = input.read()) != -1) { + actual.write(value); + } + assertThat(input.read(new byte[1], 0, 0)).isZero(); + } + assertThat(actual.toByteArray()).isEqualTo(Arrays.copyOf(manifest, header.length + 300)); + assertThat(stream.readLengths).containsExactly(300); + assertThat(stream.seeks).containsExactly((long) header.length); + assertThat(stream.closed).isTrue(); + } + + @Test + void blockReadsSkipGapsAndEmptySelections() throws Exception { + byte[] header = header(); + byte[] manifest = Arrays.copyOf(header, header.length + 400); + Arrays.fill(manifest, header.length + 100, header.length + 300, (byte) 7); + Path path = new Path(temp.toString(), "manifest-golden"); + for (long point : new long[] {20, 16}) { + CountingInput stream = new CountingInput(manifest, Integer.MAX_VALUE); + FileIO io = mock(FileIO.class); + when(io.newInputStream(path)).thenReturn(stream); + byte[] actual; + try (InputStream input = + ManifestRowIdIndex.openManifest( + io, path, select(golden(), goldenMeta(), point))) { + actual = IOUtils.readFully(input, false); + } + if (point == 20) { + assertThat(actual).isEqualTo(Arrays.copyOf(header, header.length + 200)); + assertThat(stream.readLengths).containsExactly(100, 100); + assertThat(stream.seeks) + .containsExactly((long) header.length, header.length + 300L); + } else { + assertThat(actual).isEqualTo(header); + assertThat(stream.readLengths).isEmpty(); + assertThat(stream.seeks).isEmpty(); + } + assertThat(stream.closed).isTrue(); + } + } + + @Test + void largeBlockSpansUseBoundedReads() throws Exception { + byte[] header = header(); + ManifestRowIdIndex.Builder builder = new ManifestRowIdIndex.Builder(settings, header); + long offset = header.length; + for (int length : new int[] {512 * 1024, 512 * 1024, 257}) { + builder.beginBlock(offset, length, 1); + builder.add(20L, 1); + builder.endBlock(); + offset += length; + } + byte[] data = builder.serialize("manifest-large", offset, 3); + byte[] manifest = Arrays.copyOf(header, (int) offset); + CountingInput stream = new CountingInput(manifest, Integer.MAX_VALUE); + FileIO io = mock(FileIO.class); + Path path = new Path(temp.toString(), "manifest-large"); + when(io.newInputStream(path)).thenReturn(stream); + try (InputStream input = + ManifestRowIdIndex.openManifest( + io, path, select(data, meta("manifest-large", offset, 3), 20))) { + assertThat(IOUtils.readFully(input, false)).isEqualTo(manifest); + } + assertThat(stream.readLengths).containsExactly(1 << 20, 257); + assertThat(stream.seeks).containsExactly((long) header.length); + assertThat(stream.closed).isTrue(); + } + + @Test + void blockShortReadsAndTruncation() throws Exception { + byte[] header = header(); + Path path = new Path(temp.toString(), "manifest-golden"); + ManifestRowIdIndex.Selection selected = + ManifestRowIdIndex.select( + golden(), + goldenMeta(), + RowRangeIndex.create( + Collections.singletonList(new Range(0, Long.MAX_VALUE))), + settings); + for (int bodyLength : new int[] {400, 399}) { + byte[] manifest = Arrays.copyOf(header, header.length + bodyLength); + CountingInput stream = new CountingInput(manifest, 7); + FileIO io = mock(FileIO.class); + when(io.newInputStream(path)).thenReturn(stream); + try (InputStream input = ManifestRowIdIndex.openManifest(io, path, selected)) { + if (bodyLength == 400) { + assertThat(IOUtils.readFully(input, false)).isEqualTo(manifest); + } else { + assertThatThrownBy(() -> IOUtils.readFully(input, false)) + .isInstanceOf(EOFException.class); + } + } + assertThat(stream.closed).isTrue(); + } + } + + private static class CountingInput extends ByteArraySeekableStream { + private final int maxRead; + private final List requests = new ArrayList<>(); + private final List readLengths = new ArrayList<>(); + private final List seeks = new ArrayList<>(); + private boolean closed; + + private CountingInput(byte[] data, int maxRead) { + super(data); + this.maxRead = maxRead; + } + + @Override + public int read(byte[] bytes, int offset, int length) throws IOException { + requests.add(length); + int count = super.read(bytes, offset, Math.min(length, maxRead)); + if (count > 0) { + readLengths.add(count); + } + return count; + } + + @Override + public void seek(long position) throws IOException { + seeks.add(position); + super.seek(position); + } + + @Override + public void close() throws IOException { + closed = true; + super.close(); + } + } + private ManifestRowIdIndex.Selection select(byte[] data, ManifestFileMeta meta, long point) throws IOException { return ManifestRowIdIndex.select( diff --git a/paimon-python/pypaimon/manifest/row_id_index.py b/paimon-python/pypaimon/manifest/row_id_index.py index 98bfe25c71a6..03312d4d9399 100644 --- a/paimon-python/pypaimon/manifest/row_id_index.py +++ b/paimon-python/pypaimon/manifest/row_id_index.py @@ -37,6 +37,7 @@ MAGIC = b'PAIMRIDX' MAX_ROW_ID = (1 << 63) - 1 MAX_AVRO_HEADER = 1024 * 1024 +READ_BUFFER_BYTES = 1024 * 1024 HEADER = struct.Struct('>8sHHI32sqqI') BLOCK = struct.Struct('>qqqqI') PAIR = struct.Struct('>qq') @@ -267,7 +268,7 @@ def read_index(file_io, manifest_path, manifest, query, settings): with file_io.new_input_stream(index_path) as stream: data = bytearray() while True: - chunk = stream.read(min(8192, settings.max_bytes + 1 - len(data))) + chunk = stream.read(min(READ_BUFFER_BYTES, settings.max_bytes + 1 - len(data))) if not chunk: break data.extend(chunk) @@ -288,16 +289,21 @@ def read_selected_bytes(file_io, manifest_path, selected): """ data = bytearray(selected.header) with file_io.new_input_stream(manifest_path) as stream: - previous_end = -1 - for block in selected.blocks: - if block.offset != previous_end: - stream.seek(block.offset) - remaining = block.length + block_position = 0 + while block_position < len(selected.blocks): + block = selected.blocks[block_position] + block_position += 1 + end = block.offset + block.length + while (block_position < len(selected.blocks) + and selected.blocks[block_position].offset == end): + end += selected.blocks[block_position].length + block_position += 1 + stream.seek(block.offset) + remaining = end - block.offset while remaining: - chunk = stream.read(min(remaining, 1024 * 1024)) + chunk = stream.read(min(remaining, READ_BUFFER_BYTES)) if not chunk: raise EOFError('Truncated manifest block') data.extend(chunk) remaining -= len(chunk) - previous_end = block.offset + block.length return bytes(data) diff --git a/paimon-python/pypaimon/tests/manifest/row_id_index_test.py b/paimon-python/pypaimon/tests/manifest/row_id_index_test.py index a3a208540986..52fbaac68b49 100644 --- a/paimon-python/pypaimon/tests/manifest/row_id_index_test.py +++ b/paimon-python/pypaimon/tests/manifest/row_id_index_test.py @@ -32,7 +32,7 @@ from pypaimon.common.options.core_options import CoreOptions from pypaimon.globalindex.global_index_result import GlobalIndexResult from pypaimon.manifest.row_id_index import ( - Builder, Settings, SUFFIX, MAX_ROW_ID, Query, select, read_index, + Block, Builder, Selection, Settings, SUFFIX, MAX_ROW_ID, Query, select, read_index, read_selected_bytes, ) from pypaimon.manifest.schema.manifest_entry import ManifestEntry @@ -67,6 +67,121 @@ def intersects(data, meta, ranges, settings): return bool(select(data, meta, ranges, settings).blocks) +class CountingInput(BytesIO): + def __init__(self, data, max_read=None): + super().__init__(data) + self.max_read = max_read + self.reads = [] + self.requests = [] + self.seeks = [] + + def read(self, size=-1): + if size < 0: + raise AssertionError('Unbounded read') + self.requests.append(size) + position = self.tell() + data = super().read(size if self.max_read is None else min(size, self.max_read)) + if data: + self.reads.append((position, len(data))) + return data + + def seek(self, offset, whence=0): + self.seeks.append(offset) + return super().seek(offset, whence) + + +class RowIdIndexReadTest(unittest.TestCase): + def test_index_reads_use_bounded_bulk_requests(self): + header = avro_header() + for block_count in (5000, 25000): + with self.subTest(block_count=block_count): + builder = Builder(Settings(), header) + for block_number in range(block_count): + builder.begin_block(len(header) + block_number * 100, 100, 1) + builder.add(block_number, 1) + builder.end_block() + size = len(header) + block_count * 100 + data = builder.serialize('manifest-large', size, block_count) + meta = SimpleNamespace(file_name='manifest-large', file_size=size, + num_added_files=block_count, num_deleted_files=0, + index_file_name='manifest-large' + SUFFIX) + stream = CountingInput(data) + file_io = SimpleNamespace(new_input_stream=lambda path: stream) + actual = read_index(file_io, '/manifest/manifest-large', meta, + [Range(0, 0)], Settings()) + self.assertEqual(actual, select(data, meta, [Range(0, 0)], Settings())) + self.assertEqual(len(stream.reads), (len(data) + (1 << 20) - 1) // (1 << 20)) + self.assertLessEqual(max(stream.requests), 1 << 20) + self.assertTrue(stream.closed) + + def test_index_short_reads_and_exact_budget(self): + data, meta = golden(), golden_meta() + meta.index_file_name = meta.file_name + SUFFIX + for max_read in (None, 7): + with self.subTest(max_read=max_read): + stream = CountingInput(data, max_read) + file_io = SimpleNamespace(new_input_stream=lambda path: stream) + settings = Settings(max_bytes=len(data)) + actual = read_index(file_io, '/manifest/manifest-golden', meta, + [Range(20, 20)], settings) + self.assertEqual(actual, select(data, meta, [Range(20, 20)], settings)) + self.assertTrue(stream.closed) + + def test_index_over_budget_stops_after_one_extra_byte(self): + data, meta = golden(), golden_meta() + meta.index_file_name = meta.file_name + SUFFIX + stream = CountingInput(data) + file_io = SimpleNamespace(new_input_stream=lambda path: stream) + self.assertIsNone(read_index(file_io, '/manifest/manifest-golden', meta, + [Range(20, 20)], Settings(max_bytes=128))) + self.assertEqual(stream.reads, [(0, 129)]) + self.assertTrue(stream.closed) + + def test_adjacent_blocks_share_reads_without_reading_gaps(self): + header = avro_header() + body = bytes(range(200)) * 2 + for points, spans in [([0, 8254058425445], [(0, 300)]), + ([20], [(0, 100), (300, 100)]), ([16], [])]: + with self.subTest(points=points): + selected = select(golden(), golden_meta(), + [Range(point, point) for point in points], Settings()) + stream = CountingInput(header + body) + file_io = SimpleNamespace(new_input_stream=lambda path: stream) + actual = read_selected_bytes(file_io, '/manifest/manifest-golden', selected) + expected = header + b''.join(body[start:start + size] for start, size in spans) + self.assertEqual(actual, expected) + self.assertEqual(stream.reads, [(len(header) + start, size) for start, size in spans]) + self.assertEqual(stream.seeks, [len(header) + start for start, _ in spans]) + self.assertTrue(stream.closed) + + def test_large_block_spans_use_bounded_reads(self): + header = avro_header() + block_size = 512 * 1024 + body = bytes(2 * block_size + 257) + selected = Selection(header, (Block(len(header), block_size, 0, 1), + Block(len(header) + block_size, block_size, 1, 1), + Block(len(header) + 2 * block_size, 257, 2, 1))) + stream = CountingInput(header + body) + file_io = SimpleNamespace(new_input_stream=lambda path: stream) + self.assertEqual(read_selected_bytes(file_io, '/manifest/manifest-large', selected), header + body) + self.assertEqual(stream.reads, [(len(header), 1 << 20), (len(header) + (1 << 20), 257)]) + self.assertEqual(stream.seeks, [len(header)]) + self.assertTrue(stream.closed) + + def test_block_short_reads_and_truncation(self): + header = avro_header() + body = bytes(range(200)) * 2 + selected = select(golden(), golden_meta(), [Range(0, MAX_ROW_ID)], Settings()) + stream = CountingInput(header + body, 7) + file_io = SimpleNamespace(new_input_stream=lambda path: stream) + self.assertEqual(read_selected_bytes(file_io, '/manifest/manifest-golden', selected), header + body) + self.assertTrue(stream.closed) + stream = CountingInput(header + body[:-1], 7) + with self.assertRaises(EOFError): + read_selected_bytes(file_io, '/manifest/manifest-golden', selected) + self.assertTrue(stream.closed) + + class RowIdIndexFormatTest(unittest.TestCase): def test_cross_language_and_block_ordinals(self): data, meta, header = golden(), golden_meta(), avro_header() From d71c531416e4df73791f0add22194f1bdff9bb3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Sat, 12 Sep 2026 11:44:06 +0800 Subject: [PATCH 03/23] [test] Stabilize manifest sidecar regression tests Forward selected_blocks through the append-only reader test wrapper. Fix the manifest target size and assert explicit retained and expired manifest sets so snapshot and tag retention coverage does not depend on randomized file sizes. --- .../paimon/operation/ExpireSnapshotsTest.java | 39 ++++++++++++------- .../pypaimon/tests/reader_append_only_test.py | 6 ++- 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/ExpireSnapshotsTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/ExpireSnapshotsTest.java index abbbbde9a2f7..c8d2d64f89b1 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/ExpireSnapshotsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/ExpireSnapshotsTest.java @@ -41,6 +41,7 @@ import org.apache.paimon.manifest.ManifestIndexTestUtils; import org.apache.paimon.mergetree.compact.DeduplicateMergeFunction; import org.apache.paimon.options.ExpireConfig; +import org.apache.paimon.options.MemorySize; import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; @@ -745,23 +746,29 @@ public void testExpirePlansManifestsConcurrentlyWithSkippingSet() throws Excepti @Test void testSidecarsFollowSnapshotAndTagRetention() throws Exception { store.options().toConfiguration().set(CoreOptions.MANIFEST_MERGE_MIN_COUNT, 2); + store.options() + .toConfiguration() + .set(CoreOptions.MANIFEST_TARGET_FILE_SIZE, MemorySize.parse("8 mb")); List allData = new ArrayList<>(); List snapshotPositions = new ArrayList<>(); commit(8, allData, snapshotPositions); int latest = requireNonNull(snapshotManager.latestSnapshotId()).intValue(); Set manifests = new HashSet<>(); - for (int i = 1; i <= latest; i++) { - rewriteSnapshotTime(i, 0); - ManifestIndexTestUtils.registerIndexReferences(store, i); + Set retainedManifests = new HashSet<>(); + for (int snapshotId = 1; snapshotId <= latest; snapshotId++) { + rewriteSnapshotTime(snapshotId, 0); + ManifestIndexTestUtils.registerIndexReferences(store, snapshotId); snapshotManager.invalidateCache(); - store.manifestListFactory() - .create() - .readDataManifests(snapshotManager.snapshot(i)) - .forEach( - meta -> - manifests.add( - store.pathFactory() - .toManifestFilePath(meta.fileName()))); + for (ManifestFileMeta meta : + store.manifestListFactory() + .create() + .readDataManifests(snapshotManager.snapshot(snapshotId))) { + Path manifest = store.pathFactory().toManifestFilePath(meta.fileName()); + manifests.add(manifest); + if (snapshotId == 3 || snapshotId == latest) { + retainedManifests.add(manifest); + } + } } store.newTagManager() .createTag( @@ -770,22 +777,24 @@ void testSidecarsFollowSnapshotAndTagRetention() throws Exception { store.options().tagDefaultTimeRetained(), Collections.emptyList(), false); + Set expiredManifests = new HashSet<>(manifests); + expiredManifests.removeAll(retainedManifests); + assertThat(expiredManifests).isNotEmpty(); ExpireSnapshotsImpl expire = (ExpireSnapshotsImpl) store.newExpire(expireAllButLatestConfig()); expire.setCurrentTimeMillis(() -> 1000L); expire.expire(); - boolean reclaimed = false; for (Path manifest : manifests) { - boolean retained = fileIO.exists(manifest); + boolean retained = retainedManifests.contains(manifest); + assertThat(fileIO.exists(manifest)).as("manifest %s", manifest).isEqualTo(retained); assertThat( fileIO.exists( new Path( manifest.getParent(), "index-for-" + manifest.getName()))) + .as("sidecar for %s", manifest) .isEqualTo(retained); - reclaimed |= !retained; } - assertThat(reclaimed).isTrue(); for (ManifestFileMeta meta : store.manifestListFactory() .create() diff --git a/paimon-python/pypaimon/tests/reader_append_only_test.py b/paimon-python/pypaimon/tests/reader_append_only_test.py index 480cd4ebbe4b..1c783bfe0611 100644 --- a/paimon-python/pypaimon/tests/reader_append_only_test.py +++ b/paimon-python/pypaimon/tests/reader_append_only_test.py @@ -1090,7 +1090,8 @@ def test_is_in_with_partitions(self): def counting_read(self_mgr, manifest_file_name, manifest_entry_filter=None, drop_stats=True, early_entry_filter=None, - early_record_filter=None, partition_filter=None): + early_record_filter=None, partition_filter=None, + selected_blocks=None): # avro_total = every entry in the manifest (no manifest-file pruning # here: single file, is_in spans its partition stats). path = f"{self_mgr.manifest_path}/{manifest_file_name}" @@ -1100,7 +1101,8 @@ def counting_read(self_mgr, manifest_file_name, return original_read( self_mgr, manifest_file_name, manifest_entry_filter, drop_stats, - early_entry_filter, early_record_filter, partition_filter) + early_entry_filter, early_record_filter, partition_filter, + selected_blocks=selected_blocks) def counting_dfm_init(self_dfm, *args, **kwargs): entry_counts['constructed'] += 1 From 914897fcfc2b01b76a3d259341eabb36cc261e59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Sat, 12 Sep 2026 15:21:43 +0800 Subject: [PATCH 04/23] [core][python] Preserve cancellation through manifest index fallback Propagate PyArrow cancellations and inspect chained and suppressed failures before falling back to full manifests. Preserve Java interruption state and fatal failures, guard against exception cycles, and cover stream open/read/close behavior with regression tests. --- docs/docs/concepts/spec/manifest.md | 1 + .../paimon/manifest/ManifestRowIdIndex.java | 18 +++- .../manifest/ManifestRowIdIndexTest.java | 85 +++++++++++++++ .../pypaimon/manifest/row_id_index.py | 18 +++- .../tests/manifest/row_id_index_test.py | 101 ++++++++++++++++++ 5 files changed, 221 insertions(+), 2 deletions(-) diff --git a/docs/docs/concepts/spec/manifest.md b/docs/docs/concepts/spec/manifest.md index f884dcd55c17..8895415b1f8b 100644 --- a/docs/docs/concepts/spec/manifest.md +++ b/docs/docs/concepts/spec/manifest.md @@ -77,6 +77,7 @@ default to `false`. Old manifests, null or empty extra-file lists, and lists con other extra-file types use the normal manifest read path. Missing, unsupported, corrupt, or over-budget indexes also fall back to that path. Writers omit the sidecar if complete row-ID coverage cannot be established within the configured range and byte budgets. +Cancellation and interruption errors propagate instead of triggering a full-manifest fallback. Selected blocks still pass through entry filtering and ADD/DELETE reconciliation. Snapshot, tag, changelog, orphan-file and failed-commit cleanup retain or remove the sidecar through diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestRowIdIndex.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestRowIdIndex.java index 0a3bdde60640..ab3c91cdd918 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestRowIdIndex.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestRowIdIndex.java @@ -48,8 +48,10 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.IdentityHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.TreeMap; import java.util.concurrent.CancellationException; @@ -409,7 +411,17 @@ public static Selection read( } catch (CancellationException failure) { throw failure; } catch (IOException | RuntimeException failure) { - for (Throwable cause = failure; cause != null; cause = cause.getCause()) { + List pending = new ArrayList<>(); + Set visited = Collections.newSetFromMap(new IdentityHashMap<>()); + pending.add(failure); + for (int position = 0; position < pending.size(); position++) { + Throwable cause = pending.get(position); + if (!visited.add(cause)) { + continue; + } + if (cause instanceof Error) { + throw (Error) cause; + } if (cause instanceof CancellationException) { throw (CancellationException) cause; } @@ -420,6 +432,10 @@ public static Selection read( Thread.currentThread().interrupt(); throw interrupted(failure); } + if (cause.getCause() != null) { + pending.add(cause.getCause()); + } + Collections.addAll(pending, cause.getSuppressed()); } if (Thread.currentThread().isInterrupted()) { throw interrupted(failure); diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestRowIdIndexTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestRowIdIndexTest.java index e61629552ead..6a5f5f0a9764 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestRowIdIndexTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestRowIdIndexTest.java @@ -44,6 +44,8 @@ import java.util.Collections; import java.util.List; import java.util.Properties; +import java.util.concurrent.CancellationException; +import java.util.concurrent.atomic.AtomicBoolean; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -342,6 +344,89 @@ public org.apache.paimon.fs.SeekableInputStream newInputStream(Path path) { .isInstanceOf(AssertionError.class); } + @Test + void suppressedCancellationInterruptionAndFatalErrorsPropagate() { + ManifestFileMeta manifest = meta("m", 1, 1); + RowRangeIndex query = RowRangeIndex.create(Collections.singletonList(new Range(1, 1))); + for (Throwable closeFailure : + Arrays.asList( + new CancellationException("cancelled"), + new java.io.InterruptedIOException("interrupted"), + new AssertionError("fatal"))) { + AtomicBoolean closed = new AtomicBoolean(); + LocalFileIO fileIO = + new LocalFileIO() { + @Override + public org.apache.paimon.fs.SeekableInputStream newInputStream(Path path) { + return new ByteArraySeekableStream(new byte[0]) { + @Override + public int read(byte[] bytes, int offset, int length) + throws IOException { + throw new IOException("read failed"); + } + + @Override + public void close() throws IOException { + super.close(); + closed.set(true); + if (closeFailure instanceof IOException) { + throw (IOException) closeFailure; + } + if (closeFailure instanceof Error) { + throw (Error) closeFailure; + } + throw (RuntimeException) closeFailure; + } + }; + } + }; + try { + assertThatThrownBy( + () -> + ManifestRowIdIndex.read( + fileIO, + new Path(temp.toString(), "m"), + manifest, + query, + settings)) + .isInstanceOf( + closeFailure instanceof java.io.InterruptedIOException + ? java.io.UncheckedIOException.class + : closeFailure.getClass()); + assertThat(Thread.currentThread().isInterrupted()) + .isEqualTo(closeFailure instanceof java.io.InterruptedIOException); + assertThat(closed).isTrue(); + } finally { + Thread.interrupted(); + } + } + } + + @Test + void ordinaryExceptionCyclesFallBack() { + IOException first = new IOException("first"); + IOException second = new IOException("second"); + first.initCause(second); + second.addSuppressed(first); + LocalFileIO fileIO = + new LocalFileIO() { + @Override + public org.apache.paimon.fs.SeekableInputStream newInputStream(Path path) + throws IOException { + throw first; + } + }; + assertThat( + ManifestRowIdIndex.read( + fileIO, + new Path(temp.toString(), "m"), + meta("m", 1, 1), + RowRangeIndex.create(Collections.singletonList(new Range(1, 1))), + settings)) + .isNull(); + assertThat(Thread.currentThread().isInterrupted()).isFalse(); + } + @Test void indexReadsUseBoundedBulkRequests() throws Exception { byte[] header = header(); diff --git a/paimon-python/pypaimon/manifest/row_id_index.py b/paimon-python/pypaimon/manifest/row_id_index.py index 36918a3784e7..3a3bba51ee41 100644 --- a/paimon-python/pypaimon/manifest/row_id_index.py +++ b/paimon-python/pypaimon/manifest/row_id_index.py @@ -29,6 +29,8 @@ from io import BytesIO from typing import Tuple +from pyarrow import ArrowCancelled + from pypaimon.common.options.core_options import CoreOptions from pypaimon.utils.range import Range @@ -42,6 +44,7 @@ BLOCK = struct.Struct('>qqqqI') PAIR = struct.Struct('>qq') LONG = struct.Struct('>q') +_PROPAGATED_ERRORS = (InterruptedError, CancelledError, ArrowCancelled, MemoryError, RecursionError) @dataclass @@ -279,9 +282,22 @@ def read_index(file_io, manifest_path, manifest, query, settings): data.extend(chunk) _require(len(data) <= settings.max_bytes) return select(data, manifest, query, settings) - except (InterruptedError, CancelledError, MemoryError, RecursionError): + except _PROPAGATED_ERRORS: raise except Exception as error: + pending = [error] + visited = set() + while pending: + cause = pending.pop() + if id(cause) in visited: + continue + visited.add(id(cause)) + if not isinstance(cause, Exception) or isinstance(cause, _PROPAGATED_ERRORS): + raise cause + if cause.__cause__ is not None: + pending.append(cause.__cause__) + if cause.__context__ is not None: + pending.append(cause.__context__) LOG.debug('Cannot use row-id block index for %s; reading manifest: %s', manifest_path, error) return None diff --git a/paimon-python/pypaimon/tests/manifest/row_id_index_test.py b/paimon-python/pypaimon/tests/manifest/row_id_index_test.py index 095b85a2053e..3bf398a471c5 100644 --- a/paimon-python/pypaimon/tests/manifest/row_id_index_test.py +++ b/paimon-python/pypaimon/tests/manifest/row_id_index_test.py @@ -20,10 +20,12 @@ import os import struct import unittest +from concurrent.futures import CancelledError from copy import deepcopy from io import BytesIO import fastavro +from pyarrow import ArrowCancelled from dataclasses import replace from pathlib import Path from types import SimpleNamespace @@ -90,6 +92,27 @@ def seek(self, offset, whence=0): return super().seek(offset, whence) +class FailingIndexInput(BytesIO): + def __init__(self, data, failure, phase, close_failure=None): + super().__init__(data) + self.failure = failure + self.phase = phase + self.close_failure = close_failure + + def read(self, size=-1): + if self.phase == 'read': + raise self.failure + return super().read(size) + + def close(self): + was_closed = self.closed + super().close() + if not was_closed and self.close_failure is not None: + raise self.close_failure + if self.phase == 'close' and not was_closed: + raise self.failure + + class RowIdIndexReadTest(unittest.TestCase): def test_index_reads_use_bounded_bulk_requests(self): header = avro_header() @@ -440,6 +463,84 @@ def test_delete_union_no_resurrection_and_no_query_no_index_io(self): with self.assertRaises(InterruptedError): read_index(self.table.file_io, path, metas[0], [Range(0, 0)], Settings()) + def test_sidecar_cancellation_during_open(self): + self._check_sidecar_cancellation('open') + + def test_sidecar_cancellation_during_read(self): + self._check_sidecar_cancellation('read') + + def test_sidecar_cancellation_during_close(self): + self._check_sidecar_cancellation('close') + + def _check_sidecar_cancellation(self, phase): + for failure_type in (ArrowCancelled, CancelledError, InterruptedError): + with self.subTest(phase=phase, failure_type=failure_type): + self._check_sidecar_io_failure(phase, failure_type('cancelled'), cancelled=True) + + def test_sidecar_io_failures_fall_back_to_manifest(self): + for phase in ('open', 'read', 'close'): + for failure_type in (FileNotFoundError, TimeoutError, OSError): + with self.subTest(phase=phase, failure_type=failure_type): + self._check_sidecar_io_failure(phase, failure_type('unavailable'), cancelled=False) + + def test_sidecar_cancellation_survives_close_failure(self): + for failure_type in (ArrowCancelled, CancelledError, InterruptedError): + with self.subTest(failure_type=failure_type): + self._check_sidecar_io_failure('read', failure_type('cancelled'), cancelled=True, + close_failure=OSError('close failed')) + + def test_sidecar_wrapped_cancellation_propagates(self): + for failure_type in (ArrowCancelled, CancelledError, InterruptedError): + with self.subTest(failure_type=failure_type): + cancellation = failure_type('cancelled') + wrapped = OSError('wrapped failure') + wrapped.__cause__ = cancellation + self._check_sidecar_io_failure('open', wrapped, cancelled=True, + expected_failure=cancellation) + + def test_sidecar_exception_cycle_falls_back(self): + first = OSError('first') + second = OSError('second') + first.__cause__ = second + second.__cause__ = first + self._check_sidecar_io_failure('open', first, cancelled=False) + + def _check_sidecar_io_failure(self, phase, failure, cancelled, close_failure=None, + expected_failure=None): + manager = self.manifest_file_manager + meta = self.write_meta('failure-' + phase + '-' + type(failure).__name__, + [self.entry('data.parquet', 100)]) + index_path = str(Path(manager.manifest_path, index_file_name(meta))) + body_path = str(Path(manager.manifest_path, meta.file_name)) + stream = (FailingIndexInput(Path(index_path).read_bytes(), failure, phase, close_failure) + if phase != 'open' else None) + original_open = self.table.file_io.new_input_stream + + def open_stream(path): + if path == index_path: + if phase == 'open': + raise failure + return stream + return original_open(path) + + with patch.object(self.table.file_io, 'new_input_stream', side_effect=open_stream) as opened, \ + patch.object(manager, 'read', wraps=manager.read) as read_body: + if cancelled: + expected = failure if expected_failure is None else expected_failure + with self.assertRaises(type(expected)) as raised: + manager.read_entries_parallel([meta], row_ranges=[Range(100, 100)]) + self.assertIs(raised.exception, expected) + read_body.assert_not_called() + self.assertEqual([call.args[0] for call in opened.call_args_list], [index_path]) + else: + entries = manager.read_entries_parallel([meta], row_ranges=[Range(100, 100)]) + self.assertEqual([entry.file.file_name for entry in entries], ['data.parquet']) + read_body.assert_called_once() + self.assertIsNone(read_body.call_args.kwargs['selected_blocks']) + self.assertEqual([call.args[0] for call in opened.call_args_list], [index_path, body_path]) + if stream is not None: + self.assertTrue(stream.closed) + def test_rolling_merge_limits_and_abort_cleanup(self): entries = [self.entry('file-%d' % i, i * 1000) for i in range(300)] manager = self.manifest_file_manager From 00b99d7ad5d32e7b1c7a75d9095141aa865f7155 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Mon, 14 Sep 2026 10:57:12 +0800 Subject: [PATCH 05/23] [core] Share manifest block indexes for partitions and row IDs Add a shared partition dictionary and independently framed partition and row-id payloads per block. Preserve complete directories while degrading optional coverage, support partition-only planning and unknown payload encodings, and retain version-2 reads and cancellation handling. --- docs/docs/concepts/spec/manifest.md | 88 +++- .../java/org/apache/paimon/CoreOptions.java | 22 +- .../apache/paimon/manifest/ManifestFile.java | 12 +- .../paimon/manifest/ManifestRowIdIndex.java | 473 +++++++++++++++--- .../operation/AbstractFileStoreScan.java | 4 +- .../manifest/ManifestBlockIndexTest.java | 320 ++++++++++++ .../paimon/manifest/ManifestFileTest.java | 56 ++- .../manifest/ManifestRowIdIndexTest.java | 30 +- .../resources/manifest-row-id-index-v2.txt | 4 + .../pypaimon/common/options/core_options.py | 12 + .../manifest/manifest_file_manager.py | 8 +- .../pypaimon/manifest/row_id_index.py | 297 +++++++++-- .../pypaimon/read/scanner/file_scanner.py | 1 + .../manifest/manifest_block_index_test.py | 218 ++++++++ .../tests/manifest/row_id_index_test.py | 78 ++- 15 files changed, 1472 insertions(+), 151 deletions(-) create mode 100644 paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java create mode 100644 paimon-python/pypaimon/tests/manifest/manifest_block_index_test.py diff --git a/docs/docs/concepts/spec/manifest.md b/docs/docs/concepts/spec/manifest.md index 8895415b1f8b..ca4522023155 100644 --- a/docs/docs/concepts/spec/manifest.md +++ b/docs/docs/concepts/spec/manifest.md @@ -63,22 +63,100 @@ skip manifests before opening them. Each extra file belongs exclusively to one manifest. It is retained and cleaned up together with that manifest during snapshot, tag, or changelog deletion. -### Row-ID Block Index +### Partition and Row-ID Block Index With `manifest.row-id-index.write` enabled, a manifest writer can create a binary `.row-id-index` sidecar. Its name is stored in the manifest-list record's `_EXTRA_FILES`; the existing Avro schemas and `_VERSION` identifiers are unchanged. -Readers identify the row-ID index by the `.row-id-index` suffix among these explicit +Readers identify the block index by the historical `.row-id-index` suffix among these explicit references, not by probing for a derived file name. Other extra-file references are preserved. -With `manifest.row-id-index.read` enabled and a row-ID filter available, readers can use +With `manifest.row-id-index.read` enabled and a partition or row-ID filter available, readers can use the sidecar to select complete Avro blocks before reading manifest entries. Both options default to `false`. Old manifests, null or empty extra-file lists, and lists containing only other extra-file types use the normal manifest read path. Missing, unsupported, corrupt, -or over-budget indexes also fall back to that path. Writers omit the sidecar if complete -row-ID coverage cannot be established within the configured range and byte budgets. +or over-budget containers also fall back to that path. Each block's partition and row-ID +coverage is independently usable; an unavailable dimension cannot exclude a block. Cancellation and interruption errors propagate instead of triggering a full-manifest fallback. +Version 3 uses the following layout. Container integers and encoding-1 payload integers +are fixed-width big endian. Encoding IDs are unsigned bytes with separate namespaces. + +```text +magic : 8 bytes // ASCII PAIMRIDX +formatVersion : int // 3 +manifestNameHash : 32 bytes // SHA-256 of the UTF-8 basename +manifestLength : long +manifestEntryCount : long // ADD + DELETE +avroHeaderLength : int +avroHeader : bytes // original schema, codec and sync marker +partitionCount : int +partitionDictionary[] + partitionByteLength : int + partitionBytes : bytes // existing manifest BinaryRow serialization +blockCount : int +blocks[] // original physical order + offset : long + length : long // complete encoded block, including sync marker + recordCount : long + partitionEncoding : byte + partitionPayloadLength : int + partitionPayload : bytes + rowIdEncoding : byte + rowIdPayloadLength : int + rowIdPayload : bytes +checksum : 32 bytes // SHA-256 of all preceding bytes +``` + +The block ID is its position. Its first entry ordinal is the sum of preceding record +counts and is not stored. Each complete partition tuple appears once in the dictionary, +including all its fields and nulls. The scan's partition type interprets the existing +serialized tuple. Partition predicates are evaluated once per dictionary entry. + +| Dimension | Encoding | Payload | +| --- | --- | --- | +| Either | `0` | Unavailable; payload length must be zero. | +| Partition | `1` | Positive `partitionIdCount: int` followed by sorted unique dictionary IDs (`int`). | +| Row ID | `1` | Positive `rangeCount: int` followed by sorted disjoint inclusive `(start: long, end: long)` pairs. Coverage may conservatively include gaps. | +| Either | Other nonzero ID | Skip exactly the bounded payload length; treat only this dimension as unavailable. | + +Payload lengths exclude the encoding and length fields. Invalid lengths, known-payload +framing, dictionary references, interval order, checksums or physical coverage invalidate +the container. Byte spans must cover the entire original manifest after its header; +record counts must sum to the manifest entry count. Readers continue validating blocks +and known payloads even when a predicate has already rejected a block. + +All entries contribute, including ADD, DELETE and every file format/column group. +Row-ID ranges are never expanded into individual values. If an exact union exceeds its +range budget, it becomes the inclusive `[min,max]` envelope with encoding 1. Processing +continues through the end of the block to extend those bounds and detect unknown row IDs. +An unknown or invalid row-ID range makes only that block's row-ID payload unavailable. +Partition budget exhaustion independently makes that block's partition payload unavailable. +The dictionary can consequently be incomplete for the manifest: a dictionary miss never +excludes a block with unavailable partition coverage. Later blocks can still use existing IDs. + +`manifest.row-id-index.max-ranges` bounds stored intervals (default 131072). +`manifest.index.max-partitions` bounds dictionary entries (default 65536), and +`manifest.index.max-partition-bytes` bounds dictionary bytes including length fields +(default 1048576). `manifest.row-id-index.max-bytes` bounds the whole serialized container +(default 8388608); the Avro header is also capped at 1 MiB and the directory at 131072 blocks. +Writers discard optional row-ID payloads, then partition payloads/dictionary if necessary, +to fit the complete directory. If the directory itself cannot fit, no sidecar is published. +No emitted sidecar omits block descriptors. These are encoded-size bounds; construction +also incurs bounded object/buffer overhead. Query concurrency multiplies per-reader costs. + +For conjunctive filters a block is retained only if each dimension is either unavailable +or matches. Matches in the two dimensions can come from different entries in the block, +so entry filtering and deletion merging remain necessary. Block min/max is derived from +the first/last interval before testing the individual intervals. + +Readers still consume and validate the whole bounded sidecar. A partition-only query +therefore reads row-ID payload bytes too; payload lengths save decoding work for unknown +encodings, not storage I/O. Selected compressed blocks are read by byte range with adjacent +spans coalesced. Version-2 row-ID-only sidecars remain readable with partition coverage +treated as unavailable; older readers safely fall back on version 3. Existing immutable +manifests are not backfilled by enabling the write option. + Selected blocks still pass through entry filtering and ADD/DELETE reconciliation. Snapshot, tag, changelog, orphan-file and failed-commit cleanup retain or remove the sidecar through its extra-file reference together with the owning manifest. diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java index 5561d31abf5d..cc8a95315bf1 100644 --- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java @@ -515,28 +515,42 @@ public InlineElement getDescription() { .booleanType() .defaultValue(false) .withDescription( - "Write complete row-id block indexes for newly created manifests."); + "Write block indexes with independent partition and row-id coverage for newly created manifests."); public static final ConfigOption MANIFEST_ROW_ID_INDEX_READ = key("manifest.row-id-index.read") .booleanType() .defaultValue(false) .withDescription( - "Read optional row-id sidecars after coarse manifest pruning. Missing or invalid indexes fall back to manifest reads."); + "Read optional manifest block indexes for partition or row-id filters after coarse pruning. Missing or invalid indexes fall back to manifest reads."); public static final ConfigOption MANIFEST_ROW_ID_INDEX_MAX_RANGES = key("manifest.row-id-index.max-ranges") .intType() .defaultValue(131072) .withDescription( - "Maximum disjoint row-id intervals across all Avro blocks in a manifest. Exceeding the limit disables the entire index. Range: 1 to 1048576."); + "Maximum disjoint row-id intervals across all Avro blocks in a manifest. On exhaustion, coarsen coverage to min/max or mark row-id coverage unavailable. Range: 1 to 1048576."); + + public static final ConfigOption MANIFEST_INDEX_MAX_PARTITIONS = + key("manifest.index.max-partitions") + .intType() + .defaultValue(65536) + .withDescription( + "Maximum partition dictionary entries per manifest block index. Further unknown partitions disable partition coverage only for their blocks."); + + public static final ConfigOption MANIFEST_INDEX_MAX_PARTITION_BYTES = + key("manifest.index.max-partition-bytes") + .intType() + .defaultValue(1048576) + .withDescription( + "Maximum serialized partition dictionary bytes per manifest block index. Exceeding this budget preserves independently available row-id coverage."); public static final ConfigOption MANIFEST_ROW_ID_INDEX_MAX_BYTES = key("manifest.row-id-index.max-bytes") .intType() .defaultValue(8388608) .withDescription( - "Maximum serialized row-id sidecar bytes, including header and checksum. Exceeding the limit disables the entire index. Range: 128 to 67108864."); + "Maximum serialized manifest block index bytes, including header and checksum. Optional payloads are dropped before omitting an index whose complete block directory cannot fit. Range: 128 to 67108864."); public static final ConfigOption MANIFEST_COMPRESSION = key("manifest.compression") diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java index f14860f48bfb..69b9770266a7 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java @@ -387,13 +387,23 @@ public Path toPath(String fileName) { @Nullable public ManifestRowIdIndex.Selection selectBlocks( ManifestFileMeta manifest, @Nullable RowRangeIndex query) { - return !rowIdIndexSettings.read || query == null + return selectBlocks(manifest, query, null); + } + + @Nullable + public ManifestRowIdIndex.Selection selectBlocks( + ManifestFileMeta manifest, + @Nullable RowRangeIndex query, + @Nullable PartitionPredicate partitionFilter) { + return !rowIdIndexSettings.read || (query == null && partitionFilter == null) ? null : ManifestRowIdIndex.read( fileIO, pathFactory.toPath(manifest.fileName()), manifest, query, + partitionFilter, + partitionType, rowIdIndexSettings); } diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestRowIdIndex.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestRowIdIndex.java index ab3c91cdd918..f90bd1b2db26 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestRowIdIndex.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestRowIdIndex.java @@ -19,11 +19,15 @@ package org.apache.paimon.manifest; import org.apache.paimon.CoreOptions; +import org.apache.paimon.data.BinaryRow; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.SeekableInputStream; import org.apache.paimon.options.Options; +import org.apache.paimon.partition.PartitionPredicate; +import org.apache.paimon.types.RowType; import org.apache.paimon.utils.RowRangeIndex; +import org.apache.paimon.utils.SerializationUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -49,20 +53,26 @@ import java.util.Arrays; import java.util.Collections; import java.util.IdentityHashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeMap; +import java.util.TreeSet; import java.util.concurrent.CancellationException; import static org.apache.paimon.utils.Preconditions.checkArgument; -/** Complete row-id interval unions and physical locations of a manifest's Avro blocks. */ +/** Independently usable partition and row-id coverage for each physical manifest block. */ public final class ManifestRowIdIndex { public static final String SUFFIX = ".row-id-index"; private static final Logger LOG = LoggerFactory.getLogger(ManifestRowIdIndex.class); private static final long MAGIC = 0x5041494d52494458L; - private static final int HEADER_BYTES = 68; + private static final int HEADER_BYTES = 60; + private static final int LEGACY_HEADER_BYTES = 68; + private static final int BLOCK_BYTES = 34; + private static final int MAX_BLOCKS = 131072; + private static final byte[] EMPTY = new byte[0]; private static final int DIGEST_BYTES = 32; private static final int MAX_AVRO_HEADER = 1024 * 1024; private static final int READ_BUFFER_BYTES = 1024 * 1024; @@ -91,12 +101,22 @@ public static final class Settings { public final boolean read; public final int maxRanges; public final int maxBytes; + public final int maxPartitions; + public final int maxPartitionBytes; public Settings(Options options) { write = options.get(CoreOptions.MANIFEST_ROW_ID_INDEX_WRITE); read = options.get(CoreOptions.MANIFEST_ROW_ID_INDEX_READ); maxRanges = options.get(CoreOptions.MANIFEST_ROW_ID_INDEX_MAX_RANGES); maxBytes = options.get(CoreOptions.MANIFEST_ROW_ID_INDEX_MAX_BYTES); + maxPartitions = options.get(CoreOptions.MANIFEST_INDEX_MAX_PARTITIONS); + maxPartitionBytes = options.get(CoreOptions.MANIFEST_INDEX_MAX_PARTITION_BYTES); + checkArgument( + maxPartitions > 0 && maxPartitions <= 1048576, + "Invalid manifest.index.max-partitions"); + checkArgument( + maxPartitionBytes > 0 && maxPartitionBytes <= 64 * 1024 * 1024, + "Invalid manifest.index.max-partition-bytes"); checkArgument( maxRanges > 0 && maxRanges <= 1048576, "manifest.row-id-index.max-ranges must be in [1, 1048576]"); @@ -136,57 +156,79 @@ public List blocks() { } } - /** Bounded range union. No row-id enumeration, even for a range ending at Long.MAX_VALUE. */ + /** + * Builds complete block descriptors even when either optional dimension becomes unavailable. + */ public static final class Builder { private final Settings settings; + private final byte[] header; private final TreeMap ranges = new TreeMap<>(); - private final ByteArrayOutputStream payload = new ByteArrayOutputStream(); - private final DataOutputStream out = new DataOutputStream(payload); - private final int countPosition; + private final Map dictionary = new LinkedHashMap<>(); + private final TreeSet partitionIds = new TreeSet<>(); + private final List blocks = new ArrayList<>(); private boolean complete; private long nextOffset; private long nextRecord; private Block current; private long entriesInBlock; - private int blocks; - private int rangeCount; - - public Builder(Settings settings, @Nullable byte[] header) throws IOException { + private boolean rowAvailable; + private boolean partitionAvailable; + private boolean coarse; + private long min; + private long max; + private int dictionaryBytes; + private int optionalBytes; + private int savedRanges; + + public Builder(Settings settings, @Nullable byte[] header) { this.settings = settings; - this.complete = + this.header = header; + complete = header != null && header.length <= MAX_AVRO_HEADER - && header.length + HEADER_BYTES + DIGEST_BYTES + 8 <= settings.maxBytes; - countPosition = complete ? 4 + header.length : 0; - if (complete) { - out.writeInt(header.length); - out.write(header); - out.writeInt(0); - nextOffset = header.length; - } + && HEADER_BYTES + DIGEST_BYTES + 12L + header.length + <= settings.maxBytes; + nextOffset = header == null ? 0 : header.length; } public boolean complete() { return complete; } - private void disable(String reason) { - complete = false; - ranges.clear(); - payload.reset(); - LOG.debug("Omitting manifest row-id block index: {}", reason); - } - public void beginBlock(long offset, long length, long records) throws IOException { if (!complete) { return; } require(current == null && offset == nextOffset && length > 0 && records > 0); + // Optional payloads can be discarded later, but descriptors must never be truncated. + if (blocks.size() == MAX_BLOCKS + || HEADER_BYTES + + DIGEST_BYTES + + 12L + + header.length + + (blocks.size() + 1L) * BLOCK_BYTES + > settings.maxBytes) { + complete = false; + blocks.clear(); + dictionary.clear(); + return; + } current = new Block(offset, length, nextRecord, records); entriesInBlock = 0; + rowAvailable = true; + partitionAvailable = true; + coarse = false; + min = Long.MAX_VALUE; + max = -1; + ranges.clear(); + partitionIds.clear(); } public void add(@Nullable Long first, long count) { + add(first, count, null); + } + + public void add(@Nullable Long first, long count, @Nullable byte[] partition) { if (!complete) { return; } @@ -194,12 +236,23 @@ public void add(@Nullable Long first, long count) { throw new IllegalStateException("No current Avro block"); } entriesInBlock++; + addPartition(partition); + if (!rowAvailable) { + return; + } if (first == null || first < 0 || count <= 0 || count - 1 > Long.MAX_VALUE - first) { - disable("unknown or invalid row-id coverage"); + rowAvailable = false; + ranges.clear(); return; } - long start = first; long end = first + (count - 1); + min = Math.min(min, first); + max = Math.max(max, end); + // Keep checking subsequent entries, including missing row IDs, after coarsening. + if (coarse) { + return; + } + long start = first; Map.Entry before = ranges.floorEntry(start); if (before != null && before.getValue() >= start - 1) { start = before.getKey(); @@ -212,37 +265,83 @@ public void add(@Nullable Long first, long count) { end = Math.max(end, next.getValue()); ranges.remove(next.getKey()); } - if (rangeCount + ranges.size() >= settings.maxRanges) { - disable("range budget exceeded"); + if (ranges.size() >= Math.min(settings.maxRanges, settings.maxBytes / 16)) { + coarse = true; + ranges.clear(); + } else { + ranges.put(start, end); + } + } + + private void addPartition(@Nullable byte[] bytes) { + if (!partitionAvailable) { return; } - ranges.put(start, end); + if (bytes == null) { + partitionAvailable = false; + partitionIds.clear(); + return; + } + Integer id = dictionary.get(ByteBuffer.wrap(bytes)); + if (id == null) { + if (dictionary.size() >= settings.maxPartitions + || bytes.length + 4L + > Math.min(settings.maxPartitionBytes, settings.maxBytes) + - dictionaryBytes) { + partitionAvailable = false; + partitionIds.clear(); + return; + } + id = dictionary.size(); + dictionary.put(ByteBuffer.wrap(bytes.clone()), id); + dictionaryBytes += 4 + bytes.length; + } + partitionIds.add(id); } public void endBlock() throws IOException { if (!complete) { return; } - require(current != null && entriesInBlock == current.recordCount && !ranges.isEmpty()); - if (HEADER_BYTES + DIGEST_BYTES + (long) payload.size() + 36 + 16L * ranges.size() - > settings.maxBytes) { - disable("serialized byte budget exceeded"); - return; + require(current != null && entriesInBlock == current.recordCount); + byte[] rowPayload = EMPTY; + byte[] partitionPayload = EMPTY; + int rowCount = 0; + if (rowAvailable) { + if (coarse + || ranges.size() > settings.maxRanges - savedRanges + || 4L + 16L * ranges.size() > settings.maxBytes - optionalBytes) { + ranges.clear(); + ranges.put(min, max); + } + if (savedRanges < settings.maxRanges + && 4L + 16L * ranges.size() <= settings.maxBytes - optionalBytes) { + ByteBuffer out = ByteBuffer.allocate(4 + 16 * ranges.size()); + out.putInt(ranges.size()); + for (Map.Entry range : ranges.entrySet()) { + out.putLong(range.getKey()).putLong(range.getValue()); + } + rowPayload = out.array(); + rowCount = ranges.size(); + optionalBytes += rowPayload.length; + } } - out.writeLong(current.offset); - out.writeLong(current.length); - out.writeLong(current.firstRecord); - out.writeLong(current.recordCount); - out.writeInt(ranges.size()); - for (Map.Entry range : ranges.entrySet()) { - out.writeLong(range.getKey()); - out.writeLong(range.getValue()); + if (partitionAvailable + && 4L + 4L * partitionIds.size() <= settings.maxBytes - optionalBytes) { + ByteBuffer out = ByteBuffer.allocate(4 + 4 * partitionIds.size()); + out.putInt(partitionIds.size()); + for (int id : partitionIds) { + out.putInt(id); + } + partitionPayload = out.array(); + optionalBytes += partitionPayload.length; } + blocks.add(new IndexedBlock(current, partitionPayload, rowPayload)); + savedRanges += rowCount; nextOffset = Math.addExact(current.offset, current.length); nextRecord = Math.addExact(current.firstRecord, current.recordCount); - rangeCount += ranges.size(); - blocks++; ranges.clear(); + partitionIds.clear(); current = null; } @@ -252,23 +351,77 @@ public byte[] serialize(String name, long fileSize, long entryCount) throws IOEx return null; } require(current == null && nextOffset == fileSize && nextRecord == entryCount); - byte[] body = payload.toByteArray(); - ByteBuffer.wrap(body).putInt(countPosition, blocks); - ByteArrayOutputStream buffer = - new ByteArrayOutputStream(HEADER_BYTES + body.length + DIGEST_BYTES); - DataOutputStream envelope = new DataOutputStream(buffer); - envelope.writeLong(MAGIC); - envelope.writeShort(2); - envelope.writeShort(2); // sorted inclusive interval unions per Avro block - envelope.writeInt(1); // COMPLETE; all other bits reserved - envelope.write(digest(name.getBytes(StandardCharsets.UTF_8))); - envelope.writeLong(fileSize); - envelope.writeLong(entryCount); - envelope.writeInt(body.length); - envelope.write(body); - envelope.write(digest(buffer.toByteArray())); + long size = + HEADER_BYTES + + DIGEST_BYTES + + 12L + + header.length + + dictionaryBytes + + blocks.size() * (long) BLOCK_BYTES + + optionalBytes; + // Give directory growth priority over optional coverage. Never remove a descriptor. + for (IndexedBlock block : blocks) { + if (size <= settings.maxBytes) { + break; + } + size -= block.rowIds.length; + optionalBytes -= block.rowIds.length; + block.rowIds = EMPTY; + } + if (size > settings.maxBytes) { + size -= dictionaryBytes; + dictionaryBytes = 0; + dictionary.clear(); + for (IndexedBlock block : blocks) { + size -= block.partitions.length; + optionalBytes -= block.partitions.length; + block.partitions = EMPTY; + } + } + require(size <= settings.maxBytes); + ByteArrayOutputStream buffer = new ByteArrayOutputStream((int) size); + DataOutputStream out = new DataOutputStream(buffer); + out.writeLong(MAGIC); + out.writeInt(3); + out.write(digest(name.getBytes(StandardCharsets.UTF_8))); + out.writeLong(fileSize); + out.writeLong(entryCount); + out.writeInt(header.length); + out.write(header); + out.writeInt(dictionary.size()); + for (ByteBuffer bytes : dictionary.keySet()) { + out.writeInt(bytes.remaining()); + out.write(bytes.array()); + } + out.writeInt(blocks.size()); + for (IndexedBlock block : blocks) { + out.writeLong(block.block.offset); + out.writeLong(block.block.length); + out.writeLong(block.block.recordCount); + writePayload(out, block.partitions); + writePayload(out, block.rowIds); + } + out.write(digest(buffer.toByteArray())); return buffer.toByteArray(); } + + private static void writePayload(DataOutputStream out, byte[] payload) throws IOException { + out.writeByte(payload.length == 0 ? 0 : 1); + out.writeInt(payload.length); + out.write(payload); + } + } + + private static final class IndexedBlock { + private final Block block; + private byte[] partitions; + private byte[] rowIds; + + private IndexedBlock(Block block, byte[] partitions, byte[] rowIds) { + this.block = block; + this.partitions = partitions; + this.rowIds = rowIds; + } } /** Rebuild from the final physical blocks, including raw-copy and encoded rewrite paths. */ @@ -286,7 +439,10 @@ public static byte[] build(FileIO io, Path path, long size, long records, Settin ManifestAvroReader.RowIterator rows = block.toRows(projection.projectedType()); while (builder.complete() && rows.hasNext()) { entry.replace(rows.next()); - builder.add(entry.file().firstRowId(), entry.file().rowCount()); + builder.add( + entry.file().firstRowId(), + entry.file().rowCount(), + entry.partitionBytes()); } builder.endBlock(); } @@ -294,9 +450,172 @@ public static byte[] build(FileIO io, Path path, long size, long records, Settin } } - /** Validate the complete index before allowing any negative decision. */ public static Selection select( - byte[] data, ManifestFileMeta manifest, RowRangeIndex query, Settings settings) + byte[] data, + ManifestFileMeta manifest, + @Nullable RowRangeIndex query, + Settings settings) + throws IOException { + return select(data, manifest, query, null, null, settings); + } + + /** Validates framing and known payloads before applying the two independent dimensions. */ + public static Selection select( + byte[] data, + ManifestFileMeta manifest, + @Nullable RowRangeIndex query, + @Nullable PartitionPredicate partitionFilter, + @Nullable RowType partitionType, + Settings settings) + throws IOException { + require(data.length >= 128 && data.length <= settings.maxBytes); + if (ByteBuffer.wrap(data).getInt(8) == 0x00020002) { + // Published prototype files have row-ID coverage only. + return selectLegacy(data, manifest, query, settings); + } + int limit = data.length - DIGEST_BYTES; + require( + MessageDigest.isEqual( + digest(Arrays.copyOf(data, limit)), + Arrays.copyOfRange(data, limit, data.length))); + ByteBuffer in = ByteBuffer.wrap(data, 0, limit).slice(); + require(in.getLong() == MAGIC && in.getInt() == 3); + byte[] hash = new byte[DIGEST_BYTES]; + in.get(hash); + require( + MessageDigest.isEqual( + hash, digest(manifest.fileName().getBytes(StandardCharsets.UTF_8)))); + require(in.getLong() == manifest.fileSize()); + long entries = Math.addExact(manifest.numAddedFiles(), manifest.numDeletedFiles()); + require(in.getLong() == entries); + int headerLength = in.getInt(); + require( + headerLength >= 21 + && headerLength <= MAX_AVRO_HEADER + && headerLength <= in.remaining() - 8); + byte[] header = new byte[headerLength]; + in.get(header); + require(header[0] == 'O' && header[1] == 'b' && header[2] == 'j' && header[3] == 1); + int partitions = in.getInt(); + require( + partitions >= 0 + && partitions <= settings.maxPartitions + && partitions <= in.remaining() / 16); + boolean[] matches = new boolean[partitions]; + Set unique = new java.util.HashSet<>(); + int dictionaryBytes = 0; + for (int id = 0; id < partitions; id++) { + require(in.remaining() >= 4); + int length = in.getInt(); + require( + length >= 12 + && length <= in.remaining() + && length + 4L <= settings.maxPartitionBytes - dictionaryBytes); + dictionaryBytes += 4 + length; + ByteBuffer encoded = in.slice(); + encoded.limit(length); + int arity = encoded.getInt(0); + require(arity >= 0 && 4L + ((arity + 71L) / 64) * 8 + arity * 8L <= length); + require(partitionType == null || arity == partitionType.getFieldCount()); + require(unique.add(encoded.asReadOnlyBuffer())); + if (partitionFilter == null) { + matches[id] = true; + } else { + byte[] bytes = new byte[length]; + encoded.get(bytes); + BinaryRow partition = SerializationUtils.deserializeBinaryRow(bytes); + matches[id] = partitionFilter.test(partition); + } + in.position(in.position() + length); + } + require(in.remaining() >= 4); + int count = in.getInt(); + require(count >= 0 && count <= MAX_BLOCKS && count <= in.remaining() / BLOCK_BYTES); + long nextOffset = headerLength; + long firstRecord = 0; + int totalRanges = 0; + List selected = new ArrayList<>(); + for (int i = 0; i < count; i++) { + require(in.remaining() >= BLOCK_BYTES); + long offset = in.getLong(); + long length = in.getLong(); + long records = in.getLong(); + require(offset == nextOffset && length > 0 && length <= manifest.fileSize() - offset); + require(records > 0 && records <= entries - firstRecord); + int partitionEncoding = Byte.toUnsignedInt(in.get()); + ByteBuffer partitionPayload = payload(in); + boolean partitionHit = true; + if (partitionEncoding == 0) { + require(!partitionPayload.hasRemaining()); + } else if (partitionEncoding == 1) { + require(partitionPayload.remaining() >= 4); + int ids = partitionPayload.getInt(); + require(ids > 0 && ids <= partitions && partitionPayload.remaining() == 4L * ids); + partitionHit = partitionFilter == null; + int previous = -1; + for (int j = 0; j < ids; j++) { + int id = partitionPayload.getInt(); + require(id > previous && id < partitions); + previous = id; + partitionHit |= matches[id]; + } + } + require(in.remaining() >= 5); + int rowEncoding = Byte.toUnsignedInt(in.get()); + ByteBuffer rowPayload = payload(in); + boolean rowHit = true; + if (rowEncoding == 0) { + require(!rowPayload.hasRemaining()); + } else if (rowEncoding == 1) { + require(rowPayload.remaining() >= 4); + int ranges = rowPayload.getInt(); + require( + ranges > 0 + && ranges <= settings.maxRanges - totalRanges + && rowPayload.remaining() == 16L * ranges); + totalRanges += ranges; + long min = rowPayload.getLong(rowPayload.position()); + long max = rowPayload.getLong(rowPayload.limit() - 8); + require(min >= 0 && max >= min); + boolean candidate = query == null || query.intersects(min, max); + rowHit = query == null; + long previous = -1; + for (int j = 0; j < ranges; j++) { + long start = rowPayload.getLong(); + long end = rowPayload.getLong(); + require(start >= 0 && end >= start && start > previous); + previous = end; + if (candidate && !rowHit) { + rowHit = ranges == 1 || query.intersects(start, end); + } + } + } + if (partitionHit && rowHit) { + selected.add(new Block(offset, length, firstRecord, records)); + } + nextOffset = offset + length; + firstRecord += records; + } + require(!in.hasRemaining() && nextOffset == manifest.fileSize() && firstRecord == entries); + return new Selection(header, selected); + } + + private static ByteBuffer payload(ByteBuffer in) throws IOException { + require(in.remaining() >= 4); + int length = in.getInt(); + require(length >= 0 && length <= in.remaining()); + ByteBuffer result = in.slice(); + result.limit(length); + in.position(in.position() + length); + return result; + } + + /** Validate the complete index before allowing any negative decision. */ + private static Selection selectLegacy( + byte[] data, + ManifestFileMeta manifest, + @Nullable RowRangeIndex query, + Settings settings) throws IOException { require(data.length >= 128 && data.length <= settings.maxBytes); int checksumOffset = data.length - DIGEST_BYTES; @@ -317,7 +636,7 @@ public static Selection select( nameHash, digest(manifest.fileName().getBytes(StandardCharsets.UTF_8)))); require(in.readLong() == manifest.fileSize()); long entries = Math.addExact(manifest.numAddedFiles(), manifest.numDeletedFiles()); - require(in.readLong() == entries && in.readInt() == checksumOffset - HEADER_BYTES); + require(in.readLong() == entries && in.readInt() == checksumOffset - LEGACY_HEADER_BYTES); int headerLength = in.readInt(); require( headerLength >= 21 @@ -353,8 +672,12 @@ public static Selection select( // adding redundant fields to the format or materializing the interval list. long maxRowId = ranges == 1 ? firstEnd : view.getLong(rangesEnd - Long.BYTES); require(minRowId >= 0 && firstEnd >= minRowId && maxRowId >= firstEnd); - boolean candidate = query.intersects(minRowId, maxRowId); - boolean hit = candidate && (ranges == 1 || query.intersects(minRowId, firstEnd)); + boolean candidate = query == null || query.intersects(minRowId, maxRowId); + boolean hit = + candidate + && (query == null + || ranges == 1 + || query.intersects(minRowId, firstEnd)); long previousEnd = firstEnd; for (int j = 1; j < ranges; j++) { long start = in.readLong(); @@ -383,7 +706,19 @@ public static Selection read( FileIO io, Path path, ManifestFileMeta manifest, - RowRangeIndex query, + @Nullable RowRangeIndex query, + Settings settings) { + return read(io, path, manifest, query, null, null, settings); + } + + @Nullable + public static Selection read( + FileIO io, + Path path, + ManifestFileMeta manifest, + @Nullable RowRangeIndex query, + @Nullable PartitionPredicate partitionFilter, + @Nullable RowType partitionType, Settings settings) { String indexFileName = fileName(manifest); if (indexFileName == null) { @@ -407,7 +742,7 @@ public static Selection read( } data = out.toByteArray(); } - return select(data, manifest, query, settings); + return select(data, manifest, query, partitionFilter, partitionType, settings); } catch (CancellationException failure) { throw failure; } catch (IOException | RuntimeException failure) { diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreScan.java b/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreScan.java index 5635d6458d49..4bf51db25bb6 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreScan.java @@ -500,7 +500,9 @@ private List readManifest( @Nullable Filter additionalTFilter) { ManifestFile manifestFile = manifestFileFactory.create(); - ManifestRowIdIndex.Selection selected = manifestFile.selectBlocks(manifest, rowRangeIndex); + ManifestRowIdIndex.Selection selected = + manifestFile.selectBlocks( + manifest, rowRangeIndex, manifestsReader.partitionFilter()); if (selected != null && selected.blocks().isEmpty()) { return Collections.emptyList(); } diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java new file mode 100644 index 000000000000..c9cf3664aa99 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java @@ -0,0 +1,320 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.manifest; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.data.BinaryRowWriter; +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.options.Options; +import org.apache.paimon.partition.PartitionPredicate; +import org.apache.paimon.predicate.PredicateBuilder; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.Range; +import org.apache.paimon.utils.RowRangeIndex; +import org.apache.paimon.utils.SerializationUtils; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.security.MessageDigest; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Base64; +import java.util.Collections; +import java.util.List; +import java.util.Properties; + +import static org.apache.paimon.manifest.ManifestRowIdIndexTest.meta; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +/** Independent partition/row-ID payloads and conservative resource degradation. */ +class ManifestBlockIndexTest { + private final RowType type = RowType.of(DataTypes.INT(), DataTypes.STRING()); + private final ManifestRowIdIndex.Settings defaults = + new ManifestRowIdIndex.Settings(new Options()); + + private byte[] fixture(String field) throws IOException { + Properties p = new Properties(); + try (java.io.InputStream in = + getClass().getResourceAsStream("/manifest-row-id-index-v2.txt")) { + p.load(in); + } + return Base64.getDecoder().decode(p.getProperty(field)); + } + + private byte[] partition(int p, String q) { + BinaryRow row = new BinaryRow(2); + BinaryRowWriter writer = new BinaryRowWriter(row); + writer.writeInt(0, p); + if (q == null) { + writer.setNullAt(1); + } else { + writer.writeString(1, BinaryString.fromString(q)); + } + writer.complete(); + return SerializationUtils.serializeBinaryRow(row); + } + + private RowRangeIndex query(long point) { + return RowRangeIndex.create(Collections.singletonList(new Range(point, point))); + } + + private PartitionPredicate part(int value) { + return PartitionPredicate.fromPredicate(type, new PredicateBuilder(type).equal(0, value)); + } + + @Test + void jointGoldenPreservesTuplesNullsAndDerivedOrdinals() throws Exception { + byte[] a = partition(7, "left"); + byte[] b = partition(9, null); + assertThat(a).isEqualTo(fixture("partitionA")); + assertThat(b).isEqualTo(fixture("partitionB")); + byte[] header = fixture("avroHeader"); + ManifestRowIdIndex.Builder builder = new ManifestRowIdIndex.Builder(defaults, header); + builder.beginBlock(header.length, 100, 3); + builder.add(0L, 10, a); + builder.add(5L, 5, a); + builder.add(20L, 5, b); + builder.endBlock(); + builder.beginBlock(header.length + 100, 200, 2); + builder.add((1L << 32) - 2, 5, b); + builder.add(8254058425445L, 1, a); + builder.endBlock(); + builder.beginBlock(header.length + 300, 100, 2); + builder.add(20L, 5, a); + builder.add(Long.MAX_VALUE, 1, b); + builder.endBlock(); + byte[] data = builder.serialize("manifest-golden", header.length + 400, 7); + assertThat(data).isEqualTo(fixture("indexJointV3")); + ManifestFileMeta meta = meta("manifest-golden", header.length + 400, 7); + PartitionPredicate filter = spy(part(7)); + assertThat( + ManifestRowIdIndex.select(data, meta, query(20), filter, type, defaults) + .blocks()) + .extracting(block -> block.firstRecord) + .containsExactly(0L, 5L); + verify(filter, times(2)).test(any(BinaryRow.class)); + PartitionPredicate nullFilter = + PartitionPredicate.fromPredicate(type, new PredicateBuilder(type).isNull(1)); + assertThat(ManifestRowIdIndex.select(data, meta, null, nullFilter, type, defaults).blocks()) + .hasSize(3); + assertThat(ManifestRowIdIndex.select(data, meta, null, part(99), type, defaults).blocks()) + .isEmpty(); + // The old container has no partition coverage; dictionary misses must not prune it. + assertThat( + ManifestRowIdIndex.select( + fixture("index"), meta, null, part(99), type, defaults) + .blocks()) + .hasSize(3); + } + + @Test + void unavailableDimensionsAreIndependentAndDoNotPoisonLaterBlocks() throws Exception { + Options options = new Options(); + options.set(CoreOptions.MANIFEST_INDEX_MAX_PARTITIONS, 1); + ManifestRowIdIndex.Settings settings = new ManifestRowIdIndex.Settings(options); + byte[] header = fixture("avroHeader"); + ManifestRowIdIndex.Builder builder = new ManifestRowIdIndex.Builder(settings, header); + builder.beginBlock(header.length, 100, 1); + builder.add(null, 10, partition(7, "left")); + builder.endBlock(); + builder.beginBlock(header.length + 100, 100, 1); + builder.add(200L, 10, partition(9, null)); // dictionary budget exceeded + builder.endBlock(); + builder.beginBlock(header.length + 200, 100, 1); + builder.add(300L, 10, partition(7, "left")); // an existing dictionary ID remains usable + builder.endBlock(); + byte[] data = builder.serialize("m", header.length + 300, 3); + ManifestFileMeta meta = meta("m", header.length + 300, 3); + assertThat(ManifestRowIdIndex.select(data, meta, null, part(9), type, settings).blocks()) + .extracting(block -> block.firstRecord) + .containsExactly(1L); + assertThat( + ManifestRowIdIndex.select(data, meta, query(999), part(7), type, settings) + .blocks()) + .extracting(block -> block.firstRecord) + .containsExactly(0L); + assertThat( + ManifestRowIdIndex.select(data, meta, query(200), part(9), type, settings) + .blocks()) + .extracting(block -> block.firstRecord) + .containsExactly(1L); + } + + @Test + void coarseningContinuesThroughTheEntireBlockAndDetectsUnknownRows() throws Exception { + Options options = new Options(); + options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_MAX_RANGES, 1); + ManifestRowIdIndex.Settings settings = new ManifestRowIdIndex.Settings(options); + byte[] header = fixture("avroHeader"); + for (boolean unknown : new boolean[] {false, true}) { + ManifestRowIdIndex.Builder builder = new ManifestRowIdIndex.Builder(settings, header); + builder.beginBlock(header.length, 100, 4); + builder.add(100L, 10, partition(7, "left")); + builder.add(300L, 10, partition(7, "left")); + builder.add(10L, 10, partition(7, "left")); + builder.add(unknown ? null : Long.MAX_VALUE, 1, partition(7, "left")); + builder.endBlock(); + byte[] data = builder.serialize("m", header.length + 100, 4); + ManifestFileMeta meta = meta("m", header.length + 100, 4); + for (long point : new long[] {10, 100, 200, Long.MAX_VALUE}) { + assertThat(ManifestRowIdIndex.select(data, meta, query(point), settings).blocks()) + .hasSize(1); + } + assertThat(ManifestRowIdIndex.select(data, meta, query(0), settings).blocks()) + .hasSize(unknown ? 1 : 0); + assertThat( + ManifestRowIdIndex.select(data, meta, null, part(9), type, settings) + .blocks()) + .isEmpty(); + } + } + + private List positions(byte[] data) { + ByteBuffer in = ByteBuffer.wrap(data); + in.position(60); + int header = in.getInt(); + in.position(in.position() + header); + int partitions = in.getInt(); + for (int i = 0; i < partitions; i++) { + int length = in.getInt(); + in.position(in.position() + length); + } + int blocks = in.getInt(); + List result = new ArrayList<>(); + for (int i = 0; i < blocks; i++) { + int block = in.position(); + in.position(block + 24); + int partition = in.position(); + in.get(); + int length = in.getInt(); + in.position(in.position() + length); + int row = in.position(); + in.get(); + length = in.getInt(); + in.position(in.position() + length); + result.add(new int[] {block, partition, row}); + } + return result; + } + + private byte[] checksum(byte[] data) throws Exception { + byte[] hash = + MessageDigest.getInstance("SHA-256").digest(Arrays.copyOf(data, data.length - 32)); + System.arraycopy(hash, 0, data, data.length - 32, 32); + return data; + } + + @Test + void unknownUnsignedEncodingsSkipOnlyTheirDimensionAndMalformedPayloadsFail() throws Exception { + byte[] good = fixture("indexJointV3"); + int[] first = positions(good).get(0); + ManifestFileMeta meta = meta("manifest-golden", fixture("avroHeader").length + 400, 7); + byte[] data = good.clone(); + data[first[1]] = (byte) 200; + assertThat( + ManifestRowIdIndex.select( + checksum(data), meta, query(0), part(99), type, defaults) + .blocks()) + .extracting(block -> block.firstRecord) + .containsExactly(0L); + data = good.clone(); + data[first[2]] = (byte) 201; + assertThat( + ManifestRowIdIndex.select( + checksum(data), meta, query(16), part(7), type, defaults) + .blocks()) + .extracting(block -> block.firstRecord) + .containsExactly(0L); + for (int position : new int[] {first[1], first[2]}) { + byte[] bad = good.clone(); + bad[position] = 0; // encoding 0 cannot have payload bytes + checksum(bad); + assertThatThrownBy(() -> ManifestRowIdIndex.select(bad, meta, query(0), defaults)) + .isInstanceOf(IOException.class); + byte[] invalidLength = good.clone(); + ByteBuffer.wrap(invalidLength).putInt(position + 1, -1); + checksum(invalidLength); + assertThatThrownBy( + () -> + ManifestRowIdIndex.select( + invalidLength, meta, query(0), defaults)) + .isInstanceOf(IOException.class); + } + // A checksummed directory with missing bytes/entries must still be rejected. + byte[] bad = good.clone(); + ByteBuffer.wrap(bad).putLong(first[0] + 16, 2); + checksum(bad); + assertThatThrownBy(() -> ManifestRowIdIndex.select(bad, meta, query(0), defaults)) + .isInstanceOf(IOException.class); + byte[] badRange = good.clone(); + // Row payload begins after its encoding and length, then the range-count integer. + ByteBuffer.wrap(badRange).putLong(first[2] + 9 + 16, 9L); + checksum(badRange); + assertThatThrownBy( + () -> + ManifestRowIdIndex.select( + badRange, meta, query(999), part(99), type, defaults)) + .isInstanceOf(IOException.class); + byte[] badId = good.clone(); + ByteBuffer.wrap(badId).putInt(first[1] + 9, 999); + checksum(badId); + assertThatThrownBy(() -> ManifestRowIdIndex.select(badId, meta, query(999), defaults)) + .isInstanceOf(IOException.class); + } + + @Test + void tightByteBudgetKeepsAllDescriptorsOrOmitsTheWholeFile() throws Exception { + Options options = new Options(); + options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_MAX_BYTES, 280); + ManifestRowIdIndex.Settings settings = new ManifestRowIdIndex.Settings(options); + byte[] header = fixture("avroHeader"); + ManifestRowIdIndex.Builder builder = new ManifestRowIdIndex.Builder(settings, header); + for (int i = 0; i < 3; i++) { + builder.beginBlock(header.length + 100L * i, 100, 1); + builder.add(i * 100L, 10, partition(7, "left")); + builder.endBlock(); + } + byte[] data = builder.serialize("m", header.length + 300, 3); + assertThat(data.length).isLessThanOrEqualTo(280); + assertThat( + ManifestRowIdIndex.select( + data, + meta("m", header.length + 300, 3), + query(999), + part(99), + type, + settings) + .blocks()) + .extracting(block -> block.firstRecord) + .containsExactly(0L, 1L, 2L); + builder.beginBlock(header.length + 300, 100, 1); + builder.add(300L, 1, partition(7, "left")); + builder.endBlock(); + assertThat(builder.serialize("m", header.length + 400, 4)).isNull(); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java index 2e29c7ff8527..d8aa610987d3 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java @@ -19,6 +19,7 @@ package org.apache.paimon.manifest; import org.apache.paimon.CoreOptions; +import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.GenericRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.format.FileFormat; @@ -1530,7 +1531,56 @@ public PositionOutputStream newOutputStream(Path path, boolean overwrite) } @Test - void testUnknownRowIdDisablesIndexAndNoQueryDoesNotReadSidecar() { + void testPartitionOnlyPlanningUsesBlocksWithoutRowIds() { + Options options = new Options(); + options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_WRITE, true); + options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_READ, true); + RecordingFileIO io = new RecordingFileIO(); + ManifestFile.Factory factory = + createManifestFileFactory(tempDir.toString(), Long.MAX_VALUE, options, io); + BinaryRow first = gen.next().partition(); + BinaryRow second = gen.next().partition(); + while (second.equals(first)) { + second = gen.next().partition(); + } + List entries = new ArrayList<>(); + for (int i = 0; i < 4000; i++) { + ManifestEntry entry = gen.next(); + entries.add( + ManifestEntry.create( + FileKind.ADD, + i < 1000 ? first : second, + entry.bucket(), + entry.totalBuckets(), + entry.file())); + } + ManifestFileMeta meta = factory.create().write(entries).get(0); + ManifestsReader lists = mock(ManifestsReader.class); + when(lists.partitionFilter()) + .thenReturn( + PartitionPredicate.fromMultiple( + DEFAULT_PART_TYPE, Collections.singletonList(first))); + AppendOnlyFileStoreScan scan = + new AppendOnlyFileStoreScan( + lists, + null, + null, + null, + mock(TableSchema.class), + factory, + 2, + false, + false, + false); + io.reset(); + List actual = scan.readManifest(meta); + assertThat(actual).containsExactlyElementsOf(entries.subList(0, 1000)); + assertThat(io.bytes.get()).isLessThan(meta.fileSize()); + assertThat(io.opened).hasSize(2); + } + + @Test + void testUnknownRowIdKeepsPartitionIndexAndNoQueryDoesNotReadSidecar() { Options options = new Options(); options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_WRITE, true); options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_READ, true); @@ -1539,12 +1589,12 @@ void testUnknownRowIdDisablesIndexAndNoQueryDoesNotReadSidecar() { createManifestFileFactory(tempDir.toString(), Long.MAX_VALUE, options, fileIO) .create(); ManifestFileMeta meta = manifests.write(Collections.singletonList(gen.next())).get(0); - assertThat(ManifestRowIdIndex.fileName(meta)).isNull(); + assertThat(ManifestRowIdIndex.fileName(meta)).isNotNull(); assertThat( java.nio.file.Files.exists( tempDir.resolve("manifest") .resolve(meta.fileName() + ManifestRowIdIndex.SUFFIX))) - .isFalse(); + .isTrue(); fileIO.reset(); assertThat(manifests.mayContainRowIds(meta, null)).isTrue(); diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestRowIdIndexTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestRowIdIndexTest.java index 6a5f5f0a9764..8b1810bfe0b0 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestRowIdIndexTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestRowIdIndexTest.java @@ -111,7 +111,7 @@ void crossLanguageFormatAndBlockOrdinals() throws Exception { builder.add(Long.MAX_VALUE, 1); builder.endBlock(); byte[] data = builder.serialize("manifest-golden", header.length + 400, 7); - assertThat(data).isEqualTo(golden()); + assertThat(data).isEqualTo(Base64.getDecoder().decode(fixture().getProperty("indexV3"))); ManifestFileMeta meta = goldenMeta(); for (long point : new long[] { @@ -217,7 +217,7 @@ void malformedIntervalsStillFallbackAfterMinMaxRejectionOrAnEarlyHit() throws Ex } @Test - void hugeRangesAreNotExpandedAndInvalidCoverageDisablesIndex() throws Exception { + void hugeRangesAreNotExpandedAndInvalidCoverageRetainsBlocks() throws Exception { byte[] header = header(); ManifestRowIdIndex.Builder builder = new ManifestRowIdIndex.Builder(settings, header); builder.beginBlock(header.length, 100, 2); @@ -233,13 +233,26 @@ void hugeRangesAreNotExpandedAndInvalidCoverageDisablesIndex() throws Exception builder.beginBlock(header.length, 100, 1); builder.add(first, 2); builder.endBlock(); - assertThat(builder.serialize("m", header.length + 100, 1)).isNull(); + assertThat( + select( + builder.serialize("m", header.length + 100, 1), + meta("m", header.length + 100, 1), + 100) + .blocks()) + .hasSize(1); } for (long count : new long[] {0, -1}) { builder = new ManifestRowIdIndex.Builder(settings, header); builder.beginBlock(header.length, 100, 1); builder.add(0L, count); - assertThat(builder.serialize("m", 1, 1)).isNull(); + builder.endBlock(); + assertThat( + select( + builder.serialize("m", header.length + 100, 1), + meta("m", header.length + 100, 1), + 100) + .blocks()) + .hasSize(1); } Options options = new Options(); options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_MAX_RANGES, 1); @@ -247,7 +260,14 @@ void hugeRangesAreNotExpandedAndInvalidCoverageDisablesIndex() throws Exception builder.beginBlock(header.length, 100, 2); builder.add(0L, 1); builder.add(10L, 1); - assertThat(builder.serialize("m", 1, 2)).isNull(); + builder.endBlock(); + assertThat( + select( + builder.serialize("m", header.length + 100, 2), + meta("m", header.length + 100, 2), + 5) + .blocks()) + .hasSize(1); options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_MAX_BYTES, 128); builder = new ManifestRowIdIndex.Builder(new ManifestRowIdIndex.Settings(options), header); assertThat(builder.serialize("m", 1, 2)).isNull(); diff --git a/paimon-core/src/test/resources/manifest-row-id-index-v2.txt b/paimon-core/src/test/resources/manifest-row-id-index-v2.txt index c58cf599b276..4ff6fc842c8d 100644 --- a/paimon-core/src/test/resources/manifest-row-id-index-v2.txt +++ b/paimon-core/src/test/resources/manifest-row-id-index-v2.txt @@ -17,3 +17,7 @@ avroHeader=T2JqAQQUYXZyby5jb2RlYwhudWxsFmF2cm8uc2NoZW1hDCJsb25nIgAAAAAAAAAAAAAAAAAAAAAA index=UEFJTVJJRFgAAgACAAAAAS9Hlrm5B3S+oqDXSD494xrafUwJPN8G7QLJnPsSVcDPAAAAAAAAAckAAAAAAAAABwAAAQ0AAAA5T2JqAQQUYXZyby5jb2RlYwhudWxsFmF2cm8uc2NoZW1hDCJsb25nIgAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAA5AAAAAAAAAGQAAAAAAAAAAAAAAAAAAAADAAAAAgAAAAAAAAAAAAAAAAAAAAkAAAAAAAAAFAAAAAAAAAAYAAAAAAAAAJ0AAAAAAAAAyAAAAAAAAAADAAAAAAAAAAIAAAACAAAAAP////4AAAABAAAAAgAAB4HMOGxlAAAHgcw4bGUAAAAAAAABZQAAAAAAAABkAAAAAAAAAAUAAAAAAAAAAgAAAAIAAAAAAAAAFAAAAAAAAAAYf/////////9//////////xekCXjgYJlWPiPlQ80IyyKSmIPH5z5iMhyRa8BhBph5 +partitionA=AAAAAgAAAAAAAAAABwAAAAAAAABsZWZ0AAAAhA== +partitionB=AAAAAgACAAAAAAAACQAAAAAAAAAAAAAAAAAAAA== +indexV3=UEFJTVJJRFgAAAADL0eWubkHdL6ioNdIPj3jGtp9TAk83wbtAsmc+xJVwM8AAAAAAAAByQAAAAAAAAAHAAAAOU9iagEEFGF2cm8uY29kZWMIbnVsbBZhdnJvLnNjaGVtYQwibG9uZyIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAADkAAAAAAAAAZAAAAAAAAAADAAAAAAABAAAAJAAAAAIAAAAAAAAAAAAAAAAAAAAJAAAAAAAAABQAAAAAAAAAGAAAAAAAAACdAAAAAAAAAMgAAAAAAAAAAgAAAAAAAQAAACQAAAACAAAAAP////4AAAABAAAAAgAAB4HMOGxlAAAHgcw4bGUAAAAAAAABZQAAAAAAAABkAAAAAAAAAAIAAAAAAAEAAAAkAAAAAgAAAAAAAAAUAAAAAAAAABh//////////3//////////ViRH8zjNd2Q9F4olffQ2ZQiExUDkv3wPwoBnsEGW3sQ= +indexJointV3=UEFJTVJJRFgAAAADL0eWubkHdL6ioNdIPj3jGtp9TAk83wbtAsmc+xJVwM8AAAAAAAAByQAAAAAAAAAHAAAAOU9iagEEFGF2cm8uY29kZWMIbnVsbBZhdnJvLnNjaGVtYQwibG9uZyIAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAcAAAAAgAAAAAAAAAABwAAAAAAAABsZWZ0AAAAhAAAABwAAAACAAIAAAAAAAAJAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAA5AAAAAAAAAGQAAAAAAAAAAwEAAAAMAAAAAgAAAAAAAAABAQAAACQAAAACAAAAAAAAAAAAAAAAAAAACQAAAAAAAAAUAAAAAAAAABgAAAAAAAAAnQAAAAAAAADIAAAAAAAAAAIBAAAADAAAAAIAAAAAAAAAAQEAAAAkAAAAAgAAAAD////+AAAAAQAAAAIAAAeBzDhsZQAAB4HMOGxlAAAAAAAAAWUAAAAAAAAAZAAAAAAAAAACAQAAAAwAAAACAAAAAAAAAAEBAAAAJAAAAAIAAAAAAAAAFAAAAAAAAAAYf/////////9//////////4mo+QwqXuGgI0WNm1HdwCQA6dlu7UP8TigmfZSh14mZ diff --git a/paimon-python/pypaimon/common/options/core_options.py b/paimon-python/pypaimon/common/options/core_options.py index 3d500fd7a5a7..d9a084a2ec21 100644 --- a/paimon-python/pypaimon/common/options/core_options.py +++ b/paimon-python/pypaimon/common/options/core_options.py @@ -321,6 +321,18 @@ class CoreOptions: .default_value(8388608) ) + MANIFEST_INDEX_MAX_PARTITIONS: ConfigOption[int] = ( + ConfigOptions.key("manifest.index.max-partitions") + .int_type() + .default_value(65536) + ) + + MANIFEST_INDEX_MAX_PARTITION_BYTES: ConfigOption[int] = ( + ConfigOptions.key("manifest.index.max-partition-bytes") + .int_type() + .default_value(1048576) + ) + MANIFEST_COMPRESSION: ConfigOption[str] = ( ConfigOptions.key("manifest.compression") .string_type() diff --git a/paimon-python/pypaimon/manifest/manifest_file_manager.py b/paimon-python/pypaimon/manifest/manifest_file_manager.py index 9ec5a1ef8ba7..11030bf83236 100644 --- a/paimon-python/pypaimon/manifest/manifest_file_manager.py +++ b/paimon-python/pypaimon/manifest/manifest_file_manager.py @@ -62,16 +62,20 @@ def read_entries_parallel(self, manifest_files: List[ManifestFileMeta], manifest early_record_filter: Optional[Callable[[dict], bool]] = None, partition_filter=None, row_ranges=None, + index_partition_filter=None, ) -> List[ManifestEntry]: settings = Settings.from_options(self.table.options) query = Query(row_ranges) if settings.read and row_ranges is not None else None + if index_partition_filter is None: + index_partition_filter = partition_filter def _process_single_manifest(manifest_file: ManifestFileMeta): path = f"{self.manifest_path}/{manifest_file.file_name}" selected = None - if query is not None: - selected = read_index(self.file_io, path, manifest_file, query, settings) + if settings.read and (query is not None or index_partition_filter is not None): + selected = read_index(self.file_io, path, manifest_file, query, settings, + index_partition_filter, self.partition_keys_fields) if selected is not None and not selected.blocks: return [] return self.read( diff --git a/paimon-python/pypaimon/manifest/row_id_index.py b/paimon-python/pypaimon/manifest/row_id_index.py index 3a3bba51ee41..47d319f1ba2d 100644 --- a/paimon-python/pypaimon/manifest/row_id_index.py +++ b/paimon-python/pypaimon/manifest/row_id_index.py @@ -17,7 +17,7 @@ """Complete row-id interval unions with Avro block offsets and entry ordinals. -Version 2 uses fixed-width big-endian integers; no library-specific bitmap encoding. +Version 3 shares a partition dictionary and independently framed block payloads. """ import hashlib @@ -33,6 +33,7 @@ from pypaimon.common.options.core_options import CoreOptions from pypaimon.utils.range import Range +from pypaimon.table.row.generic_row import GenericRowSerializer, GenericRowDeserializer LOG = logging.getLogger(__name__) SUFFIX = '.row-id-index' @@ -40,8 +41,12 @@ MAX_ROW_ID = (1 << 63) - 1 MAX_AVRO_HEADER = 1024 * 1024 READ_BUFFER_BYTES = 1024 * 1024 -HEADER = struct.Struct('>8sHHI32sqqI') -BLOCK = struct.Struct('>qqqqI') +LEGACY_HEADER = struct.Struct('>8sHHI32sqqI') +LEGACY_BLOCK = struct.Struct('>qqqqI') +HEADER = struct.Struct('>8sI32sqq') +BLOCK = struct.Struct('>qqq') +MAX_BLOCKS = 131072 +BLOCK_BYTES = 34 PAIR = struct.Struct('>qq') LONG = struct.Struct('>q') _PROPAGATED_ERRORS = (InterruptedError, CancelledError, ArrowCancelled, MemoryError, RecursionError) @@ -53,8 +58,14 @@ class Settings: read: bool = False max_ranges: int = 131072 max_bytes: int = 8 * 1024 * 1024 + max_partitions: int = 65536 + max_partition_bytes: int = 1024 * 1024 def __post_init__(self): + if not 1 <= self.max_partitions <= 1048576: + raise ValueError('Invalid manifest.index.max-partitions') + if not 1 <= self.max_partition_bytes <= 64 * 1024 * 1024: + raise ValueError('Invalid manifest.index.max-partition-bytes') if not 1 <= self.max_ranges <= 1048576: raise ValueError('manifest.row-id-index.max-ranges must be in [1, 1048576]') if not 128 <= self.max_bytes <= 64 * 1024 * 1024: @@ -66,7 +77,9 @@ def from_options(cls, options): options.options.get(CoreOptions.MANIFEST_ROW_ID_INDEX_WRITE), options.options.get(CoreOptions.MANIFEST_ROW_ID_INDEX_READ), options.options.get(CoreOptions.MANIFEST_ROW_ID_INDEX_MAX_RANGES), - options.options.get(CoreOptions.MANIFEST_ROW_ID_INDEX_MAX_BYTES)) + options.options.get(CoreOptions.MANIFEST_ROW_ID_INDEX_MAX_BYTES), + options.options.get(CoreOptions.MANIFEST_INDEX_MAX_PARTITIONS), + options.options.get(CoreOptions.MANIFEST_INDEX_MAX_PARTITION_BYTES)) @dataclass(frozen=True) @@ -97,45 +110,57 @@ def intersects(self, first, last): class Builder: def __init__(self, settings, header): self.settings = settings + self.header = header self.complete = (header is not None and len(header) <= MAX_AVRO_HEADER - and len(header) + HEADER.size + 40 <= settings.max_bytes) - self.payload = bytearray() + and HEADER.size + 44 + len(header) <= settings.max_bytes) + self.dictionary = {} + self.dictionary_bytes = 0 + self.optional_bytes = 0 + self.saved_ranges = 0 + self.blocks = [] self.ranges = [] - self.count_position = 4 + len(header) if self.complete else 0 - self.next_offset = len(header) if self.complete else 0 + self.partition_ids = set() + self.next_offset = len(header) if header is not None else 0 self.next_record = 0 - self.range_count = 0 - self.blocks = 0 self.current = None - self.entries_in_block = 0 - if self.complete: - self.payload.extend(struct.pack('>I', len(header))) - self.payload.extend(header) - self.payload.extend(struct.pack('>I', 0)) - - def _disable(self, reason): - self.complete = False - self.payload.clear() - self.ranges.clear() - LOG.debug('Omitting manifest row-id block index: %s', reason) def begin_block(self, offset, length, records): if not self.complete: return _require(self.current is None and offset == self.next_offset and length > 0 and records > 0) + if (len(self.blocks) == MAX_BLOCKS or HEADER.size + 44 + len(self.header) + + (len(self.blocks) + 1) * BLOCK_BYTES > self.settings.max_bytes): + self.complete = False + self.blocks.clear() + self.dictionary.clear() + return self.current = Block(offset, length, self.next_record, records) self.entries_in_block = 0 + self.row_available = self.partition_available = True + self.coarse = False + self.min = MAX_ROW_ID + self.max = -1 + self.ranges.clear() + self.partition_ids.clear() - def add(self, first, count): + def add(self, first, count, partition=None): if not self.complete: return _require(self.current is not None) self.entries_in_block += 1 + self._add_partition(partition) + if not self.row_available: + return if (first is None or first < 0 or count <= 0 or first > MAX_ROW_ID or count - 1 > MAX_ROW_ID - first): - self._disable('unknown or invalid row-id coverage') + self.row_available = False + self.ranges.clear() return end = first + count - 1 + self.min, self.max = min(self.min, first), max(self.max, end) + # Even after coarsening, inspect every entry to extend bounds or mark coverage unknown. + if self.coarse: + return left = bisect_left(self.ranges, (first, -1)) if left and self.ranges[left - 1][1] >= first - 1: left -= 1 @@ -144,39 +169,98 @@ def add(self, first, count): first = min(first, self.ranges[right][0]) end = max(end, self.ranges[right][1]) right += 1 - if self.range_count + len(self.ranges) - (right - left) >= self.settings.max_ranges: - self._disable('range budget exceeded') + if len(self.ranges) - (right - left) >= min(self.settings.max_ranges, self.settings.max_bytes // 16): + self.coarse = True + self.ranges.clear() + else: + self.ranges[left:right] = [(first, end)] + + def _add_partition(self, partition): + if not self.partition_available: + return + if partition is None: + self.partition_available = False + self.partition_ids.clear() return - self.ranges[left:right] = [(first, end)] + partition = bytes(partition) + id_ = self.dictionary.get(partition) + if id_ is None: + if (len(self.dictionary) >= self.settings.max_partitions + or len(partition) + 4 > min(self.settings.max_partition_bytes, self.settings.max_bytes) + - self.dictionary_bytes): + self.partition_available = False + self.partition_ids.clear() + return + id_ = len(self.dictionary) + self.dictionary[partition] = id_ + self.dictionary_bytes += 4 + len(partition) + self.partition_ids.add(id_) def end_block(self): if not self.complete: return block = self.current - _require(block is not None and self.entries_in_block == block.record_count and self.ranges) - if HEADER.size + 32 + len(self.payload) + BLOCK.size + 16 * len(self.ranges) > self.settings.max_bytes: - self._disable('serialized byte budget exceeded') - return - self.payload.extend(BLOCK.pack(block.offset, block.length, block.first_record, - block.record_count, len(self.ranges))) - for first, end in self.ranges: - self.payload.extend(PAIR.pack(first, end)) + _require(block is not None and self.entries_in_block == block.record_count) + row_payload = partition_payload = b'' + row_count = 0 + if self.row_available: + if (self.coarse or len(self.ranges) > self.settings.max_ranges - self.saved_ranges + or 4 + 16 * len(self.ranges) > self.settings.max_bytes - self.optional_bytes): + self.ranges = [(self.min, self.max)] + if (self.saved_ranges < self.settings.max_ranges + and 4 + 16 * len(self.ranges) <= self.settings.max_bytes - self.optional_bytes): + row_payload = struct.pack('>I', len(self.ranges)) + b''.join(PAIR.pack(*r) for r in self.ranges) + row_count = len(self.ranges) + self.optional_bytes += len(row_payload) + if (self.partition_available + and 4 + 4 * len(self.partition_ids) <= self.settings.max_bytes - self.optional_bytes): + partition_payload = struct.pack('>I', len(self.partition_ids)) + partition_payload += b''.join(struct.pack('>I', id_) for id_ in sorted(self.partition_ids)) + self.optional_bytes += len(partition_payload) + self.blocks.append([block, partition_payload, row_payload]) + self.saved_ranges += row_count self.next_offset = block.offset + block.length self.next_record = block.first_record + block.record_count - self.range_count += len(self.ranges) - self.blocks += 1 self.ranges.clear() + self.partition_ids.clear() self.current = None def serialize(self, name, file_size, entry_count): if not self.complete: return None _require(self.current is None and self.next_offset == file_size and self.next_record == entry_count) - struct.pack_into('>I', self.payload, self.count_position, self.blocks) - header = HEADER.pack(MAGIC, 2, 2, 1, hashlib.sha256(name.encode('utf-8')).digest(), - file_size, entry_count, len(self.payload)) - data = header + self.payload - return data + hashlib.sha256(data).digest() + size = (HEADER.size + 44 + len(self.header) + self.dictionary_bytes + + len(self.blocks) * BLOCK_BYTES + self.optional_bytes) + for item in self.blocks: + if size <= self.settings.max_bytes: + break + size -= len(item[2]) + self.optional_bytes -= len(item[2]) + item[2] = b'' + if size > self.settings.max_bytes: + size -= self.dictionary_bytes + self.dictionary_bytes = 0 + self.dictionary.clear() + for item in self.blocks: + size -= len(item[1]) + self.optional_bytes -= len(item[1]) + item[1] = b'' + _require(size <= self.settings.max_bytes) + data = bytearray(HEADER.pack( + MAGIC, 3, hashlib.sha256(name.encode('utf-8')).digest(), file_size, entry_count)) + data.extend(struct.pack('>I', len(self.header))) + data.extend(self.header) + data.extend(struct.pack('>I', len(self.dictionary))) + for partition in self.dictionary: + data.extend(struct.pack('>I', len(partition))) + data.extend(partition) + data.extend(struct.pack('>I', len(self.blocks))) + for block, partitions, row_ids in self.blocks: + data.extend(BLOCK.pack(block.offset, block.length, block.record_count)) + for payload in (partitions, row_ids): + data.extend(struct.pack('>BI', 1 if payload else 0, len(payload))) + data.extend(payload) + return bytes(data) + hashlib.sha256(data).digest() def build_from_entries(avro_bytes, entries, name, settings): @@ -193,7 +277,7 @@ def build_from_entries(avro_bytes, entries, name, settings): _require(end <= len(entries)) for i in range(position, end): entry = entries[i] - builder.add(entry.file.first_row_id, entry.file.row_count) + builder.add(entry.file.first_row_id, entry.file.row_count, GenericRowSerializer.to_bytes(entry.partition)) if not builder.complete: break builder.end_block() @@ -207,17 +291,124 @@ def _require(condition): raise ValueError('Invalid, unsupported, mismatched or over-budget manifest row-id block index') -def select(data, manifest, query, settings): - if not isinstance(query, Query): +def select(data, manifest, query, settings, partition_filter=None, partition_fields=None): + if query is not None and not isinstance(query, Query): + query = Query(query) + _require(128 <= len(data) <= settings.max_bytes) + if struct.unpack_from('>I', data, 8)[0] == 0x00020002: + return _select_legacy(data, manifest, query, settings) + limit = len(data) - 32 + _require(hashlib.sha256(data[:limit]).digest() == data[limit:]) + magic, version, name_hash, size, entries = HEADER.unpack_from(data) + _require(magic == MAGIC and version == 3) + _require(name_hash == hashlib.sha256(manifest.file_name.encode('utf-8')).digest()) + _require(size == manifest.file_size and entries == manifest.num_added_files + manifest.num_deleted_files) + header_length, = struct.unpack_from('>I', data, HEADER.size) + offset = HEADER.size + 4 + _require(21 <= header_length <= MAX_AVRO_HEADER and header_length <= limit - offset - 8) + header = bytes(data[offset:offset + header_length]) + _require(header[:4] == b'Obj\x01') + offset += header_length + partitions, = struct.unpack_from('>I', data, offset) + offset += 4 + _require(partitions <= settings.max_partitions and partitions <= (limit - offset) // 16) + matches = [] + unique = set() + dictionary_bytes = 0 + for _ in range(partitions): + _require(offset + 4 <= limit) + length, = struct.unpack_from('>I', data, offset) + offset += 4 + _require(12 <= length <= limit - offset and length + 4 <= settings.max_partition_bytes - dictionary_bytes) + dictionary_bytes += 4 + length + partition = bytes(data[offset:offset + length]) + arity, = struct.unpack_from('>i', partition) + _require(arity >= 0 and 4 + ((arity + 71) // 64) * 8 + arity * 8 <= length) + _require(partition_fields is None or arity == len(partition_fields)) + _require(partition not in unique) + unique.add(partition) + if partition_filter is None: + matches.append(True) + else: + _require(partition_fields is not None) + matches.append(partition_filter.test(GenericRowDeserializer.from_bytes(partition, partition_fields))) + offset += length + _require(offset + 4 <= limit) + blocks, = struct.unpack_from('>I', data, offset) + offset += 4 + _require(blocks <= MAX_BLOCKS and blocks <= (limit - offset) // BLOCK_BYTES) + next_offset = header_length + first_record = 0 + total_ranges = 0 + selected = [] + for _ in range(blocks): + _require(offset + BLOCK_BYTES <= limit) + file_offset, length, count = BLOCK.unpack_from(data, offset) + offset += BLOCK.size + _require(file_offset == next_offset and 0 < length <= size - file_offset) + _require(0 < count <= entries - first_record) + partition_encoding, payload_length = struct.unpack_from('>BI', data, offset) + offset += 5 + _require(payload_length <= limit - offset) + partition_hit = True + if partition_encoding == 0: + _require(payload_length == 0) + elif partition_encoding == 1: + _require(payload_length >= 4) + ids, = struct.unpack_from('>I', data, offset) + _require(0 < ids <= partitions and 4 + 4 * ids == payload_length) + partition_hit = partition_filter is None + previous = -1 + for j in range(ids): + id_, = struct.unpack_from('>i', data, offset + 4 + 4 * j) + _require(previous < id_ < partitions) + previous = id_ + partition_hit |= matches[id_] + offset += payload_length + _require(offset + 5 <= limit) + row_encoding, payload_length = struct.unpack_from('>BI', data, offset) + offset += 5 + _require(payload_length <= limit - offset) + row_hit = True + if row_encoding == 0: + _require(payload_length == 0) + elif row_encoding == 1: + _require(payload_length >= 4) + ranges, = struct.unpack_from('>I', data, offset) + _require(0 < ranges <= settings.max_ranges - total_ranges and 4 + 16 * ranges == payload_length) + total_ranges += ranges + min_row_id, = LONG.unpack_from(data, offset + 4) + max_row_id, = LONG.unpack_from(data, offset + payload_length - 8) + _require(min_row_id >= 0 and max_row_id >= min_row_id) + candidate = query is None or query.intersects(min_row_id, max_row_id) + row_hit = query is None + previous = -1 + for j in range(ranges): + start, end = PAIR.unpack_from(data, offset + 4 + 16 * j) + _require(start >= 0 and end >= start and start > previous) + previous = end + if candidate and not row_hit: + row_hit = ranges == 1 or query.intersects(start, end) + offset += payload_length + if partition_hit and row_hit: + selected.append(Block(file_offset, length, first_record, count)) + next_offset = file_offset + length + first_record += count + _require(offset == limit and next_offset == size and first_record == entries) + return Selection(header, tuple(selected)) + + +def _select_legacy(data, manifest, query, settings): + if query is not None and not isinstance(query, Query): query = Query(query) _require(128 <= len(data) <= settings.max_bytes) _require(hashlib.sha256(data[:-32]).digest() == data[-32:]) - magic, version, codec, flags, name_hash, size, entries, length = HEADER.unpack_from(data) + magic, version, codec, flags, name_hash, size, entries, length = LEGACY_HEADER.unpack_from(data) _require((magic, version, codec, flags) == (MAGIC, 2, 2, 1)) _require(name_hash == hashlib.sha256(manifest.file_name.encode('utf-8')).digest()) _require(size == manifest.file_size and entries == manifest.num_added_files + manifest.num_deleted_files) - _require(length == len(data) - HEADER.size - 32) - offset = HEADER.size + _require(length == len(data) - LEGACY_HEADER.size - 32) + offset = LEGACY_HEADER.size header_length, = struct.unpack_from('>I', data, offset) offset += 4 _require(21 <= header_length <= MAX_AVRO_HEADER and header_length <= len(data) - offset - 36) @@ -232,8 +423,8 @@ def select(data, manifest, query, settings): total_ranges = 0 selected = [] for _ in range(blocks): - file_offset, block_length, first, count, ranges = BLOCK.unpack_from(data, offset) - offset += BLOCK.size + file_offset, block_length, first, count, ranges = LEGACY_BLOCK.unpack_from(data, offset) + offset += LEGACY_BLOCK.size _require(file_offset == next_offset and 0 < block_length <= size - file_offset) _require(first == next_record and 0 < count <= entries - first) _require(0 < ranges <= settings.max_ranges - total_ranges and ranges <= (len(data) - 32 - offset) // 16) @@ -244,8 +435,8 @@ def select(data, manifest, query, settings): # The sorted interval list already contains min/max; no format change or extra fields. max_row_id = first_end if ranges == 1 else LONG.unpack_from(data, ranges_end - LONG.size)[0] _require(min_row_id >= 0 and first_end >= min_row_id and max_row_id >= first_end) - candidate = query.intersects(min_row_id, max_row_id) - hit = candidate and (ranges == 1 or query.intersects(min_row_id, first_end)) + candidate = query is None or query.intersects(min_row_id, max_row_id) + hit = candidate and (query is None or ranges == 1 or query.intersects(min_row_id, first_end)) previous_end = first_end for _ in range(1, ranges): start, end = PAIR.unpack_from(data, offset) @@ -267,7 +458,7 @@ def index_file_name(manifest): return next((name for name in manifest.extra_files or [] if name.endswith(SUFFIX)), None) -def read_index(file_io, manifest_path, manifest, query, settings): +def read_index(file_io, manifest_path, manifest, query, settings, partition_filter=None, partition_fields=None): name = index_file_name(manifest) if name is None: return None @@ -281,7 +472,7 @@ def read_index(file_io, manifest_path, manifest, query, settings): break data.extend(chunk) _require(len(data) <= settings.max_bytes) - return select(data, manifest, query, settings) + return select(data, manifest, query, settings, partition_filter, partition_fields) except _PROPAGATED_ERRORS: raise except Exception as error: diff --git a/paimon-python/pypaimon/read/scanner/file_scanner.py b/paimon-python/pypaimon/read/scanner/file_scanner.py index 477d0f2c0772..134a1e1c7a20 100755 --- a/paimon-python/pypaimon/read/scanner/file_scanner.py +++ b/paimon-python/pypaimon/read/scanner/file_scanner.py @@ -598,6 +598,7 @@ def read_manifest_entries(self, manifest_files: List[ManifestFileMeta], early_record_filter=early_row_filter, partition_filter=partition_filter, row_ranges=row_ranges, + index_partition_filter=self.partition_key_predicate, ) def _build_early_bucket_filter(self): diff --git a/paimon-python/pypaimon/tests/manifest/manifest_block_index_test.py b/paimon-python/pypaimon/tests/manifest/manifest_block_index_test.py new file mode 100644 index 000000000000..03bd38108b79 --- /dev/null +++ b/paimon-python/pypaimon/tests/manifest/manifest_block_index_test.py @@ -0,0 +1,218 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import base64 +import hashlib +import struct +import random +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +import pytest + +from pypaimon.manifest.row_id_index import Builder, Settings, select +from pypaimon.schema.data_types import AtomicType, DataField +from pypaimon.table.row.generic_row import GenericRow, GenericRowSerializer +from pypaimon.tests.manifest.row_id_index_test import avro_header, golden, golden_meta +from pypaimon.utils.range import Range + +FIELDS = [DataField(0, 'p', AtomicType('INT')), DataField(1, 'q', AtomicType('STRING'))] + + +def partition(p, q): + return GenericRowSerializer.to_bytes(GenericRow([p, q], FIELDS)) + + +def part(p): + return SimpleNamespace(test=lambda row: row.values[0] == p) + + +def fixture(key): + path = Path(__file__).resolve().parents[4] / 'paimon-core/src/test/resources/manifest-row-id-index-v2.txt' + value = next(line.split('=', 1)[1] for line in path.read_text().splitlines() if line.startswith(key + '=')) + return base64.b64decode(value) + + +def meta(name, size, count): + return SimpleNamespace(file_name=name, file_size=size, num_added_files=count, num_deleted_files=0) + + +def test_partition_dictionary_golden_tuples_nulls_and_derived_ordinals(): + a, b, header = partition(7, 'left'), partition(9, None), avro_header() + assert a == fixture('partitionA') + assert b == fixture('partitionB') + builder = Builder(Settings(), header) + values = [(0, 100, [(0, 10, a), (5, 5, a), (20, 5, b)]), + (100, 200, [((1 << 32) - 2, 5, b), (8254058425445, 1, a)]), + (300, 100, [(20, 5, a), ((1 << 63) - 1, 1, b)])] + for offset, length, entries in values: + builder.begin_block(len(header) + offset, length, len(entries)) + for first, count, p in entries: + builder.add(first, count, p) + builder.end_block() + data = builder.serialize('manifest-golden', len(header) + 400, 7) + assert data == fixture('indexJointV3') + predicate = part(7) + with patch.object(predicate, 'test', wraps=predicate.test) as evaluated: + selected = select(data, golden_meta(), [Range(20, 20)], Settings(), predicate, FIELDS) + assert evaluated.call_count == 2 + assert [b.first_record for b in selected.blocks] == [0, 5] + nulls = SimpleNamespace(test=lambda row: row.values[1] is None) + assert len(select(data, golden_meta(), None, Settings(), nulls, FIELDS).blocks) == 3 + assert not select(data, golden_meta(), None, Settings(), part(99), FIELDS).blocks + assert len(select(golden(), golden_meta(), None, Settings(), part(99), FIELDS).blocks) == 3 + + +@pytest.mark.parametrize('settings', [Settings(max_partitions=1), Settings(max_partition_bytes=32)]) +def test_unavailable_dimensions_are_independent_and_dictionary_misses_keep_unknown_blocks(settings): + header = avro_header() + builder = Builder(settings, header) + for i, (first, p) in enumerate([(None, partition(7, 'left')), (200, partition(9, None)), + (300, partition(7, 'left'))]): + builder.begin_block(len(header) + 100 * i, 100, 1) + builder.add(first, 10, p) + builder.end_block() + data = builder.serialize('m', len(header) + 300, 3) + metadata = meta('m', len(header) + 300, 3) + assert [b.first_record for b in select(data, metadata, None, settings, part(9), FIELDS).blocks] == [1] + selected = select(data, metadata, [Range(999, 999)], settings, part(7), FIELDS) + assert [b.first_record for b in selected.blocks] == [0] + selected = select(data, metadata, [Range(200, 200)], settings, part(9), FIELDS) + assert [b.first_record for b in selected.blocks] == [1] + + +@pytest.mark.parametrize('unknown', [False, True]) +def test_coarse_row_ranges_keep_extending_bounds_and_detect_late_unknowns(unknown): + settings, header = Settings(max_ranges=1), avro_header() + builder = Builder(settings, header) + builder.begin_block(len(header), 100, 4) + for first, count in [(100, 10), (300, 10), (10, 10), (None if unknown else (1 << 63) - 1, 1)]: + builder.add(first, count, partition(7, 'left')) + builder.end_block() + data = builder.serialize('m', len(header) + 100, 4) + metadata = meta('m', len(header) + 100, 4) + for point in [10, 100, 200, (1 << 63) - 1]: + assert len(select(data, metadata, [Range(point, point)], settings).blocks) == 1 + assert len(select(data, metadata, [Range(0, 0)], settings).blocks) == int(unknown) + assert not select(data, metadata, None, settings, part(9), FIELDS).blocks + + +def positions(data): + offset = 64 + struct.unpack_from('>I', data, 60)[0] + count, = struct.unpack_from('>I', data, offset) + offset += 4 + for _ in range(count): + length, = struct.unpack_from('>I', data, offset) + offset += 4 + length + count, = struct.unpack_from('>I', data, offset) + offset += 4 + result = [] + for _ in range(count): + block = offset + p = offset + 24 + r = p + 5 + struct.unpack_from('>I', data, p + 1)[0] + offset = r + 5 + struct.unpack_from('>I', data, r + 1)[0] + result.append((block, p, r)) + return result + + +def checksum(data): + data[-32:] = hashlib.sha256(data[:-32]).digest() + return data + + +def test_unsigned_unknown_encodings_skip_only_one_payload_and_validate_lengths(): + good = fixture('indexJointV3') + block, p, r = positions(good)[0] + data = bytearray(good) + data[p] = 200 + selected = select(checksum(data), golden_meta(), [Range(0, 0)], Settings(), part(99), FIELDS) + assert [b.first_record for b in selected.blocks] == [0] + data = bytearray(good) + data[r] = 201 + selected = select(checksum(data), golden_meta(), [Range(16, 16)], Settings(), part(7), FIELDS) + assert [b.first_record for b in selected.blocks] == [0] + for position in (p, r): + data = bytearray(good) + data[position] = 0 + with pytest.raises(ValueError): + select(checksum(data), golden_meta(), None, Settings()) + struct.pack_into('>i', data, position + 1, -1) + with pytest.raises(ValueError): + select(checksum(data), golden_meta(), None, Settings()) + data = bytearray(good) + struct.pack_into('>q', data, block + 16, 2) + with pytest.raises(ValueError): + select(checksum(data), golden_meta(), None, Settings()) + data = bytearray(good) + struct.pack_into('>i', data, p + 9, 999) # out-of-dictionary ID + with pytest.raises(ValueError): + select(checksum(data), golden_meta(), None, Settings()) + data = bytearray(good) + struct.pack_into('>q', data, r + 9 + 16, 9) # overlap first interval, even in a rejected block + with pytest.raises(ValueError): + select(checksum(data), golden_meta(), [Range(999, 999)], Settings(), part(99), FIELDS) + + +def test_optional_payload_exhaustion_never_truncates_the_block_directory(): + settings, header = Settings(max_bytes=280), avro_header() + builder = Builder(settings, header) + for i in range(3): + builder.begin_block(len(header) + 100 * i, 100, 1) + builder.add(100 * i, 10, partition(7, 'left')) + builder.end_block() + data = builder.serialize('m', len(header) + 300, 3) + assert len(data) <= 280 + selected = select(data, meta('m', len(header) + 300, 3), [Range(999, 999)], settings, part(99), FIELDS) + assert [b.first_record for b in selected.blocks] == [0, 1, 2] + assert builder.serialize('m', len(header) + 300, 3) == data + builder.begin_block(len(header) + 300, 100, 1) + builder.add(300, 1, partition(7, 'left')) + builder.end_block() + assert builder.serialize('m', len(header) + 400, 4) is None + + +def test_randomized_budget_degradation_has_no_false_negatives(): + rng = random.Random(9743) + header = avro_header() + for _ in range(60): + settings = Settings(max_ranges=rng.choice([1, 4, 100]), + max_partitions=rng.choice([1, 3, 20]), + max_partition_bytes=rng.choice([32, 128, 1024]), + max_bytes=rng.choice([512, 1024, 8192])) + builder = Builder(settings, header) + blocks = [] + for i in range(5): + values = [(rng.choice([None, rng.randrange(100)]), rng.randrange(1, 10), rng.randrange(5)) + for _ in range(6)] + blocks.append(values) + builder.begin_block(len(header) + 100 * i, 100, len(values)) + for first, count, p in values: + builder.add(first, count, partition(p, None)) + builder.end_block() + data = builder.serialize('m', len(header) + 500, 30) + assert data is not None and len(data) <= settings.max_bytes + metadata = meta('m', len(header) + 500, 30) + for point in range(0, 110, 11): + for p in range(5): + selected = select(data, metadata, [Range(point, point)], settings, part(p), FIELDS) + ordinals = {b.first_record for b in selected.blocks} + for i, values in enumerate(blocks): + if (any(row_p == p for _, _, row_p in values) + and any(first is None or first <= point < first + count for first, count, _ in values)): + assert i * 6 in ordinals diff --git a/paimon-python/pypaimon/tests/manifest/row_id_index_test.py b/paimon-python/pypaimon/tests/manifest/row_id_index_test.py index 3bf398a471c5..482f4aeedec3 100644 --- a/paimon-python/pypaimon/tests/manifest/row_id_index_test.py +++ b/paimon-python/pypaimon/tests/manifest/row_id_index_test.py @@ -39,9 +39,12 @@ ) from pypaimon.manifest.schema.manifest_entry import ManifestEntry from pypaimon.manifest.manifest_list_manager import ManifestListManager +from pypaimon.manifest.manifest_file_manager import ManifestFileManager from pypaimon.manifest.schema.manifest_file_meta import MANIFEST_FILE_META_SCHEMA from pypaimon.read.scanner.file_scanner import FileScanner from pypaimon.tests.manifest import manifest_entry_identifier_test as existing +from pypaimon.schema.schema import Schema +from pypaimon.table.row.generic_row import GenericRow from pypaimon.utils.range import Range @@ -49,7 +52,7 @@ def fixture(): path = (Path(__file__).resolve().parents[4] / 'paimon-core/src/test/resources' / 'manifest-row-id-index-v2.txt') return dict(line.split('=', 1) for line in path.read_text().splitlines() - if line.startswith(('index=', 'avroHeader='))) + if line.startswith(('index=', 'indexV3=', 'avroHeader='))) def golden(): @@ -233,7 +236,8 @@ def test_cross_language_and_block_ordinals(self): for first, count in values: b.add(first, count) b.end_block() - self.assertEqual(b.serialize(meta.file_name, meta.file_size, 7), golden()) + self.assertEqual(b.serialize(meta.file_name, meta.file_size, 7), + base64.b64decode(fixture()['indexV3'])) def test_minmax_skips_exact_checks_and_one_interval_is_already_exact(self): header = avro_header() @@ -274,7 +278,11 @@ def test_coverage_and_budgets(self): b = Builder(Settings(), header) b.begin_block(len(header), 100, 1) b.add(first, count) - self.assertIsNone(b.serialize('m', 1, 1)) + b.end_block() + data = b.serialize('m', len(header) + 100, 1) + meta = SimpleNamespace(file_name='m', file_size=len(header) + 100, + num_added_files=1, num_deleted_files=0) + self.assertEqual(len(select(data, meta, [Range(100, 100)], Settings()).blocks), 1) b = Builder(Settings(), header) b.begin_block(len(header), 100, 2) b.add(0, MAX_ROW_ID) @@ -285,7 +293,11 @@ def test_coverage_and_budgets(self): b.begin_block(len(header), 100, 2) b.add(1, 1) b.add(1 << 32, 1) - self.assertIsNone(b.serialize('m', 1, 2)) + b.end_block() + data = b.serialize('m', len(header) + 100, 2) + meta = SimpleNamespace(file_name='m', file_size=len(header) + 100, + num_added_files=2, num_deleted_files=0) + self.assertEqual(len(select(data, meta, [Range(10, 10)], Settings()).blocks), 1) b = Builder(Settings(max_bytes=128), header) self.assertIsNone(b.serialize('m', 1, 1)) @@ -323,6 +335,51 @@ def write_meta(self, name, entries): manager = self.manifest_file_manager return manager.write(name, entries) + def test_partition_only_and_conjunctive_planning_keep_entry_and_delete_filters(self): + import pyarrow as pa + schema = Schema.from_pyarrow_schema( + pa.schema([('p', pa.int32()), ('q', pa.string()), ('value', pa.string())]), + partition_keys=['p', 'q'], + options={'manifest.row-id-index.write': 'true', 'manifest.row-id-index.read': 'true'}) + self.catalog.create_table('default.partition_block_index', schema, False) + self.table = self.catalog.get_table('default.partition_block_index') + self.manifest_file_manager = ManifestFileManager(self.table) + fields = self.table.partition_keys_fields + + def entry(name, first, p, kind=0): + return replace(self.entry(name, first, kind=kind), partition=GenericRow([p, None], fields)) + + # Build many blocks without any row tracking; partition-only planning must use the index. + entries = [entry('part-%d.parquet' % i, None, i // 1000) for i in range(4000)] + manifest = self.write_meta('partitioned', entries) + from pypaimon.common.predicate import Predicate + predicate = Predicate('equal', 0, 'p', [1]) + results = [] + for enabled in (False, True): + self.table.options.options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_READ, enabled) + scanner = FileScanner(self.table, lambda: ([manifest], None), partition_predicate=predicate) + with patch('pypaimon.manifest.manifest_file_manager.read_selected_bytes', + wraps=read_selected_bytes) as read_blocks: + actual = scanner.read_manifest_entries([manifest]) + results.append([e.file.file_name for e in actual]) + if enabled: + read_blocks.assert_called_once() + selected = read_blocks.call_args[0][2] + self.assertLess(sum(b.record_count for b in selected.blocks), 1500) + else: + read_blocks.assert_not_called() + self.assertEqual(results[0], results[1]) + self.assertEqual(len(results[1]), 1000) + + # A block can match the two dimensions through different entries; keep entry filtering. + mixed = self.write_meta('mixed', [entry('a.parquet', 100, 1), entry('b.parquet', 5, 2)]) + scanner = FileScanner(self.table, lambda: ([mixed], None), partition_predicate=predicate) + self.assertEqual(scanner.read_manifest_entries([mixed], row_ranges=[Range(5, 5)]), []) + # Both column groups and DELETE blocks must survive the same partition + row-id filter. + add = [entry('data.parquet', 100, 1), entry('data.blob', 100, 1)] + metas = [self.write_meta('adds', add), self.write_meta('deletes', [replace(e, kind=1) for e in add])] + self.assertEqual(scanner.read_manifest_entries(metas, row_ranges=[Range(105, 105)]), []) + def test_explicit_reference_and_null_does_not_probe(self): manager = self.manifest_file_manager written = self.write_meta('explicit', [self.entry('data.parquet', 100)]) @@ -355,6 +412,7 @@ def test_explicit_reference_and_null_does_not_probe(self): def test_manifest_list_index_reference_compatibility(self): indexed = self.write_meta('indexed', [self.entry('data.parquet', 100)]) indexed = replace(indexed, extra_files=['other-index'] + indexed.extra_files) + self.table.options.options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_WRITE, False) unindexed = self.write_meta('legacy-entry', [self.entry('old.parquet', None)]) self.assertIsNone(index_file_name(unindexed)) lists = ManifestListManager(self.table) @@ -575,8 +633,12 @@ def fail(path): manager.write('failed', entries[:1]) self.assertFalse(Path(manager.manifest_path, 'failed').exists()) self.assertFalse(Path(manager.manifest_path, 'failed' + SUFFIX).exists()) - manager.write('unknown', [self.entry('legacy', None)]) - self.assertFalse(Path(manager.manifest_path, 'unknown' + SUFFIX).exists()) + unknown = manager.write('unknown', [self.entry('legacy', None)]) + self.assertIsNotNone(index_file_name(unknown)) + data = Path(manager.manifest_path, index_file_name(unknown)).read_bytes() + self.assertEqual(len(select(data, unknown, [Range(100, 100)], Settings()).blocks), 1) self.table.options.options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_MAX_RANGES, 1) - manager.write('huge', [self.entry('one', 0, 10), self.entry('two', 100, 10)]) - self.assertFalse(Path(manager.manifest_path, 'huge' + SUFFIX).exists()) + huge = manager.write('huge', [self.entry('one', 0, 10), self.entry('two', 100, 10)]) + self.assertIsNotNone(index_file_name(huge)) + data = Path(manager.manifest_path, index_file_name(huge)).read_bytes() + self.assertEqual(len(select(data, huge, [Range(50, 50)], Settings()).blocks), 1) From 7e28f900b2a7155cdeea53f8b033fb1e46c2784b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Mon, 14 Sep 2026 11:11:47 +0800 Subject: [PATCH 06/23] [core] Add nullable bucket payloads to manifest block indexes Record complete bucket and total-bucket pairs per block, use existing bucket filters for point lookups, and retain independent partition and row-id coverage. Preserve v2/v3 reads and treat missing, invalid or over-budget bucket coverage conservatively. --- docs/docs/concepts/spec/manifest.md | 36 +++- .../java/org/apache/paimon/CoreOptions.java | 11 +- .../apache/paimon/manifest/BucketFilter.java | 16 ++ .../apache/paimon/manifest/ManifestFile.java | 13 +- .../paimon/manifest/ManifestRowIdIndex.java | 151 +++++++++++++-- .../manifest/ProjectedManifestEntry.java | 9 + .../operation/AbstractFileStoreScan.java | 5 +- .../manifest/ManifestBlockIndexTest.java | 174 +++++++++++++++++- .../paimon/manifest/ManifestFileTest.java | 52 ++++++ .../manifest/ManifestRowIdIndexTest.java | 2 +- .../resources/manifest-row-id-index-v2.txt | 3 + .../pypaimon/common/options/core_options.py | 6 + .../manifest/manifest_file_manager.py | 4 +- .../pypaimon/manifest/row_id_index.py | 91 +++++++-- .../manifest/manifest_block_index_test.py | 64 ++++++- .../tests/manifest/row_id_index_test.py | 37 +++- 16 files changed, 623 insertions(+), 51 deletions(-) diff --git a/docs/docs/concepts/spec/manifest.md b/docs/docs/concepts/spec/manifest.md index ca4522023155..745d6f85b46e 100644 --- a/docs/docs/concepts/spec/manifest.md +++ b/docs/docs/concepts/spec/manifest.md @@ -63,7 +63,7 @@ skip manifests before opening them. Each extra file belongs exclusively to one manifest. It is retained and cleaned up together with that manifest during snapshot, tag, or changelog deletion. -### Partition and Row-ID Block Index +### Partition, Row-ID and Bucket Block Index With `manifest.row-id-index.write` enabled, a manifest writer can create a binary `.row-id-index` sidecar. Its name is stored in the manifest-list @@ -71,20 +71,20 @@ record's `_EXTRA_FILES`; the existing Avro schemas and `_VERSION` identifiers ar Readers identify the block index by the historical `.row-id-index` suffix among these explicit references, not by probing for a derived file name. Other extra-file references are preserved. -With `manifest.row-id-index.read` enabled and a partition or row-ID filter available, readers can use +With `manifest.row-id-index.read` enabled and a partition, row-ID or bucket filter available, readers can use the sidecar to select complete Avro blocks before reading manifest entries. Both options default to `false`. Old manifests, null or empty extra-file lists, and lists containing only other extra-file types use the normal manifest read path. Missing, unsupported, corrupt, -or over-budget containers also fall back to that path. Each block's partition and row-ID +or over-budget containers also fall back to that path. Each block's partition, row-ID and bucket coverage is independently usable; an unavailable dimension cannot exclude a block. Cancellation and interruption errors propagate instead of triggering a full-manifest fallback. -Version 3 uses the following layout. Container integers and encoding-1 payload integers +Version 4 uses the following layout. Container integers and payload integers are fixed-width big endian. Encoding IDs are unsigned bytes with separate namespaces. ```text magic : 8 bytes // ASCII PAIMRIDX -formatVersion : int // 3 +formatVersion : int // 4 manifestNameHash : 32 bytes // SHA-256 of the UTF-8 basename manifestLength : long manifestEntryCount : long // ADD + DELETE @@ -105,6 +105,8 @@ blocks[] // original physical order rowIdEncoding : byte rowIdPayloadLength : int rowIdPayload : bytes + bucketPayloadLength : int // -1 means null (unavailable) + bucketPayload : bytes // omitted when null checksum : 32 bytes // SHA-256 of all preceding bytes ``` @@ -126,6 +128,20 @@ the container. Byte spans must cover the entire original manifest after its head record counts must sum to the manifest entry count. Readers continue validating blocks and known payloads even when a predicate has already rejected a block. +The nullable bucket payload contains a positive `pairCount: int` followed by that many +`(bucket: int, totalBuckets: int)` pairs. Pairs are sorted by bucket, then totalBuckets, +and deduplicated. They preserve bucket-count changes between writes; the bucket number +alone is not sufficient for point lookup after rescaling. A valid pair satisfies +`0 <= bucket < totalBuckets`. Missing, invalid, negative/synthetic or over-budget bucket +metadata makes that block's bucket payload null. Partition and row-ID coverage remain +independently usable; no mutual-exclusion restriction is imposed. + +Readers test bucket-only queries using the existing bucket-selection logic, including +the total-bucket count. Java uses conservative partition-independent bounds for +`ManifestBucketFilter`; arbitrary partition-dependent callbacks remain at the entry +filter stage. A null bucket payload cannot exclude a block. Malformed payload lengths, +pair counts, ordering or values invalidate the container rather than excluding a block. + All entries contribute, including ADD, DELETE and every file format/column group. Row-ID ranges are never expanded into individual values. If an exact union exceeds its range budget, it becomes the inclusive `[min,max]` envelope with encoding 1. Processing @@ -138,9 +154,10 @@ excludes a block with unavailable partition coverage. Later blocks can still use `manifest.row-id-index.max-ranges` bounds stored intervals (default 131072). `manifest.index.max-partitions` bounds dictionary entries (default 65536), and `manifest.index.max-partition-bytes` bounds dictionary bytes including length fields -(default 1048576). `manifest.row-id-index.max-bytes` bounds the whole serialized container +(default 1048576). `manifest.index.max-bucket-pairs` bounds distinct bucket/count pairs +per block (default 4096). `manifest.row-id-index.max-bytes` bounds the whole serialized container (default 8388608); the Avro header is also capped at 1 MiB and the directory at 131072 blocks. -Writers discard optional row-ID payloads, then partition payloads/dictionary if necessary, +Writers discard optional row-ID payloads, bucket payloads, then partition payloads/dictionary if necessary, to fit the complete directory. If the directory itself cannot fit, no sidecar is published. No emitted sidecar omits block descriptors. These are encoded-size bounds; construction also incurs bounded object/buffer overhead. Query concurrency multiplies per-reader costs. @@ -153,8 +170,9 @@ the first/last interval before testing the individual intervals. Readers still consume and validate the whole bounded sidecar. A partition-only query therefore reads row-ID payload bytes too; payload lengths save decoding work for unknown encodings, not storage I/O. Selected compressed blocks are read by byte range with adjacent -spans coalesced. Version-2 row-ID-only sidecars remain readable with partition coverage -treated as unavailable; older readers safely fall back on version 3. Existing immutable +spans coalesced. Version-2 row-ID-only sidecars and version-3 partition/row-ID sidecars +remain readable with missing coverage treated as unavailable; older readers safely fall +back on version 4. Existing immutable manifests are not backfilled by enabling the write option. Selected blocks still pass through entry filtering and ADD/DELETE reconciliation. Snapshot, diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java index cc8a95315bf1..b539e3e478aa 100644 --- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java @@ -515,14 +515,14 @@ public InlineElement getDescription() { .booleanType() .defaultValue(false) .withDescription( - "Write block indexes with independent partition and row-id coverage for newly created manifests."); + "Write block indexes with independent partition, row-id and bucket coverage for newly created manifests."); public static final ConfigOption MANIFEST_ROW_ID_INDEX_READ = key("manifest.row-id-index.read") .booleanType() .defaultValue(false) .withDescription( - "Read optional manifest block indexes for partition or row-id filters after coarse pruning. Missing or invalid indexes fall back to manifest reads."); + "Read optional manifest block indexes for partition, row-id or bucket filters after coarse pruning. Missing or invalid indexes fall back to manifest reads."); public static final ConfigOption MANIFEST_ROW_ID_INDEX_MAX_RANGES = key("manifest.row-id-index.max-ranges") @@ -538,6 +538,13 @@ public InlineElement getDescription() { .withDescription( "Maximum partition dictionary entries per manifest block index. Further unknown partitions disable partition coverage only for their blocks."); + public static final ConfigOption MANIFEST_INDEX_MAX_BUCKET_PAIRS = + key("manifest.index.max-bucket-pairs") + .intType() + .defaultValue(4096) + .withDescription( + "Maximum distinct bucket and total-bucket pairs per block. Exceeding this budget disables bucket coverage for the block, preserving other index payloads."); + public static final ConfigOption MANIFEST_INDEX_MAX_PARTITION_BYTES = key("manifest.index.max-partition-bytes") .intType() diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/BucketFilter.java b/paimon-core/src/main/java/org/apache/paimon/manifest/BucketFilter.java index 536be456eef0..3e564a60f9e1 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/BucketFilter.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/BucketFilter.java @@ -78,6 +78,22 @@ public boolean test(BinaryRow partition, int bucket, int totalBucket) { || totalAwareBucketFilter.test(partition, bucket, totalBucket); } + /** Conservatively checks an indexed pair without inventing a partition for custom filters. */ + public boolean mayContain(int bucket, int totalBuckets) { + if (onlyReadRealBuckets && bucket < 0) { + return false; + } + if (specifiedBucket != null && bucket != specifiedBucket) { + return false; + } + if (bucketFilter != null && !bucketFilter.test(bucket)) { + return false; + } + return !(totalAwareBucketFilter instanceof ManifestBucketFilter) + || ((ManifestBucketFilter) totalAwareBucketFilter) + .mayContain(bucket, bucket, totalBuckets); + } + /** Conservatively tests whether a manifest's bucket metadata can contain a matching entry. */ public boolean mayContain(ManifestFileMeta manifest) { Integer minBucket = manifest.minBucket(); diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java index 69b9770266a7..a1c4774ba886 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java @@ -395,7 +395,17 @@ public ManifestRowIdIndex.Selection selectBlocks( ManifestFileMeta manifest, @Nullable RowRangeIndex query, @Nullable PartitionPredicate partitionFilter) { - return !rowIdIndexSettings.read || (query == null && partitionFilter == null) + return selectBlocks(manifest, query, partitionFilter, null); + } + + @Nullable + public ManifestRowIdIndex.Selection selectBlocks( + ManifestFileMeta manifest, + @Nullable RowRangeIndex query, + @Nullable PartitionPredicate partitionFilter, + @Nullable BucketFilter bucketFilter) { + return !rowIdIndexSettings.read + || (query == null && partitionFilter == null && bucketFilter == null) ? null : ManifestRowIdIndex.read( fileIO, @@ -404,6 +414,7 @@ public ManifestRowIdIndex.Selection selectBlocks( query, partitionFilter, partitionType, + bucketFilter, rowIdIndexSettings); } diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestRowIdIndex.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestRowIdIndex.java index f90bd1b2db26..192da1bca026 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestRowIdIndex.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestRowIdIndex.java @@ -63,14 +63,15 @@ import static org.apache.paimon.utils.Preconditions.checkArgument; -/** Independently usable partition and row-id coverage for each physical manifest block. */ +/** Independently usable partition, row-id and bucket coverage for each manifest block. */ public final class ManifestRowIdIndex { public static final String SUFFIX = ".row-id-index"; private static final Logger LOG = LoggerFactory.getLogger(ManifestRowIdIndex.class); private static final long MAGIC = 0x5041494d52494458L; private static final int HEADER_BYTES = 60; private static final int LEGACY_HEADER_BYTES = 68; - private static final int BLOCK_BYTES = 34; + private static final int BLOCK_BYTES = 38; + private static final int V3_BLOCK_BYTES = 34; private static final int MAX_BLOCKS = 131072; private static final byte[] EMPTY = new byte[0]; private static final int DIGEST_BYTES = 32; @@ -103,6 +104,7 @@ public static final class Settings { public final int maxBytes; public final int maxPartitions; public final int maxPartitionBytes; + public final int maxBucketPairs; public Settings(Options options) { write = options.get(CoreOptions.MANIFEST_ROW_ID_INDEX_WRITE); @@ -111,6 +113,10 @@ public Settings(Options options) { maxBytes = options.get(CoreOptions.MANIFEST_ROW_ID_INDEX_MAX_BYTES); maxPartitions = options.get(CoreOptions.MANIFEST_INDEX_MAX_PARTITIONS); maxPartitionBytes = options.get(CoreOptions.MANIFEST_INDEX_MAX_PARTITION_BYTES); + maxBucketPairs = options.get(CoreOptions.MANIFEST_INDEX_MAX_BUCKET_PAIRS); + checkArgument( + maxBucketPairs > 0 && maxBucketPairs <= 1048576, + "Invalid manifest.index.max-bucket-pairs"); checkArgument( maxPartitions > 0 && maxPartitions <= 1048576, "Invalid manifest.index.max-partitions"); @@ -165,6 +171,7 @@ public static final class Builder { private final TreeMap ranges = new TreeMap<>(); private final Map dictionary = new LinkedHashMap<>(); private final TreeSet partitionIds = new TreeSet<>(); + private final TreeSet bucketPairs = new TreeSet<>(); private final List blocks = new ArrayList<>(); private boolean complete; private long nextOffset; @@ -173,6 +180,7 @@ public static final class Builder { private long entriesInBlock; private boolean rowAvailable; private boolean partitionAvailable; + private boolean bucketAvailable; private boolean coarse; private long min; private long max; @@ -217,11 +225,13 @@ public void beginBlock(long offset, long length, long records) throws IOExceptio entriesInBlock = 0; rowAvailable = true; partitionAvailable = true; + bucketAvailable = true; coarse = false; min = Long.MAX_VALUE; max = -1; ranges.clear(); partitionIds.clear(); + bucketPairs.clear(); } public void add(@Nullable Long first, long count) { @@ -229,6 +239,15 @@ public void add(@Nullable Long first, long count) { } public void add(@Nullable Long first, long count, @Nullable byte[] partition) { + add(first, count, partition, null, null); + } + + public void add( + @Nullable Long first, + long count, + @Nullable byte[] partition, + @Nullable Integer bucket, + @Nullable Integer totalBuckets) { if (!complete) { return; } @@ -237,6 +256,7 @@ public void add(@Nullable Long first, long count, @Nullable byte[] partition) { } entriesInBlock++; addPartition(partition); + addBucket(bucket, totalBuckets); if (!rowAvailable) { return; } @@ -273,6 +293,30 @@ public void add(@Nullable Long first, long count, @Nullable byte[] partition) { } } + private void addBucket(@Nullable Integer bucket, @Nullable Integer totalBuckets) { + if (!bucketAvailable) { + return; + } + if (bucket == null + || totalBuckets == null + || bucket < 0 + || totalBuckets <= 0 + || bucket >= totalBuckets) { + bucketAvailable = false; + bucketPairs.clear(); + return; + } + long pair = ((long) bucket << 32) | totalBuckets; + if (!bucketPairs.contains(pair) + && bucketPairs.size() + >= Math.min(settings.maxBucketPairs, settings.maxBytes / 8)) { + bucketAvailable = false; + bucketPairs.clear(); + } else { + bucketPairs.add(pair); + } + } + private void addPartition(@Nullable byte[] bytes) { if (!partitionAvailable) { return; @@ -336,12 +380,24 @@ public void endBlock() throws IOException { partitionPayload = out.array(); optionalBytes += partitionPayload.length; } - blocks.add(new IndexedBlock(current, partitionPayload, rowPayload)); + byte[] bucketPayload = EMPTY; + if (bucketAvailable + && 4L + 8L * bucketPairs.size() <= settings.maxBytes - optionalBytes) { + ByteBuffer out = ByteBuffer.allocate(4 + 8 * bucketPairs.size()); + out.putInt(bucketPairs.size()); + for (long pair : bucketPairs) { + out.putInt((int) (pair >>> 32)).putInt((int) pair); + } + bucketPayload = out.array(); + optionalBytes += bucketPayload.length; + } + blocks.add(new IndexedBlock(current, partitionPayload, rowPayload, bucketPayload)); savedRanges += rowCount; nextOffset = Math.addExact(current.offset, current.length); nextRecord = Math.addExact(current.firstRecord, current.recordCount); ranges.clear(); partitionIds.clear(); + bucketPairs.clear(); current = null; } @@ -368,6 +424,14 @@ public byte[] serialize(String name, long fileSize, long entryCount) throws IOEx optionalBytes -= block.rowIds.length; block.rowIds = EMPTY; } + for (IndexedBlock block : blocks) { + if (size <= settings.maxBytes) { + break; + } + size -= block.buckets.length; + optionalBytes -= block.buckets.length; + block.buckets = EMPTY; + } if (size > settings.maxBytes) { size -= dictionaryBytes; dictionaryBytes = 0; @@ -382,7 +446,7 @@ public byte[] serialize(String name, long fileSize, long entryCount) throws IOEx ByteArrayOutputStream buffer = new ByteArrayOutputStream((int) size); DataOutputStream out = new DataOutputStream(buffer); out.writeLong(MAGIC); - out.writeInt(3); + out.writeInt(4); out.write(digest(name.getBytes(StandardCharsets.UTF_8))); out.writeLong(fileSize); out.writeLong(entryCount); @@ -400,6 +464,8 @@ public byte[] serialize(String name, long fileSize, long entryCount) throws IOEx out.writeLong(block.block.recordCount); writePayload(out, block.partitions); writePayload(out, block.rowIds); + out.writeInt(block.buckets.length == 0 ? -1 : block.buckets.length); + out.write(block.buckets); } out.write(digest(buffer.toByteArray())); return buffer.toByteArray(); @@ -416,11 +482,13 @@ private static final class IndexedBlock { private final Block block; private byte[] partitions; private byte[] rowIds; + private byte[] buckets; - private IndexedBlock(Block block, byte[] partitions, byte[] rowIds) { + private IndexedBlock(Block block, byte[] partitions, byte[] rowIds, byte[] buckets) { this.block = block; this.partitions = partitions; this.rowIds = rowIds; + this.buckets = buckets; } } @@ -431,7 +499,7 @@ public static byte[] build(FileIO io, Path path, long size, long records, Settin try (ManifestAvroReader reader = new ManifestAvroReader(io.newInputStream(path))) { Builder builder = new Builder(settings, reader.headerBytes()); ProjectedManifestEntry.Projection projection = - ProjectedManifestEntry.ROW_RANGE_PROJECTION; + ProjectedManifestEntry.BLOCK_INDEX_PROJECTION; ProjectedManifestEntry entry = projection.createEntry(); while (builder.complete() && reader.hasNext()) { ManifestAvroReader.RawBlock block = reader.next(); @@ -442,7 +510,9 @@ public static byte[] build(FileIO io, Path path, long size, long records, Settin builder.add( entry.file().firstRowId(), entry.file().rowCount(), - entry.partitionBytes()); + entry.partitionBytes(), + entry.bucket(), + entry.totalBuckets()); } builder.endBlock(); } @@ -459,13 +529,25 @@ public static Selection select( return select(data, manifest, query, null, null, settings); } - /** Validates framing and known payloads before applying the two independent dimensions. */ + /** Validates framing and known payloads before applying independently available dimensions. */ + public static Selection select( + byte[] data, + ManifestFileMeta manifest, + @Nullable RowRangeIndex query, + @Nullable PartitionPredicate partitionFilter, + @Nullable RowType partitionType, + Settings settings) + throws IOException { + return select(data, manifest, query, partitionFilter, partitionType, null, settings); + } + public static Selection select( byte[] data, ManifestFileMeta manifest, @Nullable RowRangeIndex query, @Nullable PartitionPredicate partitionFilter, @Nullable RowType partitionType, + @Nullable BucketFilter bucketFilter, Settings settings) throws IOException { require(data.length >= 128 && data.length <= settings.maxBytes); @@ -479,7 +561,10 @@ public static Selection select( digest(Arrays.copyOf(data, limit)), Arrays.copyOfRange(data, limit, data.length))); ByteBuffer in = ByteBuffer.wrap(data, 0, limit).slice(); - require(in.getLong() == MAGIC && in.getInt() == 3); + require(in.getLong() == MAGIC); + int version = in.getInt(); + require(version == 3 || version == 4); + int blockBytes = version == 3 ? V3_BLOCK_BYTES : BLOCK_BYTES; byte[] hash = new byte[DIGEST_BYTES]; in.get(hash); require( @@ -530,13 +615,13 @@ public static Selection select( } require(in.remaining() >= 4); int count = in.getInt(); - require(count >= 0 && count <= MAX_BLOCKS && count <= in.remaining() / BLOCK_BYTES); + require(count >= 0 && count <= MAX_BLOCKS && count <= in.remaining() / blockBytes); long nextOffset = headerLength; long firstRecord = 0; int totalRanges = 0; List selected = new ArrayList<>(); for (int i = 0; i < count; i++) { - require(in.remaining() >= BLOCK_BYTES); + require(in.remaining() >= blockBytes); long offset = in.getLong(); long length = in.getLong(); long records = in.getLong(); @@ -590,7 +675,33 @@ public static Selection select( } } } - if (partitionHit && rowHit) { + boolean bucketHit = true; + if (version >= 4) { + require(in.remaining() >= 4); + int bucketLength = in.getInt(); + if (bucketLength != -1) { + require(bucketLength >= 4 && bucketLength <= in.remaining()); + int pairs = in.getInt(); + require( + pairs > 0 + && pairs <= settings.maxBucketPairs + && bucketLength == 4L + 8L * pairs); + bucketHit = bucketFilter == null; + long previous = -1; + for (int j = 0; j < pairs; j++) { + int bucket = in.getInt(); + int totalBuckets = in.getInt(); + require(bucket >= 0 && totalBuckets > bucket); + long pair = ((long) bucket << 32) | totalBuckets; + require(pair > previous); + previous = pair; + if (!bucketHit) { + bucketHit = bucketFilter.mayContain(bucket, totalBuckets); + } + } + } + } + if (partitionHit && rowHit && bucketHit) { selected.add(new Block(offset, length, firstRecord, records)); } nextOffset = offset + length; @@ -720,6 +831,19 @@ public static Selection read( @Nullable PartitionPredicate partitionFilter, @Nullable RowType partitionType, Settings settings) { + return read(io, path, manifest, query, partitionFilter, partitionType, null, settings); + } + + @Nullable + public static Selection read( + FileIO io, + Path path, + ManifestFileMeta manifest, + @Nullable RowRangeIndex query, + @Nullable PartitionPredicate partitionFilter, + @Nullable RowType partitionType, + @Nullable BucketFilter bucketFilter, + Settings settings) { String indexFileName = fileName(manifest); if (indexFileName == null) { return null; @@ -742,7 +866,8 @@ public static Selection read( } data = out.toByteArray(); } - return select(data, manifest, query, partitionFilter, partitionType, settings); + return select( + data, manifest, query, partitionFilter, partitionType, bucketFilter, settings); } catch (CancellationException failure) { throw failure; } catch (IOException | RuntimeException failure) { diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ProjectedManifestEntry.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ProjectedManifestEntry.java index 77486d4cdd53..defd81bcd3e2 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ProjectedManifestEntry.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ProjectedManifestEntry.java @@ -27,6 +27,7 @@ import javax.annotation.Nullable; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -49,6 +50,7 @@ public final class ProjectedManifestEntry implements ManifestEntry { private static final Projection FULL_PROJECTION = Projection.create(MANIFEST_ROW_TYPE); public static final Projection DELETE_ENTRY_PROJECTION = createDeleteEntryProjection(); public static final Projection ROW_RANGE_PROJECTION = createRowRangeProjection(); + public static final Projection BLOCK_INDEX_PROJECTION = createBlockIndexProjection(); public static final Projection ENTRY_LAYOUT_PROJECTION = createEntryLayoutProjection(); private final Projection projection; @@ -133,6 +135,13 @@ private static Projection createRowRangeProjection() { DataFileMeta.FIRST_ROW_ID))))); } + private static Projection createBlockIndexProjection() { + List fields = new ArrayList<>(ROW_RANGE_PROJECTION.projectedType().getFields()); + fields.add(MANIFEST_ROW_TYPE.getField(ManifestEntry.BUCKET)); + fields.add(MANIFEST_ROW_TYPE.getField(ManifestEntry.TOTAL_BUCKETS)); + return Projection.create(new RowType(false, fields)); + } + private static Projection createEntryLayoutProjection() { RowType manifestType = MANIFEST_ROW_TYPE; return Projection.create( diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreScan.java b/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreScan.java index 4bf51db25bb6..b8989021678f 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreScan.java @@ -500,9 +500,10 @@ private List readManifest( @Nullable Filter additionalTFilter) { ManifestFile manifestFile = manifestFileFactory.create(); + BucketFilter bucketFilter = createBucketFilter(); ManifestRowIdIndex.Selection selected = manifestFile.selectBlocks( - manifest, rowRangeIndex, manifestsReader.partitionFilter()); + manifest, rowRangeIndex, manifestsReader.partitionFilter(), bucketFilter); if (selected != null && selected.blocks().isEmpty()) { return Collections.emptyList(); } @@ -518,7 +519,7 @@ private List readManifest( manifest.fileName(), manifest.fileSize(), manifestsReader.partitionFilter(), - createBucketFilter(), + bucketFilter, entryRowFilter.and(additionalFilter), entry -> (additionalTFilter == null || additionalTFilter.test(entry)) diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java index c9cf3664aa99..662e5f081cee 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java @@ -109,7 +109,7 @@ void jointGoldenPreservesTuplesNullsAndDerivedOrdinals() throws Exception { builder.add(Long.MAX_VALUE, 1, b); builder.endBlock(); byte[] data = builder.serialize("manifest-golden", header.length + 400, 7); - assertThat(data).isEqualTo(fixture("indexJointV3")); + assertThat(data).isEqualTo(fixture("indexJointV4")); ManifestFileMeta meta = meta("manifest-golden", header.length + 400, 7); PartitionPredicate filter = spy(part(7)); assertThat( @@ -217,7 +217,15 @@ private List positions(byte[] data) { in.get(); length = in.getInt(); in.position(in.position() + length); - result.add(new int[] {block, partition, row}); + int bucket = -1; + if (in.getInt(8) >= 4) { + bucket = in.position(); + length = in.getInt(); + if (length >= 0) { + in.position(in.position() + length); + } + } + result.add(new int[] {block, partition, row, bucket}); } return result; } @@ -287,6 +295,168 @@ badRange, meta, query(999), part(99), type, defaults)) .isInstanceOf(IOException.class); } + @Test + void bucketPayloadGoldenAndTotalBucketsArePreserved() throws Exception { + byte[] header = fixture("avroHeader"); + byte[] a = partition(7, "left"); + byte[] b = partition(9, null); + ManifestRowIdIndex.Builder builder = new ManifestRowIdIndex.Builder(defaults, header); + builder.beginBlock(header.length, 100, 3); + builder.add(0L, 10, a, 1, 4); + builder.add(5L, 5, a, 1, 4); + builder.add(20L, 5, b, 1, 8); + builder.endBlock(); + builder.beginBlock(header.length + 100, 200, 2); + builder.add((1L << 32) - 2, 5, b, 2, 4); + builder.add(8254058425445L, 1, a, 2, 8); + builder.endBlock(); + builder.beginBlock(header.length + 300, 100, 2); + builder.add(20L, 5, a, 0, 1); + builder.add(Long.MAX_VALUE, 1, b, 3, 4); + builder.endBlock(); + byte[] data = builder.serialize("manifest-golden", header.length + 400, 7); + assertThat(data).isEqualTo(fixture("indexBucketV4")); + ManifestFileMeta meta = meta("manifest-golden", header.length + 400, 7); + BucketFilter bucket = BucketFilter.create(false, 1, null, null); + assertThat( + ManifestRowIdIndex.select(data, meta, null, null, type, bucket, defaults) + .blocks()) + .extracting(block -> block.firstRecord) + .containsExactly(0L); + // Existing row-id and partition coverage remains independently usable. + assertThat( + ManifestRowIdIndex.select( + data, + meta, + query(0), + part(7), + type, + BucketFilter.create(false, 2, null, null), + defaults) + .blocks()) + .isEmpty(); + ManifestBucketFilter totalAware = + new ManifestBucketFilter() { + @Override + public boolean test(BinaryRow partition, Integer bucket, Integer total) { + throw new AssertionError("Block lookup must not invent a partition"); + } + + @Override + public boolean mayContain(int min, int max, int total) { + return min == 2 && max == 2 && total == 8; + } + }; + BucketFilter filter = BucketFilter.create(false, null, null, totalAware); + assertThat( + ManifestRowIdIndex.select(data, meta, null, null, type, filter, defaults) + .blocks()) + .extracting(block -> block.firstRecord) + .containsExactly(3L); + BucketFilter partitionAware = + BucketFilter.create( + false, + null, + null, + (partition, bucketId, total) -> { + throw new AssertionError("Needs actual entry partition"); + }); + assertThat( + ManifestRowIdIndex.select( + data, meta, null, null, type, partitionAware, defaults) + .blocks()) + .hasSize(3); + for (String old : new String[] {"index", "indexJointV3", "indexJointV4"}) { + assertThat( + ManifestRowIdIndex.select( + fixture(old), + meta, + null, + null, + type, + BucketFilter.create(false, 99, null, null), + defaults) + .blocks()) + .hasSize(3); + } + } + + @Test + void unknownInvalidOrOverBudgetBucketPayloadIsNull() throws Exception { + Options options = new Options(); + options.set(CoreOptions.MANIFEST_INDEX_MAX_BUCKET_PAIRS, 1); + ManifestRowIdIndex.Settings settings = new ManifestRowIdIndex.Settings(options); + byte[] header = fixture("avroHeader"); + for (Integer[] pair : + Arrays.asList( + new Integer[] {null, null}, + new Integer[] {-1, 4}, + new Integer[] {4, 4}, + new Integer[] {0, 0}, + new Integer[] {2, 8})) { + ManifestRowIdIndex.Builder builder = new ManifestRowIdIndex.Builder(settings, header); + builder.beginBlock(header.length, 100, 2); + builder.add(100L, 10, partition(7, "left"), 1, 4); + builder.add(200L, 10, partition(7, "left"), pair[0], pair[1]); + builder.endBlock(); + builder.beginBlock(header.length + 100, 100, 1); + builder.add(300L, 10, partition(7, "left"), 1, 4); + builder.endBlock(); + byte[] data = builder.serialize("m", header.length + 200, 3); + assertThat(ByteBuffer.wrap(data).getInt(positions(data).get(0)[3])).isEqualTo(-1); + ManifestFileMeta meta = meta("m", header.length + 200, 3); + assertThat( + ManifestRowIdIndex.select( + data, + meta, + null, + null, + type, + BucketFilter.create(false, 99, null, null), + settings) + .blocks()) + .extracting(block -> block.firstRecord) + .containsExactly(0L); + assertThat( + ManifestRowIdIndex.select( + data, + meta, + query(999), + null, + type, + BucketFilter.create(false, 99, null, null), + settings) + .blocks()) + .isEmpty(); + } + } + + @Test + void malformedBucketPayloadInvalidatesTheContainer() throws Exception { + byte[] good = fixture("indexBucketV4"); + int payload = positions(good).get(0)[3]; + ManifestFileMeta meta = meta("manifest-golden", fixture("avroHeader").length + 400, 7); + for (int[] mutation : + new int[][] { + {payload, -2}, + {payload, Integer.MAX_VALUE}, + {payload, 0}, + {payload + 4, 0}, + {payload + 8, -1}, + {payload + 12, 1}, + {payload + 16, 0} + }) { + byte[] bad = good.clone(); + ByteBuffer.wrap(bad).putInt(mutation[0], mutation[1]); + checksum(bad); + assertThatThrownBy( + () -> + ManifestRowIdIndex.select( + bad, meta, query(999), part(99), type, defaults)) + .isInstanceOf(IOException.class); + } + } + @Test void tightByteBudgetKeepsAllDescriptorsOrOmitsTheWholeFile() throws Exception { Options options = new Options(); diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java index d8aa610987d3..ba0d8d478cbf 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java @@ -1530,6 +1530,58 @@ public PositionOutputStream newOutputStream(Path path, boolean overwrite) } } + @Test + void testBucketOnlyPlanningAndRawRewriteUseNullableBucketPayload() throws Exception { + Options options = new Options(); + options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_WRITE, true); + options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_READ, true); + RecordingFileIO io = new RecordingFileIO(); + ManifestFile.Factory factory = + createManifestFileFactory(tempDir.toString(), Long.MAX_VALUE, options, io); + ManifestFile manifests = factory.create(); + List entries = new ArrayList<>(); + for (int i = 0; i < 4000; i++) { + ManifestEntry entry = gen.next(); + entries.add( + ManifestEntry.create( + FileKind.ADD, entry.partition(), i / 1000, 4, entry.file())); + } + ManifestFileMeta meta = manifests.write(entries).get(0); + AppendOnlyFileStoreScan scan = + new AppendOnlyFileStoreScan( + mock(ManifestsReader.class), + null, + null, + null, + mock(TableSchema.class), + factory, + 2, + false, + false, + false); + scan.withBucket(1); + io.reset(); + assertThat(scan.readManifest(meta)).containsExactlyElementsOf(entries.subList(1000, 2000)); + assertThat(io.bytes.get()).isLessThan(meta.fileSize()); + assertThat(io.seeks).isNotEmpty(); + ManifestAvroWriter writer = manifests.createAvroWriter(); + try (ManifestAvroReader reader = + manifests.scanAvroBlocks(meta.fileName(), meta.fileSize())) { + writer.writeEncodedManifest(reader, meta); + } + writer.close(); + assertThat(scan.readManifest(writer.result().get(0))) + .containsExactlyElementsOf(entries.subList(1000, 2000)); + + ManifestEntry added = entries.get(1000); + ManifestEntry deleted = + ManifestEntry.create(FileKind.DELETE, added.partition(), 1, 4, added.file()); + List changes = new ArrayList<>(); + changes.addAll(manifests.write(Collections.singletonList(added))); + changes.addAll(manifests.write(Collections.singletonList(deleted))); + assertThat(scan.readManifestEntries(changes, false)).isExhausted(); + } + @Test void testPartitionOnlyPlanningUsesBlocksWithoutRowIds() { Options options = new Options(); diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestRowIdIndexTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestRowIdIndexTest.java index 8b1810bfe0b0..aa064f91999d 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestRowIdIndexTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestRowIdIndexTest.java @@ -111,7 +111,7 @@ void crossLanguageFormatAndBlockOrdinals() throws Exception { builder.add(Long.MAX_VALUE, 1); builder.endBlock(); byte[] data = builder.serialize("manifest-golden", header.length + 400, 7); - assertThat(data).isEqualTo(Base64.getDecoder().decode(fixture().getProperty("indexV3"))); + assertThat(data).isEqualTo(Base64.getDecoder().decode(fixture().getProperty("indexV4"))); ManifestFileMeta meta = goldenMeta(); for (long point : new long[] { diff --git a/paimon-core/src/test/resources/manifest-row-id-index-v2.txt b/paimon-core/src/test/resources/manifest-row-id-index-v2.txt index 4ff6fc842c8d..b91bd5c2b0f5 100644 --- a/paimon-core/src/test/resources/manifest-row-id-index-v2.txt +++ b/paimon-core/src/test/resources/manifest-row-id-index-v2.txt @@ -21,3 +21,6 @@ partitionA=AAAAAgAAAAAAAAAABwAAAAAAAABsZWZ0AAAAhA== partitionB=AAAAAgACAAAAAAAACQAAAAAAAAAAAAAAAAAAAA== indexV3=UEFJTVJJRFgAAAADL0eWubkHdL6ioNdIPj3jGtp9TAk83wbtAsmc+xJVwM8AAAAAAAAByQAAAAAAAAAHAAAAOU9iagEEFGF2cm8uY29kZWMIbnVsbBZhdnJvLnNjaGVtYQwibG9uZyIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAADkAAAAAAAAAZAAAAAAAAAADAAAAAAABAAAAJAAAAAIAAAAAAAAAAAAAAAAAAAAJAAAAAAAAABQAAAAAAAAAGAAAAAAAAACdAAAAAAAAAMgAAAAAAAAAAgAAAAAAAQAAACQAAAACAAAAAP////4AAAABAAAAAgAAB4HMOGxlAAAHgcw4bGUAAAAAAAABZQAAAAAAAABkAAAAAAAAAAIAAAAAAAEAAAAkAAAAAgAAAAAAAAAUAAAAAAAAABh//////////3//////////ViRH8zjNd2Q9F4olffQ2ZQiExUDkv3wPwoBnsEGW3sQ= indexJointV3=UEFJTVJJRFgAAAADL0eWubkHdL6ioNdIPj3jGtp9TAk83wbtAsmc+xJVwM8AAAAAAAAByQAAAAAAAAAHAAAAOU9iagEEFGF2cm8uY29kZWMIbnVsbBZhdnJvLnNjaGVtYQwibG9uZyIAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAcAAAAAgAAAAAAAAAABwAAAAAAAABsZWZ0AAAAhAAAABwAAAACAAIAAAAAAAAJAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAA5AAAAAAAAAGQAAAAAAAAAAwEAAAAMAAAAAgAAAAAAAAABAQAAACQAAAACAAAAAAAAAAAAAAAAAAAACQAAAAAAAAAUAAAAAAAAABgAAAAAAAAAnQAAAAAAAADIAAAAAAAAAAIBAAAADAAAAAIAAAAAAAAAAQEAAAAkAAAAAgAAAAD////+AAAAAQAAAAIAAAeBzDhsZQAAB4HMOGxlAAAAAAAAAWUAAAAAAAAAZAAAAAAAAAACAQAAAAwAAAACAAAAAAAAAAEBAAAAJAAAAAIAAAAAAAAAFAAAAAAAAAAYf/////////9//////////4mo+QwqXuGgI0WNm1HdwCQA6dlu7UP8TigmfZSh14mZ +indexV4=UEFJTVJJRFgAAAAEL0eWubkHdL6ioNdIPj3jGtp9TAk83wbtAsmc+xJVwM8AAAAAAAAByQAAAAAAAAAHAAAAOU9iagEEFGF2cm8uY29kZWMIbnVsbBZhdnJvLnNjaGVtYQwibG9uZyIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAADkAAAAAAAAAZAAAAAAAAAADAAAAAAABAAAAJAAAAAIAAAAAAAAAAAAAAAAAAAAJAAAAAAAAABQAAAAAAAAAGP////8AAAAAAAAAnQAAAAAAAADIAAAAAAAAAAIAAAAAAAEAAAAkAAAAAgAAAAD////+AAAAAQAAAAIAAAeBzDhsZQAAB4HMOGxl/////wAAAAAAAAFlAAAAAAAAAGQAAAAAAAAAAgAAAAAAAQAAACQAAAACAAAAAAAAABQAAAAAAAAAGH//////////f///////////////yreFS/v5MsyAxSS6WSEBiJQTdm4As0DTJz5RJ/UsNkE= +indexJointV4=UEFJTVJJRFgAAAAEL0eWubkHdL6ioNdIPj3jGtp9TAk83wbtAsmc+xJVwM8AAAAAAAAByQAAAAAAAAAHAAAAOU9iagEEFGF2cm8uY29kZWMIbnVsbBZhdnJvLnNjaGVtYQwibG9uZyIAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAcAAAAAgAAAAAAAAAABwAAAAAAAABsZWZ0AAAAhAAAABwAAAACAAIAAAAAAAAJAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAA5AAAAAAAAAGQAAAAAAAAAAwEAAAAMAAAAAgAAAAAAAAABAQAAACQAAAACAAAAAAAAAAAAAAAAAAAACQAAAAAAAAAUAAAAAAAAABj/////AAAAAAAAAJ0AAAAAAAAAyAAAAAAAAAACAQAAAAwAAAACAAAAAAAAAAEBAAAAJAAAAAIAAAAA/////gAAAAEAAAACAAAHgcw4bGUAAAeBzDhsZf////8AAAAAAAABZQAAAAAAAABkAAAAAAAAAAIBAAAADAAAAAIAAAAAAAAAAQEAAAAkAAAAAgAAAAAAAAAUAAAAAAAAABh//////////3///////////////6c8cPEEmOjexjRnjlIO8fIsDVHyZ0EFCl2Yx26fiIzD +indexBucketV4=UEFJTVJJRFgAAAAEL0eWubkHdL6ioNdIPj3jGtp9TAk83wbtAsmc+xJVwM8AAAAAAAAByQAAAAAAAAAHAAAAOU9iagEEFGF2cm8uY29kZWMIbnVsbBZhdnJvLnNjaGVtYQwibG9uZyIAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAcAAAAAgAAAAAAAAAABwAAAAAAAABsZWZ0AAAAhAAAABwAAAACAAIAAAAAAAAJAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAA5AAAAAAAAAGQAAAAAAAAAAwEAAAAMAAAAAgAAAAAAAAABAQAAACQAAAACAAAAAAAAAAAAAAAAAAAACQAAAAAAAAAUAAAAAAAAABgAAAAUAAAAAgAAAAEAAAAEAAAAAQAAAAgAAAAAAAAAnQAAAAAAAADIAAAAAAAAAAIBAAAADAAAAAIAAAAAAAAAAQEAAAAkAAAAAgAAAAD////+AAAAAQAAAAIAAAeBzDhsZQAAB4HMOGxlAAAAFAAAAAIAAAACAAAABAAAAAIAAAAIAAAAAAAAAWUAAAAAAAAAZAAAAAAAAAACAQAAAAwAAAACAAAAAAAAAAEBAAAAJAAAAAIAAAAAAAAAFAAAAAAAAAAYf/////////9//////////wAAABQAAAACAAAAAAAAAAEAAAADAAAABEZWMYlUL4U9F6ENk1p6oZeCOokJCzCKEdTwv9geIID0 diff --git a/paimon-python/pypaimon/common/options/core_options.py b/paimon-python/pypaimon/common/options/core_options.py index d9a084a2ec21..d6aafc67be2d 100644 --- a/paimon-python/pypaimon/common/options/core_options.py +++ b/paimon-python/pypaimon/common/options/core_options.py @@ -327,6 +327,12 @@ class CoreOptions: .default_value(65536) ) + MANIFEST_INDEX_MAX_BUCKET_PAIRS: ConfigOption[int] = ( + ConfigOptions.key("manifest.index.max-bucket-pairs") + .int_type() + .default_value(4096) + ) + MANIFEST_INDEX_MAX_PARTITION_BYTES: ConfigOption[int] = ( ConfigOptions.key("manifest.index.max-partition-bytes") .int_type() diff --git a/paimon-python/pypaimon/manifest/manifest_file_manager.py b/paimon-python/pypaimon/manifest/manifest_file_manager.py index 11030bf83236..f79cce283abb 100644 --- a/paimon-python/pypaimon/manifest/manifest_file_manager.py +++ b/paimon-python/pypaimon/manifest/manifest_file_manager.py @@ -73,9 +73,9 @@ def read_entries_parallel(self, manifest_files: List[ManifestFileMeta], manifest def _process_single_manifest(manifest_file: ManifestFileMeta): path = f"{self.manifest_path}/{manifest_file.file_name}" selected = None - if settings.read and (query is not None or index_partition_filter is not None): + if settings.read and (query is not None or index_partition_filter is not None or early_entry_filter is not None): selected = read_index(self.file_io, path, manifest_file, query, settings, - index_partition_filter, self.partition_keys_fields) + index_partition_filter, self.partition_keys_fields, early_entry_filter) if selected is not None and not selected.blocks: return [] return self.read( diff --git a/paimon-python/pypaimon/manifest/row_id_index.py b/paimon-python/pypaimon/manifest/row_id_index.py index 47d319f1ba2d..7a884caca890 100644 --- a/paimon-python/pypaimon/manifest/row_id_index.py +++ b/paimon-python/pypaimon/manifest/row_id_index.py @@ -17,7 +17,7 @@ """Complete row-id interval unions with Avro block offsets and entry ordinals. -Version 3 shares a partition dictionary and independently framed block payloads. +Version 4 adds nullable bucket/total-bucket pairs to the version-3 block directory. """ import hashlib @@ -46,7 +46,8 @@ HEADER = struct.Struct('>8sI32sqq') BLOCK = struct.Struct('>qqq') MAX_BLOCKS = 131072 -BLOCK_BYTES = 34 +BLOCK_BYTES = 38 +V3_BLOCK_BYTES = 34 PAIR = struct.Struct('>qq') LONG = struct.Struct('>q') _PROPAGATED_ERRORS = (InterruptedError, CancelledError, ArrowCancelled, MemoryError, RecursionError) @@ -60,8 +61,11 @@ class Settings: max_bytes: int = 8 * 1024 * 1024 max_partitions: int = 65536 max_partition_bytes: int = 1024 * 1024 + max_bucket_pairs: int = 4096 def __post_init__(self): + if not 1 <= self.max_bucket_pairs <= 1048576: + raise ValueError('Invalid manifest.index.max-bucket-pairs') if not 1 <= self.max_partitions <= 1048576: raise ValueError('Invalid manifest.index.max-partitions') if not 1 <= self.max_partition_bytes <= 64 * 1024 * 1024: @@ -79,7 +83,8 @@ def from_options(cls, options): options.options.get(CoreOptions.MANIFEST_ROW_ID_INDEX_MAX_RANGES), options.options.get(CoreOptions.MANIFEST_ROW_ID_INDEX_MAX_BYTES), options.options.get(CoreOptions.MANIFEST_INDEX_MAX_PARTITIONS), - options.options.get(CoreOptions.MANIFEST_INDEX_MAX_PARTITION_BYTES)) + options.options.get(CoreOptions.MANIFEST_INDEX_MAX_PARTITION_BYTES), + options.options.get(CoreOptions.MANIFEST_INDEX_MAX_BUCKET_PAIRS)) @dataclass(frozen=True) @@ -120,6 +125,7 @@ def __init__(self, settings, header): self.blocks = [] self.ranges = [] self.partition_ids = set() + self.bucket_pairs = set() self.next_offset = len(header) if header is not None else 0 self.next_record = 0 self.current = None @@ -137,18 +143,21 @@ def begin_block(self, offset, length, records): self.current = Block(offset, length, self.next_record, records) self.entries_in_block = 0 self.row_available = self.partition_available = True + self.bucket_available = True self.coarse = False self.min = MAX_ROW_ID self.max = -1 self.ranges.clear() self.partition_ids.clear() + self.bucket_pairs.clear() - def add(self, first, count, partition=None): + def add(self, first, count, partition=None, bucket=None, total_buckets=None): if not self.complete: return _require(self.current is not None) self.entries_in_block += 1 self._add_partition(partition) + self._add_bucket(bucket, total_buckets) if not self.row_available: return if (first is None or first < 0 or count <= 0 @@ -175,6 +184,22 @@ def add(self, first, count, partition=None): else: self.ranges[left:right] = [(first, end)] + def _add_bucket(self, bucket, total_buckets): + if not self.bucket_available: + return + if (bucket is None or total_buckets is None or bucket < 0 or total_buckets <= bucket + or total_buckets > (1 << 31) - 1): + self.bucket_available = False + self.bucket_pairs.clear() + return + pair = (bucket, total_buckets) + if (pair not in self.bucket_pairs + and len(self.bucket_pairs) >= min(self.settings.max_bucket_pairs, self.settings.max_bytes // 8)): + self.bucket_available = False + self.bucket_pairs.clear() + else: + self.bucket_pairs.add(pair) + def _add_partition(self, partition): if not self.partition_available: return @@ -217,12 +242,19 @@ def end_block(self): partition_payload = struct.pack('>I', len(self.partition_ids)) partition_payload += b''.join(struct.pack('>I', id_) for id_ in sorted(self.partition_ids)) self.optional_bytes += len(partition_payload) - self.blocks.append([block, partition_payload, row_payload]) + bucket_payload = b'' + if (self.bucket_available + and 4 + 8 * len(self.bucket_pairs) <= self.settings.max_bytes - self.optional_bytes): + bucket_payload = struct.pack('>I', len(self.bucket_pairs)) + bucket_payload += b''.join(struct.pack('>ii', *pair) for pair in sorted(self.bucket_pairs)) + self.optional_bytes += len(bucket_payload) + self.blocks.append([block, partition_payload, row_payload, bucket_payload]) self.saved_ranges += row_count self.next_offset = block.offset + block.length self.next_record = block.first_record + block.record_count self.ranges.clear() self.partition_ids.clear() + self.bucket_pairs.clear() self.current = None def serialize(self, name, file_size, entry_count): @@ -237,6 +269,12 @@ def serialize(self, name, file_size, entry_count): size -= len(item[2]) self.optional_bytes -= len(item[2]) item[2] = b'' + for item in self.blocks: + if size <= self.settings.max_bytes: + break + size -= len(item[3]) + self.optional_bytes -= len(item[3]) + item[3] = b'' if size > self.settings.max_bytes: size -= self.dictionary_bytes self.dictionary_bytes = 0 @@ -247,7 +285,7 @@ def serialize(self, name, file_size, entry_count): item[1] = b'' _require(size <= self.settings.max_bytes) data = bytearray(HEADER.pack( - MAGIC, 3, hashlib.sha256(name.encode('utf-8')).digest(), file_size, entry_count)) + MAGIC, 4, hashlib.sha256(name.encode('utf-8')).digest(), file_size, entry_count)) data.extend(struct.pack('>I', len(self.header))) data.extend(self.header) data.extend(struct.pack('>I', len(self.dictionary))) @@ -255,11 +293,13 @@ def serialize(self, name, file_size, entry_count): data.extend(struct.pack('>I', len(partition))) data.extend(partition) data.extend(struct.pack('>I', len(self.blocks))) - for block, partitions, row_ids in self.blocks: + for block, partitions, row_ids, buckets in self.blocks: data.extend(BLOCK.pack(block.offset, block.length, block.record_count)) for payload in (partitions, row_ids): data.extend(struct.pack('>BI', 1 if payload else 0, len(payload))) data.extend(payload) + data.extend(struct.pack('>i', len(buckets) if buckets else -1)) + data.extend(buckets) return bytes(data) + hashlib.sha256(data).digest() @@ -277,7 +317,8 @@ def build_from_entries(avro_bytes, entries, name, settings): _require(end <= len(entries)) for i in range(position, end): entry = entries[i] - builder.add(entry.file.first_row_id, entry.file.row_count, GenericRowSerializer.to_bytes(entry.partition)) + builder.add(entry.file.first_row_id, entry.file.row_count, GenericRowSerializer.to_bytes(entry.partition), + entry.bucket, entry.total_buckets) if not builder.complete: break builder.end_block() @@ -291,7 +332,7 @@ def _require(condition): raise ValueError('Invalid, unsupported, mismatched or over-budget manifest row-id block index') -def select(data, manifest, query, settings, partition_filter=None, partition_fields=None): +def select(data, manifest, query, settings, partition_filter=None, partition_fields=None, bucket_filter=None): if query is not None and not isinstance(query, Query): query = Query(query) _require(128 <= len(data) <= settings.max_bytes) @@ -300,7 +341,8 @@ def select(data, manifest, query, settings, partition_filter=None, partition_fie limit = len(data) - 32 _require(hashlib.sha256(data[:limit]).digest() == data[limit:]) magic, version, name_hash, size, entries = HEADER.unpack_from(data) - _require(magic == MAGIC and version == 3) + _require(magic == MAGIC and version in (3, 4)) + block_bytes = V3_BLOCK_BYTES if version == 3 else BLOCK_BYTES _require(name_hash == hashlib.sha256(manifest.file_name.encode('utf-8')).digest()) _require(size == manifest.file_size and entries == manifest.num_added_files + manifest.num_deleted_files) header_length, = struct.unpack_from('>I', data, HEADER.size) @@ -336,13 +378,13 @@ def select(data, manifest, query, settings, partition_filter=None, partition_fie _require(offset + 4 <= limit) blocks, = struct.unpack_from('>I', data, offset) offset += 4 - _require(blocks <= MAX_BLOCKS and blocks <= (limit - offset) // BLOCK_BYTES) + _require(blocks <= MAX_BLOCKS and blocks <= (limit - offset) // block_bytes) next_offset = header_length first_record = 0 total_ranges = 0 selected = [] for _ in range(blocks): - _require(offset + BLOCK_BYTES <= limit) + _require(offset + block_bytes <= limit) file_offset, length, count = BLOCK.unpack_from(data, offset) offset += BLOCK.size _require(file_offset == next_offset and 0 < length <= size - file_offset) @@ -390,7 +432,25 @@ def select(data, manifest, query, settings, partition_filter=None, partition_fie if candidate and not row_hit: row_hit = ranges == 1 or query.intersects(start, end) offset += payload_length - if partition_hit and row_hit: + bucket_hit = True + if version >= 4: + _require(offset + 4 <= limit) + bucket_length, = struct.unpack_from('>i', data, offset) + offset += 4 + if bucket_length != -1: + _require(4 <= bucket_length <= limit - offset) + pairs, = struct.unpack_from('>I', data, offset) + _require(0 < pairs <= settings.max_bucket_pairs and bucket_length == 4 + 8 * pairs) + bucket_hit = bucket_filter is None + previous = (-1, -1) + for j in range(pairs): + bucket, total_buckets = struct.unpack_from('>ii', data, offset + 4 + 8 * j) + _require(0 <= bucket < total_buckets and (bucket, total_buckets) > previous) + previous = (bucket, total_buckets) + if not bucket_hit: + bucket_hit = bucket_filter(bucket, total_buckets) + offset += bucket_length + if partition_hit and row_hit and bucket_hit: selected.append(Block(file_offset, length, first_record, count)) next_offset = file_offset + length first_record += count @@ -458,7 +518,8 @@ def index_file_name(manifest): return next((name for name in manifest.extra_files or [] if name.endswith(SUFFIX)), None) -def read_index(file_io, manifest_path, manifest, query, settings, partition_filter=None, partition_fields=None): +def read_index(file_io, manifest_path, manifest, query, settings, partition_filter=None, partition_fields=None, + bucket_filter=None): name = index_file_name(manifest) if name is None: return None @@ -472,7 +533,7 @@ def read_index(file_io, manifest_path, manifest, query, settings, partition_filt break data.extend(chunk) _require(len(data) <= settings.max_bytes) - return select(data, manifest, query, settings, partition_filter, partition_fields) + return select(data, manifest, query, settings, partition_filter, partition_fields, bucket_filter) except _PROPAGATED_ERRORS: raise except Exception as error: diff --git a/paimon-python/pypaimon/tests/manifest/manifest_block_index_test.py b/paimon-python/pypaimon/tests/manifest/manifest_block_index_test.py index 03bd38108b79..4ce1411055f6 100644 --- a/paimon-python/pypaimon/tests/manifest/manifest_block_index_test.py +++ b/paimon-python/pypaimon/tests/manifest/manifest_block_index_test.py @@ -66,7 +66,7 @@ def test_partition_dictionary_golden_tuples_nulls_and_derived_ordinals(): builder.add(first, count, p) builder.end_block() data = builder.serialize('manifest-golden', len(header) + 400, 7) - assert data == fixture('indexJointV3') + assert data == fixture('indexJointV4') predicate = part(7) with patch.object(predicate, 'test', wraps=predicate.test) as evaluated: selected = select(data, golden_meta(), [Range(20, 20)], Settings(), predicate, FIELDS) @@ -127,7 +127,13 @@ def positions(data): p = offset + 24 r = p + 5 + struct.unpack_from('>I', data, p + 1)[0] offset = r + 5 + struct.unpack_from('>I', data, r + 1)[0] - result.append((block, p, r)) + if struct.unpack_from('>I', data, 8)[0] >= 4: + bucket = offset + length, = struct.unpack_from('>i', data, offset) + offset += 4 + max(0, length) + result.append((block, p, r, bucket)) + else: + result.append((block, p, r)) return result @@ -216,3 +222,57 @@ def test_randomized_budget_degradation_has_no_false_negatives(): if (any(row_p == p for _, _, row_p in values) and any(first is None or first <= point < first + count for first, count, _ in values)): assert i * 6 in ordinals + + +def test_bucket_payload_golden_rescale_and_old_versions(): + a, b, header = partition(7, 'left'), partition(9, None), avro_header() + builder = Builder(Settings(), header) + for offset, length, values in [ + (0, 100, [(0, 10, a, 1, 4), (5, 5, a, 1, 4), (20, 5, b, 1, 8)]), + (100, 200, [((1 << 32) - 2, 5, b, 2, 4), (8254058425445, 1, a, 2, 8)]), + (300, 100, [(20, 5, a, 0, 1), ((1 << 63) - 1, 1, b, 3, 4)])]: + builder.begin_block(len(header) + offset, length, len(values)) + for value in values: + builder.add(*value) + builder.end_block() + data = builder.serialize('manifest-golden', len(header) + 400, 7) + assert data == fixture('indexBucketV4') + selected = select(data, golden_meta(), None, Settings(), bucket_filter=lambda bucket, total: bucket == 1) + assert [b.first_record for b in selected.blocks] == [0] + selected = select(data, golden_meta(), None, Settings(), + bucket_filter=lambda bucket, total: (bucket, total) == (2, 8)) + assert [b.first_record for b in selected.blocks] == [3] + assert not select(data, golden_meta(), [Range(0, 0)], Settings(), part(7), FIELDS, + bucket_filter=lambda bucket, total: bucket == 2).blocks + for data in [golden(), fixture('indexJointV3'), fixture('indexJointV4')]: + assert len(select(data, golden_meta(), None, Settings(), bucket_filter=lambda bucket, total: False).blocks) == 3 + + +@pytest.mark.parametrize('pair', [(None, None), (-1, 4), (4, 4), (0, 0), (2, 8)]) +def test_bucket_budget_and_unknown_pairs_degrade_only_bucket_payload(pair): + header, settings = avro_header(), Settings(max_bucket_pairs=1) + builder = Builder(settings, header) + builder.begin_block(len(header), 100, 2) + builder.add(100, 10, partition(7, 'left'), 1, 4) + builder.add(200, 10, partition(7, 'left'), *pair) + builder.end_block() + builder.begin_block(len(header) + 100, 100, 1) + builder.add(300, 10, partition(7, 'left'), 1, 4) + builder.end_block() + data = builder.serialize('m', len(header) + 200, 3) + assert struct.unpack_from('>i', data, positions(data)[0][3])[0] == -1 + metadata = meta('m', len(header) + 200, 3) + selected = select(data, metadata, None, settings, bucket_filter=lambda bucket, total: False) + assert [b.first_record for b in selected.blocks] == [0] + assert not select(data, metadata, [Range(999, 999)], settings, bucket_filter=lambda bucket, total: False).blocks + + +def test_malformed_bucket_payload_invalidates_the_container(): + good = fixture('indexBucketV4') + payload = positions(good)[0][3] + for offset, value in [(payload, -2), (payload, (1 << 31) - 1), (payload, 0), (payload + 4, 0), + (payload + 8, -1), (payload + 12, 1), (payload + 16, 0)]: + bad = bytearray(good) + struct.pack_into('>i', bad, offset, value) + with pytest.raises(ValueError): + select(checksum(bad), golden_meta(), [Range(999, 999)], Settings(), part(99), FIELDS) diff --git a/paimon-python/pypaimon/tests/manifest/row_id_index_test.py b/paimon-python/pypaimon/tests/manifest/row_id_index_test.py index 482f4aeedec3..24c1ceeba3a6 100644 --- a/paimon-python/pypaimon/tests/manifest/row_id_index_test.py +++ b/paimon-python/pypaimon/tests/manifest/row_id_index_test.py @@ -52,7 +52,7 @@ def fixture(): path = (Path(__file__).resolve().parents[4] / 'paimon-core/src/test/resources' / 'manifest-row-id-index-v2.txt') return dict(line.split('=', 1) for line in path.read_text().splitlines() - if line.startswith(('index=', 'indexV3=', 'avroHeader='))) + if line.startswith(('index=', 'indexV3=', 'indexV4=', 'avroHeader='))) def golden(): @@ -237,7 +237,7 @@ def test_cross_language_and_block_ordinals(self): b.add(first, count) b.end_block() self.assertEqual(b.serialize(meta.file_name, meta.file_size, 7), - base64.b64decode(fixture()['indexV3'])) + base64.b64decode(fixture()['indexV4'])) def test_minmax_skips_exact_checks_and_one_interval_is_already_exact(self): header = avro_header() @@ -335,6 +335,39 @@ def write_meta(self, name, entries): manager = self.manifest_file_manager return manager.write(name, entries) + def test_bucket_point_lookup_with_rescale_and_delete_entries(self): + from pypaimon.common.predicate import Predicate + from pypaimon.read.scanner.bucket_select_converter import create_bucket_selector + selector = create_bucket_selector(Predicate('equal', 0, 'id', [7]), self.table.fields[:1]) + self.assertIsNotNone(selector) + entries = [replace(self.entry('%s-%s-%s.parquet' % (total, bucket, i), None), + bucket=bucket, total_buckets=total) + for total in (4, 8) for bucket in range(total) for i in range(300)] + metadata = self.write_meta('buckets', entries) + results = [] + for enabled in (False, True): + self.table.options.options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_READ, enabled) + scanner = FileScanner(self.table, lambda: ([metadata], None)) + scanner._bucket_selector = selector + with patch('pypaimon.manifest.manifest_file_manager.read_selected_bytes', + wraps=read_selected_bytes) as read_blocks: + actual = scanner.read_manifest_entries([metadata]) + results.append([e.file.file_name for e in actual]) + if enabled: + read_blocks.assert_called_once() + selected = read_blocks.call_args[0][2] + self.assertLess(sum(b.record_count for b in selected.blocks), 1000) + else: + read_blocks.assert_not_called() + self.assertEqual(results[0], results[1]) + self.assertEqual(len(results[1]), 600) + chosen = next(bucket for bucket in range(4) if selector(bucket, 4)) + added = [replace(self.entry('point.' + suffix, None), bucket=chosen, total_buckets=4) + for suffix in ('parquet', 'blob')] + metas = [self.write_meta('point-add', added), + self.write_meta('point-delete', [replace(e, kind=1) for e in added])] + self.assertEqual(scanner.read_manifest_entries(metas), []) + def test_partition_only_and_conjunctive_planning_keep_entry_and_delete_filters(self): import pyarrow as pa schema = Schema.from_pyarrow_schema( From 5dd40f3084eef5a102bddde1b2ae38f154435ace Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Mon, 14 Sep 2026 11:19:54 +0800 Subject: [PATCH 07/23] [core] Remove compatibility with draft manifest index formats --- docs/docs/concepts/spec/manifest.md | 13 +- .../paimon/manifest/ManifestRowIdIndex.java | 151 +++--------------- .../manifest/ManifestBlockIndexTest.java | 28 ++-- .../manifest/ManifestRowIdIndexTest.java | 12 +- .../test/resources/manifest-block-index.txt | 23 +++ .../resources/manifest-row-id-index-v2.txt | 26 --- .../pypaimon/manifest/row_id_index.py | 109 +++---------- .../manifest/manifest_block_index_test.py | 27 ++-- .../tests/manifest/row_id_index_test.py | 12 +- 9 files changed, 111 insertions(+), 290 deletions(-) create mode 100644 paimon-core/src/test/resources/manifest-block-index.txt delete mode 100644 paimon-core/src/test/resources/manifest-row-id-index-v2.txt diff --git a/docs/docs/concepts/spec/manifest.md b/docs/docs/concepts/spec/manifest.md index 745d6f85b46e..9120480c3f0c 100644 --- a/docs/docs/concepts/spec/manifest.md +++ b/docs/docs/concepts/spec/manifest.md @@ -68,7 +68,7 @@ that manifest during snapshot, tag, or changelog deletion. With `manifest.row-id-index.write` enabled, a manifest writer can create a binary `.row-id-index` sidecar. Its name is stored in the manifest-list record's `_EXTRA_FILES`; the existing Avro schemas and `_VERSION` identifiers are unchanged. -Readers identify the block index by the historical `.row-id-index` suffix among these explicit +Readers identify the block index by the `.row-id-index` suffix among these explicit references, not by probing for a derived file name. Other extra-file references are preserved. With `manifest.row-id-index.read` enabled and a partition, row-ID or bucket filter available, readers can use @@ -79,12 +79,12 @@ or over-budget containers also fall back to that path. Each block's partition, r coverage is independently usable; an unavailable dimension cannot exclude a block. Cancellation and interruption errors propagate instead of triggering a full-manifest fallback. -Version 4 uses the following layout. Container integers and payload integers +Version 1 uses the following layout. Container integers and payload integers are fixed-width big endian. Encoding IDs are unsigned bytes with separate namespaces. ```text magic : 8 bytes // ASCII PAIMRIDX -formatVersion : int // 4 +formatVersion : int // 1 manifestNameHash : 32 bytes // SHA-256 of the UTF-8 basename manifestLength : long manifestEntryCount : long // ADD + DELETE @@ -163,17 +163,14 @@ No emitted sidecar omits block descriptors. These are encoded-size bounds; const also incurs bounded object/buffer overhead. Query concurrency multiplies per-reader costs. For conjunctive filters a block is retained only if each dimension is either unavailable -or matches. Matches in the two dimensions can come from different entries in the block, +or matches. Matches in different dimensions can come from different entries in the block, so entry filtering and deletion merging remain necessary. Block min/max is derived from the first/last interval before testing the individual intervals. Readers still consume and validate the whole bounded sidecar. A partition-only query therefore reads row-ID payload bytes too; payload lengths save decoding work for unknown encodings, not storage I/O. Selected compressed blocks are read by byte range with adjacent -spans coalesced. Version-2 row-ID-only sidecars and version-3 partition/row-ID sidecars -remain readable with missing coverage treated as unavailable; older readers safely fall -back on version 4. Existing immutable -manifests are not backfilled by enabling the write option. +spans coalesced. Existing immutable manifests are not backfilled by enabling the write option. Selected blocks still pass through entry filtering and ADD/DELETE reconciliation. Snapshot, tag, changelog, orphan-file and failed-commit cleanup retain or remove the sidecar through diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestRowIdIndex.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestRowIdIndex.java index 192da1bca026..9f1289ee26a1 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestRowIdIndex.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestRowIdIndex.java @@ -34,9 +34,7 @@ import javax.annotation.Nullable; -import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; -import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.EOFException; import java.io.IOException; @@ -68,10 +66,9 @@ public final class ManifestRowIdIndex { public static final String SUFFIX = ".row-id-index"; private static final Logger LOG = LoggerFactory.getLogger(ManifestRowIdIndex.class); private static final long MAGIC = 0x5041494d52494458L; + private static final int FORMAT_VERSION = 1; private static final int HEADER_BYTES = 60; - private static final int LEGACY_HEADER_BYTES = 68; private static final int BLOCK_BYTES = 38; - private static final int V3_BLOCK_BYTES = 34; private static final int MAX_BLOCKS = 131072; private static final byte[] EMPTY = new byte[0]; private static final int DIGEST_BYTES = 32; @@ -446,7 +443,7 @@ public byte[] serialize(String name, long fileSize, long entryCount) throws IOEx ByteArrayOutputStream buffer = new ByteArrayOutputStream((int) size); DataOutputStream out = new DataOutputStream(buffer); out.writeLong(MAGIC); - out.writeInt(4); + out.writeInt(FORMAT_VERSION); out.write(digest(name.getBytes(StandardCharsets.UTF_8))); out.writeLong(fileSize); out.writeLong(entryCount); @@ -551,10 +548,6 @@ public static Selection select( Settings settings) throws IOException { require(data.length >= 128 && data.length <= settings.maxBytes); - if (ByteBuffer.wrap(data).getInt(8) == 0x00020002) { - // Published prototype files have row-ID coverage only. - return selectLegacy(data, manifest, query, settings); - } int limit = data.length - DIGEST_BYTES; require( MessageDigest.isEqual( @@ -562,9 +555,7 @@ public static Selection select( Arrays.copyOfRange(data, limit, data.length))); ByteBuffer in = ByteBuffer.wrap(data, 0, limit).slice(); require(in.getLong() == MAGIC); - int version = in.getInt(); - require(version == 3 || version == 4); - int blockBytes = version == 3 ? V3_BLOCK_BYTES : BLOCK_BYTES; + require(in.getInt() == FORMAT_VERSION); byte[] hash = new byte[DIGEST_BYTES]; in.get(hash); require( @@ -615,13 +606,13 @@ public static Selection select( } require(in.remaining() >= 4); int count = in.getInt(); - require(count >= 0 && count <= MAX_BLOCKS && count <= in.remaining() / blockBytes); + require(count >= 0 && count <= MAX_BLOCKS && count <= in.remaining() / BLOCK_BYTES); long nextOffset = headerLength; long firstRecord = 0; int totalRanges = 0; List selected = new ArrayList<>(); for (int i = 0; i < count; i++) { - require(in.remaining() >= blockBytes); + require(in.remaining() >= BLOCK_BYTES); long offset = in.getLong(); long length = in.getLong(); long records = in.getLong(); @@ -676,28 +667,26 @@ public static Selection select( } } boolean bucketHit = true; - if (version >= 4) { - require(in.remaining() >= 4); - int bucketLength = in.getInt(); - if (bucketLength != -1) { - require(bucketLength >= 4 && bucketLength <= in.remaining()); - int pairs = in.getInt(); - require( - pairs > 0 - && pairs <= settings.maxBucketPairs - && bucketLength == 4L + 8L * pairs); - bucketHit = bucketFilter == null; - long previous = -1; - for (int j = 0; j < pairs; j++) { - int bucket = in.getInt(); - int totalBuckets = in.getInt(); - require(bucket >= 0 && totalBuckets > bucket); - long pair = ((long) bucket << 32) | totalBuckets; - require(pair > previous); - previous = pair; - if (!bucketHit) { - bucketHit = bucketFilter.mayContain(bucket, totalBuckets); - } + require(in.remaining() >= 4); + int bucketLength = in.getInt(); + if (bucketLength != -1) { + require(bucketLength >= 4 && bucketLength <= in.remaining()); + int pairs = in.getInt(); + require( + pairs > 0 + && pairs <= settings.maxBucketPairs + && bucketLength == 4L + 8L * pairs); + bucketHit = bucketFilter == null; + long previous = -1; + for (int j = 0; j < pairs; j++) { + int bucket = in.getInt(); + int totalBuckets = in.getInt(); + require(bucket >= 0 && totalBuckets > bucket); + long pair = ((long) bucket << 32) | totalBuckets; + require(pair > previous); + previous = pair; + if (!bucketHit) { + bucketHit = bucketFilter.mayContain(bucket, totalBuckets); } } } @@ -721,96 +710,6 @@ private static ByteBuffer payload(ByteBuffer in) throws IOException { return result; } - /** Validate the complete index before allowing any negative decision. */ - private static Selection selectLegacy( - byte[] data, - ManifestFileMeta manifest, - @Nullable RowRangeIndex query, - Settings settings) - throws IOException { - require(data.length >= 128 && data.length <= settings.maxBytes); - int checksumOffset = data.length - DIGEST_BYTES; - require( - MessageDigest.isEqual( - digest(Arrays.copyOf(data, checksumOffset)), - Arrays.copyOfRange(data, checksumOffset, data.length))); - DataInputStream in = new DataInputStream(new ByteArrayInputStream(data, 0, checksumOffset)); - require( - in.readLong() == MAGIC - && in.readUnsignedShort() == 2 - && in.readUnsignedShort() == 2 - && in.readInt() == 1); - byte[] nameHash = new byte[DIGEST_BYTES]; - in.readFully(nameHash); - require( - MessageDigest.isEqual( - nameHash, digest(manifest.fileName().getBytes(StandardCharsets.UTF_8)))); - require(in.readLong() == manifest.fileSize()); - long entries = Math.addExact(manifest.numAddedFiles(), manifest.numDeletedFiles()); - require(in.readLong() == entries && in.readInt() == checksumOffset - LEGACY_HEADER_BYTES); - int headerLength = in.readInt(); - require( - headerLength >= 21 - && headerLength <= MAX_AVRO_HEADER - && headerLength <= in.available() - 4); - byte[] header = new byte[headerLength]; - in.readFully(header); - require(header[0] == 'O' && header[1] == 'b' && header[2] == 'j' && header[3] == 1); - int blocks = in.readInt(); - require(blocks >= 0 && blocks <= in.available() / 52); - long nextOffset = headerLength; - long nextRecord = 0; - int totalRanges = 0; - List selected = new ArrayList<>(); - ByteBuffer view = ByteBuffer.wrap(data); - for (int i = 0; i < blocks; i++) { - long offset = in.readLong(); - long length = in.readLong(); - long first = in.readLong(); - long count = in.readLong(); - int ranges = in.readInt(); - require(offset == nextOffset && length > 0 && length <= manifest.fileSize() - offset); - require(first == nextRecord && count > 0 && count <= entries - first); - require( - ranges > 0 - && ranges <= settings.maxRanges - totalRanges - && ranges <= in.available() / 16); - totalRanges += ranges; - int rangesEnd = checksumOffset - in.available() + 16 * ranges; - long minRowId = in.readLong(); - long firstEnd = in.readLong(); - // Sorted intervals already encode the envelope. Peek at the final endpoint without - // adding redundant fields to the format or materializing the interval list. - long maxRowId = ranges == 1 ? firstEnd : view.getLong(rangesEnd - Long.BYTES); - require(minRowId >= 0 && firstEnd >= minRowId && maxRowId >= firstEnd); - boolean candidate = query == null || query.intersects(minRowId, maxRowId); - boolean hit = - candidate - && (query == null - || ranges == 1 - || query.intersects(minRowId, firstEnd)); - long previousEnd = firstEnd; - for (int j = 1; j < ranges; j++) { - long start = in.readLong(); - long end = in.readLong(); - // Validate even rejected blocks: a checksummed but malformed interval list must - // still cause a conservative fallback, not a false negative from its envelope. - require(start >= 0 && end >= start && start > previousEnd); - previousEnd = end; - if (candidate && !hit) { - hit = query.intersects(start, end); - } - } - if (hit) { - selected.add(new Block(offset, length, first, count)); - } - nextOffset = offset + length; - nextRecord = first + count; - } - require(in.available() == 0 && nextOffset == manifest.fileSize() && nextRecord == entries); - return new Selection(header, selected); - } - /** Bounded, bulk index reads. Null means read the original manifest. */ @Nullable public static Selection read( diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java index 662e5f081cee..1aabacee89c5 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java @@ -59,8 +59,7 @@ class ManifestBlockIndexTest { private byte[] fixture(String field) throws IOException { Properties p = new Properties(); - try (java.io.InputStream in = - getClass().getResourceAsStream("/manifest-row-id-index-v2.txt")) { + try (java.io.InputStream in = getClass().getResourceAsStream("/manifest-block-index.txt")) { p.load(in); } return Base64.getDecoder().decode(p.getProperty(field)); @@ -109,7 +108,7 @@ void jointGoldenPreservesTuplesNullsAndDerivedOrdinals() throws Exception { builder.add(Long.MAX_VALUE, 1, b); builder.endBlock(); byte[] data = builder.serialize("manifest-golden", header.length + 400, 7); - assertThat(data).isEqualTo(fixture("indexJointV4")); + assertThat(data).isEqualTo(fixture("indexWithPartitions")); ManifestFileMeta meta = meta("manifest-golden", header.length + 400, 7); PartitionPredicate filter = spy(part(7)); assertThat( @@ -124,7 +123,7 @@ void jointGoldenPreservesTuplesNullsAndDerivedOrdinals() throws Exception { .hasSize(3); assertThat(ManifestRowIdIndex.select(data, meta, null, part(99), type, defaults).blocks()) .isEmpty(); - // The old container has no partition coverage; dictionary misses must not prune it. + // Missing partition payloads cannot be pruned by dictionary misses. assertThat( ManifestRowIdIndex.select( fixture("index"), meta, null, part(99), type, defaults) @@ -217,13 +216,10 @@ private List positions(byte[] data) { in.get(); length = in.getInt(); in.position(in.position() + length); - int bucket = -1; - if (in.getInt(8) >= 4) { - bucket = in.position(); - length = in.getInt(); - if (length >= 0) { - in.position(in.position() + length); - } + int bucket = in.position(); + length = in.getInt(); + if (length >= 0) { + in.position(in.position() + length); } result.add(new int[] {block, partition, row, bucket}); } @@ -239,7 +235,7 @@ private byte[] checksum(byte[] data) throws Exception { @Test void unknownUnsignedEncodingsSkipOnlyTheirDimensionAndMalformedPayloadsFail() throws Exception { - byte[] good = fixture("indexJointV3"); + byte[] good = fixture("indexWithPartitions"); int[] first = positions(good).get(0); ManifestFileMeta meta = meta("manifest-golden", fixture("avroHeader").length + 400, 7); byte[] data = good.clone(); @@ -315,7 +311,7 @@ void bucketPayloadGoldenAndTotalBucketsArePreserved() throws Exception { builder.add(Long.MAX_VALUE, 1, b, 3, 4); builder.endBlock(); byte[] data = builder.serialize("manifest-golden", header.length + 400, 7); - assertThat(data).isEqualTo(fixture("indexBucketV4")); + assertThat(data).isEqualTo(fixture("indexWithBuckets")); ManifestFileMeta meta = meta("manifest-golden", header.length + 400, 7); BucketFilter bucket = BucketFilter.create(false, 1, null, null); assertThat( @@ -366,10 +362,10 @@ public boolean mayContain(int min, int max, int total) { data, meta, null, null, type, partitionAware, defaults) .blocks()) .hasSize(3); - for (String old : new String[] {"index", "indexJointV3", "indexJointV4"}) { + for (String unavailable : new String[] {"index", "indexWithPartitions"}) { assertThat( ManifestRowIdIndex.select( - fixture(old), + fixture(unavailable), meta, null, null, @@ -433,7 +429,7 @@ void unknownInvalidOrOverBudgetBucketPayloadIsNull() throws Exception { @Test void malformedBucketPayloadInvalidatesTheContainer() throws Exception { - byte[] good = fixture("indexBucketV4"); + byte[] good = fixture("indexWithBuckets"); int payload = positions(good).get(0)[3]; ManifestFileMeta meta = meta("manifest-golden", fixture("avroHeader").length + 400, 7); for (int[] mutation : diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestRowIdIndexTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestRowIdIndexTest.java index aa064f91999d..6d17e3bd868c 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestRowIdIndexTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestRowIdIndexTest.java @@ -75,7 +75,7 @@ static ManifestFileMeta meta(String name, long size, long entries) { private Properties fixture() throws IOException { Properties properties = new Properties(); try (java.io.InputStream input = - getClass().getResourceAsStream("/manifest-row-id-index-v2.txt")) { + getClass().getResourceAsStream("/manifest-block-index.txt")) { properties.load(input); } return properties; @@ -111,7 +111,7 @@ void crossLanguageFormatAndBlockOrdinals() throws Exception { builder.add(Long.MAX_VALUE, 1); builder.endBlock(); byte[] data = builder.serialize("manifest-golden", header.length + 400, 7); - assertThat(data).isEqualTo(Base64.getDecoder().decode(fixture().getProperty("indexV4"))); + assertThat(data).isEqualTo(golden()); ManifestFileMeta meta = goldenMeta(); for (long point : new long[] { @@ -193,7 +193,7 @@ void minMaxSkipsExactIntersectionChecksAndHandlesOneInterval() throws Exception @Test void malformedIntervalsStillFallbackAfterMinMaxRejectionOrAnEarlyHit() throws Exception { byte[] data = golden(); - int firstBlockIntervals = 68 + 4 + header().length + 4 + 36; + int firstBlockIntervals = 60 + 4 + header().length + 4 + 4 + 24 + 5 + 5 + 4; // Make the second interval overlap the first, keeping the envelope unchanged. ByteBuffer.wrap(data).putLong(firstBlockIntervals + 16, 9L); byte[] hash = @@ -292,10 +292,10 @@ void corruptMissingIncompleteAndMismatchedIndexesFallback() throws Exception { LocalFileIO.create(), manifest, meta, query, settings)) .isNull(); } - // Self-consistent checksum cannot turn an unsupported or incomplete envelope into an index. - for (int position : new int[] {9, 11, 15}) { + // A valid checksum cannot make an unsupported container version readable. + for (int version : new int[] {0, 2, 99}) { byte[] bad = good.clone(); - bad[position] = 0; + ByteBuffer.wrap(bad).putInt(8, version); byte[] hash = MessageDigest.getInstance("SHA-256") .digest(Arrays.copyOf(bad, bad.length - 32)); diff --git a/paimon-core/src/test/resources/manifest-block-index.txt b/paimon-core/src/test/resources/manifest-block-index.txt new file mode 100644 index 000000000000..8f0c7fbea6c3 --- /dev/null +++ b/paimon-core/src/test/resources/manifest-block-index.txt @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +avroHeader=T2JqAQQUYXZyby5jb2RlYwhudWxsFmF2cm8uc2NoZW1hDCJsb25nIgAAAAAAAAAAAAAAAAAAAAAA +partitionA=AAAAAgAAAAAAAAAABwAAAAAAAABsZWZ0AAAAhA== +partitionB=AAAAAgACAAAAAAAACQAAAAAAAAAAAAAAAAAAAA== +index=UEFJTVJJRFgAAAABL0eWubkHdL6ioNdIPj3jGtp9TAk83wbtAsmc+xJVwM8AAAAAAAAByQAAAAAAAAAHAAAAOU9iagEEFGF2cm8uY29kZWMIbnVsbBZhdnJvLnNjaGVtYQwibG9uZyIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAADkAAAAAAAAAZAAAAAAAAAADAAAAAAABAAAAJAAAAAIAAAAAAAAAAAAAAAAAAAAJAAAAAAAAABQAAAAAAAAAGP////8AAAAAAAAAnQAAAAAAAADIAAAAAAAAAAIAAAAAAAEAAAAkAAAAAgAAAAD////+AAAAAQAAAAIAAAeBzDhsZQAAB4HMOGxl/////wAAAAAAAAFlAAAAAAAAAGQAAAAAAAAAAgAAAAAAAQAAACQAAAACAAAAAAAAABQAAAAAAAAAGH//////////f///////////////B9Fp1oyxbV6UmfRT5d+Fv/MxGHAmXjE2Bs5JxBGcn+E= +indexWithPartitions=UEFJTVJJRFgAAAABL0eWubkHdL6ioNdIPj3jGtp9TAk83wbtAsmc+xJVwM8AAAAAAAAByQAAAAAAAAAHAAAAOU9iagEEFGF2cm8uY29kZWMIbnVsbBZhdnJvLnNjaGVtYQwibG9uZyIAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAcAAAAAgAAAAAAAAAABwAAAAAAAABsZWZ0AAAAhAAAABwAAAACAAIAAAAAAAAJAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAA5AAAAAAAAAGQAAAAAAAAAAwEAAAAMAAAAAgAAAAAAAAABAQAAACQAAAACAAAAAAAAAAAAAAAAAAAACQAAAAAAAAAUAAAAAAAAABj/////AAAAAAAAAJ0AAAAAAAAAyAAAAAAAAAACAQAAAAwAAAACAAAAAAAAAAEBAAAAJAAAAAIAAAAA/////gAAAAEAAAACAAAHgcw4bGUAAAeBzDhsZf////8AAAAAAAABZQAAAAAAAABkAAAAAAAAAAIBAAAADAAAAAIAAAAAAAAAAQEAAAAkAAAAAgAAAAAAAAAUAAAAAAAAABh//////////3///////////////1Ap6GHiQax4WqX0VrJqQ5tMmn3ZtQkNqJckeYuATS+A +indexWithBuckets=UEFJTVJJRFgAAAABL0eWubkHdL6ioNdIPj3jGtp9TAk83wbtAsmc+xJVwM8AAAAAAAAByQAAAAAAAAAHAAAAOU9iagEEFGF2cm8uY29kZWMIbnVsbBZhdnJvLnNjaGVtYQwibG9uZyIAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAcAAAAAgAAAAAAAAAABwAAAAAAAABsZWZ0AAAAhAAAABwAAAACAAIAAAAAAAAJAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAA5AAAAAAAAAGQAAAAAAAAAAwEAAAAMAAAAAgAAAAAAAAABAQAAACQAAAACAAAAAAAAAAAAAAAAAAAACQAAAAAAAAAUAAAAAAAAABgAAAAUAAAAAgAAAAEAAAAEAAAAAQAAAAgAAAAAAAAAnQAAAAAAAADIAAAAAAAAAAIBAAAADAAAAAIAAAAAAAAAAQEAAAAkAAAAAgAAAAD////+AAAAAQAAAAIAAAeBzDhsZQAAB4HMOGxlAAAAFAAAAAIAAAACAAAABAAAAAIAAAAIAAAAAAAAAWUAAAAAAAAAZAAAAAAAAAACAQAAAAwAAAACAAAAAAAAAAEBAAAAJAAAAAIAAAAAAAAAFAAAAAAAAAAYf/////////9//////////wAAABQAAAACAAAAAAAAAAEAAAADAAAABKZ7n0UVSe+qqinQITnc3OYtZWHTXxbNPJmCRW2/eyB4 diff --git a/paimon-core/src/test/resources/manifest-row-id-index-v2.txt b/paimon-core/src/test/resources/manifest-row-id-index-v2.txt deleted file mode 100644 index b91bd5c2b0f5..000000000000 --- a/paimon-core/src/test/resources/manifest-row-id-index-v2.txt +++ /dev/null @@ -1,26 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -avroHeader=T2JqAQQUYXZyby5jb2RlYwhudWxsFmF2cm8uc2NoZW1hDCJsb25nIgAAAAAAAAAAAAAAAAAAAAAA -index=UEFJTVJJRFgAAgACAAAAAS9Hlrm5B3S+oqDXSD494xrafUwJPN8G7QLJnPsSVcDPAAAAAAAAAckAAAAAAAAABwAAAQ0AAAA5T2JqAQQUYXZyby5jb2RlYwhudWxsFmF2cm8uc2NoZW1hDCJsb25nIgAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAA5AAAAAAAAAGQAAAAAAAAAAAAAAAAAAAADAAAAAgAAAAAAAAAAAAAAAAAAAAkAAAAAAAAAFAAAAAAAAAAYAAAAAAAAAJ0AAAAAAAAAyAAAAAAAAAADAAAAAAAAAAIAAAACAAAAAP////4AAAABAAAAAgAAB4HMOGxlAAAHgcw4bGUAAAAAAAABZQAAAAAAAABkAAAAAAAAAAUAAAAAAAAAAgAAAAIAAAAAAAAAFAAAAAAAAAAYf/////////9//////////xekCXjgYJlWPiPlQ80IyyKSmIPH5z5iMhyRa8BhBph5 -partitionA=AAAAAgAAAAAAAAAABwAAAAAAAABsZWZ0AAAAhA== -partitionB=AAAAAgACAAAAAAAACQAAAAAAAAAAAAAAAAAAAA== -indexV3=UEFJTVJJRFgAAAADL0eWubkHdL6ioNdIPj3jGtp9TAk83wbtAsmc+xJVwM8AAAAAAAAByQAAAAAAAAAHAAAAOU9iagEEFGF2cm8uY29kZWMIbnVsbBZhdnJvLnNjaGVtYQwibG9uZyIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAADkAAAAAAAAAZAAAAAAAAAADAAAAAAABAAAAJAAAAAIAAAAAAAAAAAAAAAAAAAAJAAAAAAAAABQAAAAAAAAAGAAAAAAAAACdAAAAAAAAAMgAAAAAAAAAAgAAAAAAAQAAACQAAAACAAAAAP////4AAAABAAAAAgAAB4HMOGxlAAAHgcw4bGUAAAAAAAABZQAAAAAAAABkAAAAAAAAAAIAAAAAAAEAAAAkAAAAAgAAAAAAAAAUAAAAAAAAABh//////////3//////////ViRH8zjNd2Q9F4olffQ2ZQiExUDkv3wPwoBnsEGW3sQ= -indexJointV3=UEFJTVJJRFgAAAADL0eWubkHdL6ioNdIPj3jGtp9TAk83wbtAsmc+xJVwM8AAAAAAAAByQAAAAAAAAAHAAAAOU9iagEEFGF2cm8uY29kZWMIbnVsbBZhdnJvLnNjaGVtYQwibG9uZyIAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAcAAAAAgAAAAAAAAAABwAAAAAAAABsZWZ0AAAAhAAAABwAAAACAAIAAAAAAAAJAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAA5AAAAAAAAAGQAAAAAAAAAAwEAAAAMAAAAAgAAAAAAAAABAQAAACQAAAACAAAAAAAAAAAAAAAAAAAACQAAAAAAAAAUAAAAAAAAABgAAAAAAAAAnQAAAAAAAADIAAAAAAAAAAIBAAAADAAAAAIAAAAAAAAAAQEAAAAkAAAAAgAAAAD////+AAAAAQAAAAIAAAeBzDhsZQAAB4HMOGxlAAAAAAAAAWUAAAAAAAAAZAAAAAAAAAACAQAAAAwAAAACAAAAAAAAAAEBAAAAJAAAAAIAAAAAAAAAFAAAAAAAAAAYf/////////9//////////4mo+QwqXuGgI0WNm1HdwCQA6dlu7UP8TigmfZSh14mZ -indexV4=UEFJTVJJRFgAAAAEL0eWubkHdL6ioNdIPj3jGtp9TAk83wbtAsmc+xJVwM8AAAAAAAAByQAAAAAAAAAHAAAAOU9iagEEFGF2cm8uY29kZWMIbnVsbBZhdnJvLnNjaGVtYQwibG9uZyIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAADkAAAAAAAAAZAAAAAAAAAADAAAAAAABAAAAJAAAAAIAAAAAAAAAAAAAAAAAAAAJAAAAAAAAABQAAAAAAAAAGP////8AAAAAAAAAnQAAAAAAAADIAAAAAAAAAAIAAAAAAAEAAAAkAAAAAgAAAAD////+AAAAAQAAAAIAAAeBzDhsZQAAB4HMOGxl/////wAAAAAAAAFlAAAAAAAAAGQAAAAAAAAAAgAAAAAAAQAAACQAAAACAAAAAAAAABQAAAAAAAAAGH//////////f///////////////yreFS/v5MsyAxSS6WSEBiJQTdm4As0DTJz5RJ/UsNkE= -indexJointV4=UEFJTVJJRFgAAAAEL0eWubkHdL6ioNdIPj3jGtp9TAk83wbtAsmc+xJVwM8AAAAAAAAByQAAAAAAAAAHAAAAOU9iagEEFGF2cm8uY29kZWMIbnVsbBZhdnJvLnNjaGVtYQwibG9uZyIAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAcAAAAAgAAAAAAAAAABwAAAAAAAABsZWZ0AAAAhAAAABwAAAACAAIAAAAAAAAJAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAA5AAAAAAAAAGQAAAAAAAAAAwEAAAAMAAAAAgAAAAAAAAABAQAAACQAAAACAAAAAAAAAAAAAAAAAAAACQAAAAAAAAAUAAAAAAAAABj/////AAAAAAAAAJ0AAAAAAAAAyAAAAAAAAAACAQAAAAwAAAACAAAAAAAAAAEBAAAAJAAAAAIAAAAA/////gAAAAEAAAACAAAHgcw4bGUAAAeBzDhsZf////8AAAAAAAABZQAAAAAAAABkAAAAAAAAAAIBAAAADAAAAAIAAAAAAAAAAQEAAAAkAAAAAgAAAAAAAAAUAAAAAAAAABh//////////3///////////////6c8cPEEmOjexjRnjlIO8fIsDVHyZ0EFCl2Yx26fiIzD -indexBucketV4=UEFJTVJJRFgAAAAEL0eWubkHdL6ioNdIPj3jGtp9TAk83wbtAsmc+xJVwM8AAAAAAAAByQAAAAAAAAAHAAAAOU9iagEEFGF2cm8uY29kZWMIbnVsbBZhdnJvLnNjaGVtYQwibG9uZyIAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAcAAAAAgAAAAAAAAAABwAAAAAAAABsZWZ0AAAAhAAAABwAAAACAAIAAAAAAAAJAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAA5AAAAAAAAAGQAAAAAAAAAAwEAAAAMAAAAAgAAAAAAAAABAQAAACQAAAACAAAAAAAAAAAAAAAAAAAACQAAAAAAAAAUAAAAAAAAABgAAAAUAAAAAgAAAAEAAAAEAAAAAQAAAAgAAAAAAAAAnQAAAAAAAADIAAAAAAAAAAIBAAAADAAAAAIAAAAAAAAAAQEAAAAkAAAAAgAAAAD////+AAAAAQAAAAIAAAeBzDhsZQAAB4HMOGxlAAAAFAAAAAIAAAACAAAABAAAAAIAAAAIAAAAAAAAAWUAAAAAAAAAZAAAAAAAAAACAQAAAAwAAAACAAAAAAAAAAEBAAAAJAAAAAIAAAAAAAAAFAAAAAAAAAAYf/////////9//////////wAAABQAAAACAAAAAAAAAAEAAAADAAAABEZWMYlUL4U9F6ENk1p6oZeCOokJCzCKEdTwv9geIID0 diff --git a/paimon-python/pypaimon/manifest/row_id_index.py b/paimon-python/pypaimon/manifest/row_id_index.py index 7a884caca890..09b4437e3921 100644 --- a/paimon-python/pypaimon/manifest/row_id_index.py +++ b/paimon-python/pypaimon/manifest/row_id_index.py @@ -15,10 +15,7 @@ # specific language governing permissions and limitations # under the License. -"""Complete row-id interval unions with Avro block offsets and entry ordinals. - -Version 4 adds nullable bucket/total-bucket pairs to the version-3 block directory. -""" +"""Independent partition, row-id and nullable bucket coverage for each Avro block.""" import hashlib import logging @@ -38,16 +35,14 @@ LOG = logging.getLogger(__name__) SUFFIX = '.row-id-index' MAGIC = b'PAIMRIDX' +FORMAT_VERSION = 1 MAX_ROW_ID = (1 << 63) - 1 MAX_AVRO_HEADER = 1024 * 1024 READ_BUFFER_BYTES = 1024 * 1024 -LEGACY_HEADER = struct.Struct('>8sHHI32sqqI') -LEGACY_BLOCK = struct.Struct('>qqqqI') HEADER = struct.Struct('>8sI32sqq') BLOCK = struct.Struct('>qqq') MAX_BLOCKS = 131072 BLOCK_BYTES = 38 -V3_BLOCK_BYTES = 34 PAIR = struct.Struct('>qq') LONG = struct.Struct('>q') _PROPAGATED_ERRORS = (InterruptedError, CancelledError, ArrowCancelled, MemoryError, RecursionError) @@ -285,7 +280,7 @@ def serialize(self, name, file_size, entry_count): item[1] = b'' _require(size <= self.settings.max_bytes) data = bytearray(HEADER.pack( - MAGIC, 4, hashlib.sha256(name.encode('utf-8')).digest(), file_size, entry_count)) + MAGIC, FORMAT_VERSION, hashlib.sha256(name.encode('utf-8')).digest(), file_size, entry_count)) data.extend(struct.pack('>I', len(self.header))) data.extend(self.header) data.extend(struct.pack('>I', len(self.dictionary))) @@ -336,13 +331,10 @@ def select(data, manifest, query, settings, partition_filter=None, partition_fie if query is not None and not isinstance(query, Query): query = Query(query) _require(128 <= len(data) <= settings.max_bytes) - if struct.unpack_from('>I', data, 8)[0] == 0x00020002: - return _select_legacy(data, manifest, query, settings) limit = len(data) - 32 _require(hashlib.sha256(data[:limit]).digest() == data[limit:]) magic, version, name_hash, size, entries = HEADER.unpack_from(data) - _require(magic == MAGIC and version in (3, 4)) - block_bytes = V3_BLOCK_BYTES if version == 3 else BLOCK_BYTES + _require(magic == MAGIC and version == FORMAT_VERSION) _require(name_hash == hashlib.sha256(manifest.file_name.encode('utf-8')).digest()) _require(size == manifest.file_size and entries == manifest.num_added_files + manifest.num_deleted_files) header_length, = struct.unpack_from('>I', data, HEADER.size) @@ -378,13 +370,13 @@ def select(data, manifest, query, settings, partition_filter=None, partition_fie _require(offset + 4 <= limit) blocks, = struct.unpack_from('>I', data, offset) offset += 4 - _require(blocks <= MAX_BLOCKS and blocks <= (limit - offset) // block_bytes) + _require(blocks <= MAX_BLOCKS and blocks <= (limit - offset) // BLOCK_BYTES) next_offset = header_length first_record = 0 total_ranges = 0 selected = [] for _ in range(blocks): - _require(offset + block_bytes <= limit) + _require(offset + BLOCK_BYTES <= limit) file_offset, length, count = BLOCK.unpack_from(data, offset) offset += BLOCK.size _require(file_offset == next_offset and 0 < length <= size - file_offset) @@ -433,23 +425,22 @@ def select(data, manifest, query, settings, partition_filter=None, partition_fie row_hit = ranges == 1 or query.intersects(start, end) offset += payload_length bucket_hit = True - if version >= 4: - _require(offset + 4 <= limit) - bucket_length, = struct.unpack_from('>i', data, offset) - offset += 4 - if bucket_length != -1: - _require(4 <= bucket_length <= limit - offset) - pairs, = struct.unpack_from('>I', data, offset) - _require(0 < pairs <= settings.max_bucket_pairs and bucket_length == 4 + 8 * pairs) - bucket_hit = bucket_filter is None - previous = (-1, -1) - for j in range(pairs): - bucket, total_buckets = struct.unpack_from('>ii', data, offset + 4 + 8 * j) - _require(0 <= bucket < total_buckets and (bucket, total_buckets) > previous) - previous = (bucket, total_buckets) - if not bucket_hit: - bucket_hit = bucket_filter(bucket, total_buckets) - offset += bucket_length + _require(offset + 4 <= limit) + bucket_length, = struct.unpack_from('>i', data, offset) + offset += 4 + if bucket_length != -1: + _require(4 <= bucket_length <= limit - offset) + pairs, = struct.unpack_from('>I', data, offset) + _require(0 < pairs <= settings.max_bucket_pairs and bucket_length == 4 + 8 * pairs) + bucket_hit = bucket_filter is None + previous = (-1, -1) + for j in range(pairs): + bucket, total_buckets = struct.unpack_from('>ii', data, offset + 4 + 8 * j) + _require(0 <= bucket < total_buckets and (bucket, total_buckets) > previous) + previous = (bucket, total_buckets) + if not bucket_hit: + bucket_hit = bucket_filter(bucket, total_buckets) + offset += bucket_length if partition_hit and row_hit and bucket_hit: selected.append(Block(file_offset, length, first_record, count)) next_offset = file_offset + length @@ -458,62 +449,6 @@ def select(data, manifest, query, settings, partition_filter=None, partition_fie return Selection(header, tuple(selected)) -def _select_legacy(data, manifest, query, settings): - if query is not None and not isinstance(query, Query): - query = Query(query) - _require(128 <= len(data) <= settings.max_bytes) - _require(hashlib.sha256(data[:-32]).digest() == data[-32:]) - magic, version, codec, flags, name_hash, size, entries, length = LEGACY_HEADER.unpack_from(data) - _require((magic, version, codec, flags) == (MAGIC, 2, 2, 1)) - _require(name_hash == hashlib.sha256(manifest.file_name.encode('utf-8')).digest()) - _require(size == manifest.file_size and entries == manifest.num_added_files + manifest.num_deleted_files) - _require(length == len(data) - LEGACY_HEADER.size - 32) - offset = LEGACY_HEADER.size - header_length, = struct.unpack_from('>I', data, offset) - offset += 4 - _require(21 <= header_length <= MAX_AVRO_HEADER and header_length <= len(data) - offset - 36) - header = bytes(data[offset:offset + header_length]) - _require(header[:4] == b'Obj\x01') - offset += header_length - blocks, = struct.unpack_from('>I', data, offset) - offset += 4 - _require(blocks <= (len(data) - 32 - offset) // 52) - next_offset = header_length - next_record = 0 - total_ranges = 0 - selected = [] - for _ in range(blocks): - file_offset, block_length, first, count, ranges = LEGACY_BLOCK.unpack_from(data, offset) - offset += LEGACY_BLOCK.size - _require(file_offset == next_offset and 0 < block_length <= size - file_offset) - _require(first == next_record and 0 < count <= entries - first) - _require(0 < ranges <= settings.max_ranges - total_ranges and ranges <= (len(data) - 32 - offset) // 16) - total_ranges += ranges - ranges_end = offset + PAIR.size * ranges - min_row_id, first_end = PAIR.unpack_from(data, offset) - offset += PAIR.size - # The sorted interval list already contains min/max; no format change or extra fields. - max_row_id = first_end if ranges == 1 else LONG.unpack_from(data, ranges_end - LONG.size)[0] - _require(min_row_id >= 0 and first_end >= min_row_id and max_row_id >= first_end) - candidate = query is None or query.intersects(min_row_id, max_row_id) - hit = candidate and (query is None or ranges == 1 or query.intersects(min_row_id, first_end)) - previous_end = first_end - for _ in range(1, ranges): - start, end = PAIR.unpack_from(data, offset) - offset += PAIR.size - # Retain validation even when min/max rejects the block or an earlier interval hit. - _require(start >= 0 and end >= start and start > previous_end) - previous_end = end - if candidate and not hit: - hit = query.intersects(start, end) - if hit: - selected.append(Block(file_offset, block_length, first, count)) - next_offset = file_offset + block_length - next_record = first + count - _require(offset == len(data) - 32 and next_offset == size and next_record == entries) - return Selection(header, tuple(selected)) - - def index_file_name(manifest): return next((name for name in manifest.extra_files or [] if name.endswith(SUFFIX)), None) diff --git a/paimon-python/pypaimon/tests/manifest/manifest_block_index_test.py b/paimon-python/pypaimon/tests/manifest/manifest_block_index_test.py index 4ce1411055f6..2d3e14d2e244 100644 --- a/paimon-python/pypaimon/tests/manifest/manifest_block_index_test.py +++ b/paimon-python/pypaimon/tests/manifest/manifest_block_index_test.py @@ -43,7 +43,7 @@ def part(p): def fixture(key): - path = Path(__file__).resolve().parents[4] / 'paimon-core/src/test/resources/manifest-row-id-index-v2.txt' + path = Path(__file__).resolve().parents[4] / 'paimon-core/src/test/resources/manifest-block-index.txt' value = next(line.split('=', 1)[1] for line in path.read_text().splitlines() if line.startswith(key + '=')) return base64.b64decode(value) @@ -66,7 +66,7 @@ def test_partition_dictionary_golden_tuples_nulls_and_derived_ordinals(): builder.add(first, count, p) builder.end_block() data = builder.serialize('manifest-golden', len(header) + 400, 7) - assert data == fixture('indexJointV4') + assert data == fixture('indexWithPartitions') predicate = part(7) with patch.object(predicate, 'test', wraps=predicate.test) as evaluated: selected = select(data, golden_meta(), [Range(20, 20)], Settings(), predicate, FIELDS) @@ -127,13 +127,10 @@ def positions(data): p = offset + 24 r = p + 5 + struct.unpack_from('>I', data, p + 1)[0] offset = r + 5 + struct.unpack_from('>I', data, r + 1)[0] - if struct.unpack_from('>I', data, 8)[0] >= 4: - bucket = offset - length, = struct.unpack_from('>i', data, offset) - offset += 4 + max(0, length) - result.append((block, p, r, bucket)) - else: - result.append((block, p, r)) + bucket = offset + length, = struct.unpack_from('>i', data, offset) + offset += 4 + max(0, length) + result.append((block, p, r, bucket)) return result @@ -143,8 +140,8 @@ def checksum(data): def test_unsigned_unknown_encodings_skip_only_one_payload_and_validate_lengths(): - good = fixture('indexJointV3') - block, p, r = positions(good)[0] + good = fixture('indexWithPartitions') + block, p, r, _ = positions(good)[0] data = bytearray(good) data[p] = 200 selected = select(checksum(data), golden_meta(), [Range(0, 0)], Settings(), part(99), FIELDS) @@ -224,7 +221,7 @@ def test_randomized_budget_degradation_has_no_false_negatives(): assert i * 6 in ordinals -def test_bucket_payload_golden_rescale_and_old_versions(): +def test_bucket_payload_golden_rescale_and_nullable_payloads(): a, b, header = partition(7, 'left'), partition(9, None), avro_header() builder = Builder(Settings(), header) for offset, length, values in [ @@ -236,7 +233,7 @@ def test_bucket_payload_golden_rescale_and_old_versions(): builder.add(*value) builder.end_block() data = builder.serialize('manifest-golden', len(header) + 400, 7) - assert data == fixture('indexBucketV4') + assert data == fixture('indexWithBuckets') selected = select(data, golden_meta(), None, Settings(), bucket_filter=lambda bucket, total: bucket == 1) assert [b.first_record for b in selected.blocks] == [0] selected = select(data, golden_meta(), None, Settings(), @@ -244,7 +241,7 @@ def test_bucket_payload_golden_rescale_and_old_versions(): assert [b.first_record for b in selected.blocks] == [3] assert not select(data, golden_meta(), [Range(0, 0)], Settings(), part(7), FIELDS, bucket_filter=lambda bucket, total: bucket == 2).blocks - for data in [golden(), fixture('indexJointV3'), fixture('indexJointV4')]: + for data in [golden(), fixture('indexWithPartitions')]: assert len(select(data, golden_meta(), None, Settings(), bucket_filter=lambda bucket, total: False).blocks) == 3 @@ -268,7 +265,7 @@ def test_bucket_budget_and_unknown_pairs_degrade_only_bucket_payload(pair): def test_malformed_bucket_payload_invalidates_the_container(): - good = fixture('indexBucketV4') + good = fixture('indexWithBuckets') payload = positions(good)[0][3] for offset, value in [(payload, -2), (payload, (1 << 31) - 1), (payload, 0), (payload + 4, 0), (payload + 8, -1), (payload + 12, 1), (payload + 16, 0)]: diff --git a/paimon-python/pypaimon/tests/manifest/row_id_index_test.py b/paimon-python/pypaimon/tests/manifest/row_id_index_test.py index 24c1ceeba3a6..d04dac145631 100644 --- a/paimon-python/pypaimon/tests/manifest/row_id_index_test.py +++ b/paimon-python/pypaimon/tests/manifest/row_id_index_test.py @@ -50,9 +50,9 @@ def fixture(): path = (Path(__file__).resolve().parents[4] / 'paimon-core/src/test/resources' / - 'manifest-row-id-index-v2.txt') + 'manifest-block-index.txt') return dict(line.split('=', 1) for line in path.read_text().splitlines() - if line.startswith(('index=', 'indexV3=', 'indexV4=', 'avroHeader='))) + if line.startswith(('index=', 'avroHeader='))) def golden(): @@ -237,7 +237,7 @@ def test_cross_language_and_block_ordinals(self): b.add(first, count) b.end_block() self.assertEqual(b.serialize(meta.file_name, meta.file_size, 7), - base64.b64decode(fixture()['indexV4'])) + golden()) def test_minmax_skips_exact_checks_and_one_interval_is_already_exact(self): header = avro_header() @@ -265,7 +265,7 @@ def test_minmax_skips_exact_checks_and_one_interval_is_already_exact(self): def test_rejected_and_early_hit_blocks_still_validate_every_interval(self): data = bytearray(golden()) - first_block_intervals = 68 + 4 + len(avro_header()) + 4 + 36 + first_block_intervals = 60 + 4 + len(avro_header()) + 4 + 4 + 24 + 5 + 5 + 4 struct.pack_into('>q', data, first_block_intervals + 16, 9) data[-32:] = hashlib.sha256(data[:-32]).digest() for point in (30, 0): @@ -308,9 +308,9 @@ def test_invalid_envelopes(self): bad[index] ^= 2 with self.assertRaises(ValueError): select(bad, meta, [Range(10, 10)], Settings()) - for index in (9, 11, 15): + for version in (0, 2, 99): bad = bytearray(data[:-32]) - bad[index] = 0 + struct.pack_into('>I', bad, 8, version) bad.extend(hashlib.sha256(bad).digest()) with self.assertRaises(ValueError): select(bad, meta, [Range(10, 10)], Settings()) From bc8f465bf0eeb8577202d0c44bc962735515d85a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Mon, 14 Sep 2026 11:27:58 +0800 Subject: [PATCH 08/23] [core] Rename manifest block indexes to sidecars --- docs/docs/concepts/spec/manifest.md | 22 ++-- .../java/org/apache/paimon/CoreOptions.java | 40 +++--- .../org/apache/paimon/AbstractFileStore.java | 2 +- .../paimon/manifest/ManifestAvroWriter.java | 28 ++--- .../apache/paimon/manifest/ManifestFile.java | 40 +++--- ...stRowIdIndex.java => ManifestSidecar.java} | 48 +++---- .../operation/AbstractFileStoreScan.java | 4 +- .../manifest/ManifestBlockIndexTest.java | 101 +++++++-------- .../ManifestFileMetaSerializerTest.java | 2 +- .../paimon/manifest/ManifestFileTest.java | 68 +++++----- .../manifest/ManifestIndexTestUtils.java | 2 +- ...ndexTest.java => ManifestSidecarTest.java} | 119 +++++++++--------- .../paimon/operation/ExpireSnapshotsTest.java | 6 +- .../operation/LocalOrphanFilesCleanTest.java | 8 +- ...t-block-index.txt => manifest-sidecar.txt} | 6 +- .../pypaimon/common/options/core_options.py | 28 ++--- .../manifest/manifest_file_manager.py | 23 ++-- .../{row_id_index.py => manifest_sidecar.py} | 44 +++---- .../manifest/manifest_block_index_test.py | 6 +- ...index_test.py => manifest_sidecar_test.py} | 76 +++++------ 20 files changed, 326 insertions(+), 347 deletions(-) rename paimon-core/src/main/java/org/apache/paimon/manifest/{ManifestRowIdIndex.java => ManifestSidecar.java} (96%) rename paimon-core/src/test/java/org/apache/paimon/manifest/{ManifestRowIdIndexTest.java => ManifestSidecarTest.java} (86%) rename paimon-core/src/test/resources/{manifest-block-index.txt => manifest-sidecar.txt} (84%) rename paimon-python/pypaimon/manifest/{row_id_index.py => manifest_sidecar.py} (93%) rename paimon-python/pypaimon/tests/manifest/{row_id_index_test.py => manifest_sidecar_test.py} (93%) diff --git a/docs/docs/concepts/spec/manifest.md b/docs/docs/concepts/spec/manifest.md index 9120480c3f0c..f8467a8349fa 100644 --- a/docs/docs/concepts/spec/manifest.md +++ b/docs/docs/concepts/spec/manifest.md @@ -63,15 +63,15 @@ skip manifests before opening them. Each extra file belongs exclusively to one manifest. It is retained and cleaned up together with that manifest during snapshot, tag, or changelog deletion. -### Partition, Row-ID and Bucket Block Index +### Manifest Sidecar -With `manifest.row-id-index.write` enabled, a manifest writer can create a binary -`.row-id-index` sidecar. Its name is stored in the manifest-list +With `manifest.sidecar.write` enabled, a manifest writer can create a binary +`.avro.sidecar` sidecar. Its name is stored in the manifest-list record's `_EXTRA_FILES`; the existing Avro schemas and `_VERSION` identifiers are unchanged. -Readers identify the block index by the `.row-id-index` suffix among these explicit +Readers identify the sidecar by the `.avro.sidecar` suffix among these explicit references, not by probing for a derived file name. Other extra-file references are preserved. -With `manifest.row-id-index.read` enabled and a partition, row-ID or bucket filter available, readers can use +With `manifest.sidecar.read` enabled and a partition, row-ID or bucket filter available, readers can use the sidecar to select complete Avro blocks before reading manifest entries. Both options default to `false`. Old manifests, null or empty extra-file lists, and lists containing only other extra-file types use the normal manifest read path. Missing, unsupported, corrupt, @@ -83,7 +83,7 @@ Version 1 uses the following layout. Container integers and payload integers are fixed-width big endian. Encoding IDs are unsigned bytes with separate namespaces. ```text -magic : 8 bytes // ASCII PAIMRIDX +magic : 8 bytes // ASCII PAIMSCAR formatVersion : int // 1 manifestNameHash : 32 bytes // SHA-256 of the UTF-8 basename manifestLength : long @@ -151,11 +151,11 @@ Partition budget exhaustion independently makes that block's partition payload u The dictionary can consequently be incomplete for the manifest: a dictionary miss never excludes a block with unavailable partition coverage. Later blocks can still use existing IDs. -`manifest.row-id-index.max-ranges` bounds stored intervals (default 131072). -`manifest.index.max-partitions` bounds dictionary entries (default 65536), and -`manifest.index.max-partition-bytes` bounds dictionary bytes including length fields -(default 1048576). `manifest.index.max-bucket-pairs` bounds distinct bucket/count pairs -per block (default 4096). `manifest.row-id-index.max-bytes` bounds the whole serialized container +`manifest.sidecar.max-ranges` bounds stored intervals (default 131072). +`manifest.sidecar.max-partitions` bounds dictionary entries (default 65536), and +`manifest.sidecar.max-partition-bytes` bounds dictionary bytes including length fields +(default 1048576). `manifest.sidecar.max-bucket-pairs` bounds distinct bucket/count pairs +per block (default 4096). `manifest.sidecar.max-bytes` bounds the whole serialized container (default 8388608); the Avro header is also capped at 1 MiB and the directory at 131072 blocks. Writers discard optional row-ID payloads, bucket payloads, then partition payloads/dictionary if necessary, to fit the complete directory. If the directory itself cannot fit, no sidecar is published. diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java index b539e3e478aa..dba38a078a9b 100644 --- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java @@ -510,54 +510,54 @@ public InlineElement getDescription() { + "in the previous file. This must not exceed " + "'variant.shredding.minFieldCardinalityRatio'."); - public static final ConfigOption MANIFEST_ROW_ID_INDEX_WRITE = - key("manifest.row-id-index.write") + public static final ConfigOption MANIFEST_SIDECAR_WRITE = + key("manifest.sidecar.write") .booleanType() .defaultValue(false) .withDescription( - "Write block indexes with independent partition, row-id and bucket coverage for newly created manifests."); + "Write sidecars with independent partition, row-id and bucket coverage for newly created manifests."); - public static final ConfigOption MANIFEST_ROW_ID_INDEX_READ = - key("manifest.row-id-index.read") + public static final ConfigOption MANIFEST_SIDECAR_READ = + key("manifest.sidecar.read") .booleanType() .defaultValue(false) .withDescription( - "Read optional manifest block indexes for partition, row-id or bucket filters after coarse pruning. Missing or invalid indexes fall back to manifest reads."); + "Read optional manifest sidecars for partition, row-id or bucket filters after coarse pruning. Missing or invalid sidecars fall back to manifest reads."); - public static final ConfigOption MANIFEST_ROW_ID_INDEX_MAX_RANGES = - key("manifest.row-id-index.max-ranges") + public static final ConfigOption MANIFEST_SIDECAR_MAX_RANGES = + key("manifest.sidecar.max-ranges") .intType() .defaultValue(131072) .withDescription( "Maximum disjoint row-id intervals across all Avro blocks in a manifest. On exhaustion, coarsen coverage to min/max or mark row-id coverage unavailable. Range: 1 to 1048576."); - public static final ConfigOption MANIFEST_INDEX_MAX_PARTITIONS = - key("manifest.index.max-partitions") + public static final ConfigOption MANIFEST_SIDECAR_MAX_PARTITIONS = + key("manifest.sidecar.max-partitions") .intType() .defaultValue(65536) .withDescription( - "Maximum partition dictionary entries per manifest block index. Further unknown partitions disable partition coverage only for their blocks."); + "Maximum partition dictionary entries per manifest sidecar. Further unknown partitions disable partition coverage only for their blocks."); - public static final ConfigOption MANIFEST_INDEX_MAX_BUCKET_PAIRS = - key("manifest.index.max-bucket-pairs") + public static final ConfigOption MANIFEST_SIDECAR_MAX_BUCKET_PAIRS = + key("manifest.sidecar.max-bucket-pairs") .intType() .defaultValue(4096) .withDescription( - "Maximum distinct bucket and total-bucket pairs per block. Exceeding this budget disables bucket coverage for the block, preserving other index payloads."); + "Maximum distinct bucket and total-bucket pairs per block. Exceeding this budget disables bucket coverage for the block, preserving other sidecar payloads."); - public static final ConfigOption MANIFEST_INDEX_MAX_PARTITION_BYTES = - key("manifest.index.max-partition-bytes") + public static final ConfigOption MANIFEST_SIDECAR_MAX_PARTITION_BYTES = + key("manifest.sidecar.max-partition-bytes") .intType() .defaultValue(1048576) .withDescription( - "Maximum serialized partition dictionary bytes per manifest block index. Exceeding this budget preserves independently available row-id coverage."); + "Maximum serialized partition dictionary bytes per manifest sidecar. Exceeding this budget preserves independently available row-id coverage."); - public static final ConfigOption MANIFEST_ROW_ID_INDEX_MAX_BYTES = - key("manifest.row-id-index.max-bytes") + public static final ConfigOption MANIFEST_SIDECAR_MAX_BYTES = + key("manifest.sidecar.max-bytes") .intType() .defaultValue(8388608) .withDescription( - "Maximum serialized manifest block index bytes, including header and checksum. Optional payloads are dropped before omitting an index whose complete block directory cannot fit. Range: 128 to 67108864."); + "Maximum serialized manifest sidecar bytes, including header and checksum. Optional payloads are dropped before omitting a sidecar whose complete block directory cannot fit. Range: 128 to 67108864."); public static final ConfigOption MANIFEST_COMPRESSION = key("manifest.compression") diff --git a/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java b/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java index df0cccb7d9bd..41ded1344874 100644 --- a/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java +++ b/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java @@ -212,7 +212,7 @@ public ManifestFile.Factory manifestFileFactory() { pathFactory(), options.manifestTargetSize().getBytes(), readManifestCache) - .withRowIdIndexOptions(options.toConfiguration()); + .withSidecarOptions(options.toConfiguration()); } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestAvroWriter.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestAvroWriter.java index 5425ab132673..182989326a5d 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestAvroWriter.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestAvroWriter.java @@ -69,7 +69,7 @@ public final class ManifestAvroWriter implements AutoCloseable { private final String compression; private final PathFactory pathFactory; private final long targetFileSize; - private final ManifestRowIdIndex.Settings rowIdIndexSettings; + private final ManifestSidecar.Settings sidecarSettings; private final List results = new ArrayList<>(); private final List completedPaths = new ArrayList<>(); @@ -86,7 +86,7 @@ public final class ManifestAvroWriter implements AutoCloseable { String compression, PathFactory pathFactory, long targetFileSize, - ManifestRowIdIndex.Settings rowIdIndexSettings) { + ManifestSidecar.Settings sidecarSettings) { this.fileIO = fileIO; this.schemaManager = schemaManager; this.partitionType = partitionType; @@ -95,7 +95,7 @@ public final class ManifestAvroWriter implements AutoCloseable { this.compression = compression; this.pathFactory = pathFactory; this.targetFileSize = targetFileSize; - this.rowIdIndexSettings = rowIdIndexSettings; + this.sidecarSettings = sidecarSettings; } public void write(ManifestEntry entry) throws IOException { @@ -223,7 +223,7 @@ private void closeCurrentWriter() throws IOException { ManifestFileMeta result = currentWriter.result(); completedPaths.add(currentWriter.path); if (currentWriter.sidecarCreated) { - completedPaths.add(ManifestRowIdIndex.path(currentWriter.path)); + completedPaths.add(ManifestSidecar.path(currentWriter.path)); } results.add(result); currentWriter = null; @@ -707,7 +707,7 @@ private Throwable abortCollecting(@Nullable Throwable primaryFailure, boolean de } if (sidecarCreated) { try { - fileIO.deleteQuietly(ManifestRowIdIndex.path(path)); + fileIO.deleteQuietly(ManifestSidecar.path(path)); } catch (Throwable cleanupFailure) { primaryFailure = ExceptionUtils.firstOrSuppressed(cleanupFailure, primaryFailure); @@ -727,7 +727,7 @@ private void close() throws IOException { outputBytes = out.getPos(); out.close(); out = null; - writeRowIdIndex(); + writeSidecar(); } catch (IOException | RuntimeException | Error failure) { abortCollecting(failure, true); throw failure; @@ -736,23 +736,23 @@ private void close() throws IOException { } } - private void writeRowIdIndex() throws IOException { - if (!rowIdIndexSettings.write) { + private void writeSidecar() throws IOException { + if (!sidecarSettings.write) { return; } byte[] bytes = - ManifestRowIdIndex.build( + ManifestSidecar.build( fileIO, path, outputBytes, Math.addExact(numAddedFiles, numDeletedFiles), - rowIdIndexSettings); + sidecarSettings); if (bytes != null) { // Publish result() only after both immutable objects have closed. No rename. - try (PositionOutputStream indexOut = - fileIO.newOutputStream(ManifestRowIdIndex.path(path), false)) { + try (PositionOutputStream sidecarOut = + fileIO.newOutputStream(ManifestSidecar.path(path), false)) { sidecarCreated = true; - indexOut.write(bytes); + sidecarOut.write(bytes); } } } @@ -779,7 +779,7 @@ private ManifestFileMeta result() { rowIdStats == null ? null : rowIdStats.maxRowId, totalBucketsKnown ? totalBuckets : null, sidecarCreated - ? Collections.singletonList(ManifestRowIdIndex.path(path).getName()) + ? Collections.singletonList(ManifestSidecar.path(path).getName()) : null); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java index a1c4774ba886..94498fc0fa40 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java @@ -61,7 +61,7 @@ public class ManifestFile extends ObjectsFile { private final RowType partitionType; private final AvroFileFormat avroFileFormat; private final long suggestedFileSize; - private final ManifestRowIdIndex.Settings rowIdIndexSettings; + private final ManifestSidecar.Settings sidecarSettings; private ManifestFile( FileIO fileIO, @@ -73,7 +73,7 @@ private ManifestFile( PathFactory pathFactory, long suggestedFileSize, @Nullable SegmentsCache cache, - ManifestRowIdIndex.Settings rowIdIndexSettings) { + ManifestSidecar.Settings sidecarSettings) { super( fileIO, serializer, @@ -89,7 +89,7 @@ private ManifestFile( this.partitionType = partitionType; this.avroFileFormat = avroFileFormat; this.suggestedFileSize = suggestedFileSize; - this.rowIdIndexSettings = rowIdIndexSettings; + this.sidecarSettings = sidecarSettings; } @Override @@ -160,7 +160,7 @@ public List read( Filter readFilter, Filter readTFilter, Function convertor, - @Nullable ManifestRowIdIndex.Selection selected) { + @Nullable ManifestSidecar.Selection selected) { if (selected != null && selected.blocks().isEmpty()) { return java.util.Collections.emptyList(); } @@ -245,11 +245,11 @@ private static CloseableIterator createManifestIterator( RowType projectedType, @Nullable PartitionPredicate partitionFilter, @Nullable BucketFilter bucketFilter, - @Nullable ManifestRowIdIndex.Selection selected) + @Nullable ManifestSidecar.Selection selected) throws IOException { try { ManifestAvroReader reader = - new ManifestAvroReader(ManifestRowIdIndex.openManifest(fileIO, path, selected)); + new ManifestAvroReader(ManifestSidecar.openManifest(fileIO, path, selected)); return reader.read(projectedType, partitionFilter, bucketFilter); } catch (IOException e) { FileUtils.checkExists(fileIO, path); @@ -345,7 +345,7 @@ public ManifestAvroWriter createAvroWriter() { compression, pathFactory, suggestedFileSize, - rowIdIndexSettings); + sidecarSettings); } /** Creates an Avro manifest writer for one explicit path. */ @@ -359,7 +359,7 @@ public ManifestAvroWriter createAvroWriter(Path manifestPath) { compression, singlePathFactory(manifestPath), Long.MAX_VALUE, - rowIdIndexSettings); + sidecarSettings); } private PathFactory singlePathFactory(Path manifestPath) { @@ -385,13 +385,13 @@ public Path toPath(String fileName) { } @Nullable - public ManifestRowIdIndex.Selection selectBlocks( + public ManifestSidecar.Selection selectBlocks( ManifestFileMeta manifest, @Nullable RowRangeIndex query) { return selectBlocks(manifest, query, null); } @Nullable - public ManifestRowIdIndex.Selection selectBlocks( + public ManifestSidecar.Selection selectBlocks( ManifestFileMeta manifest, @Nullable RowRangeIndex query, @Nullable PartitionPredicate partitionFilter) { @@ -399,15 +399,15 @@ public ManifestRowIdIndex.Selection selectBlocks( } @Nullable - public ManifestRowIdIndex.Selection selectBlocks( + public ManifestSidecar.Selection selectBlocks( ManifestFileMeta manifest, @Nullable RowRangeIndex query, @Nullable PartitionPredicate partitionFilter, @Nullable BucketFilter bucketFilter) { - return !rowIdIndexSettings.read + return !sidecarSettings.read || (query == null && partitionFilter == null && bucketFilter == null) ? null - : ManifestRowIdIndex.read( + : ManifestSidecar.read( fileIO, pathFactory.toPath(manifest.fileName()), manifest, @@ -415,11 +415,11 @@ public ManifestRowIdIndex.Selection selectBlocks( partitionFilter, partitionType, bucketFilter, - rowIdIndexSettings); + sidecarSettings); } public boolean mayContainRowIds(ManifestFileMeta manifest, @Nullable RowRangeIndex query) { - ManifestRowIdIndex.Selection selected = selectBlocks(manifest, query); + ManifestSidecar.Selection selected = selectBlocks(manifest, query); return selected == null || !selected.blocks().isEmpty(); } @@ -441,8 +441,8 @@ public static class Factory { private final String compression; private final FileStorePathFactory pathFactory; private final long suggestedFileSize; - private ManifestRowIdIndex.Settings rowIdIndexSettings = - new ManifestRowIdIndex.Settings(new Options()); + private ManifestSidecar.Settings sidecarSettings = + new ManifestSidecar.Settings(new Options()); @Nullable private final SegmentsCache cache; public Factory( @@ -464,8 +464,8 @@ public Factory( this.cache = cache; } - public Factory withRowIdIndexOptions(Options options) { - rowIdIndexSettings = new ManifestRowIdIndex.Settings(options); + public Factory withSidecarOptions(Options options) { + sidecarSettings = new ManifestSidecar.Settings(options); return this; } @@ -484,7 +484,7 @@ public ManifestFile create() { pathFactory.manifestFileFactory(), suggestedFileSize, cache, - rowIdIndexSettings); + sidecarSettings); } } } diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestRowIdIndex.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java similarity index 96% rename from paimon-core/src/main/java/org/apache/paimon/manifest/ManifestRowIdIndex.java rename to paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java index 9f1289ee26a1..8bb02b5cb000 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestRowIdIndex.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java @@ -62,10 +62,10 @@ import static org.apache.paimon.utils.Preconditions.checkArgument; /** Independently usable partition, row-id and bucket coverage for each manifest block. */ -public final class ManifestRowIdIndex { - public static final String SUFFIX = ".row-id-index"; - private static final Logger LOG = LoggerFactory.getLogger(ManifestRowIdIndex.class); - private static final long MAGIC = 0x5041494d52494458L; +public final class ManifestSidecar { + public static final String SUFFIX = ".avro.sidecar"; + private static final Logger LOG = LoggerFactory.getLogger(ManifestSidecar.class); + private static final long MAGIC = 0x5041494d53434152L; private static final int FORMAT_VERSION = 1; private static final int HEADER_BYTES = 60; private static final int BLOCK_BYTES = 38; @@ -75,7 +75,7 @@ public final class ManifestRowIdIndex { private static final int MAX_AVRO_HEADER = 1024 * 1024; private static final int READ_BUFFER_BYTES = 1024 * 1024; - private ManifestRowIdIndex() {} + private ManifestSidecar() {} public static Path path(Path manifest) { return new Path(manifest.toString() + SUFFIX); @@ -104,28 +104,28 @@ public static final class Settings { public final int maxBucketPairs; public Settings(Options options) { - write = options.get(CoreOptions.MANIFEST_ROW_ID_INDEX_WRITE); - read = options.get(CoreOptions.MANIFEST_ROW_ID_INDEX_READ); - maxRanges = options.get(CoreOptions.MANIFEST_ROW_ID_INDEX_MAX_RANGES); - maxBytes = options.get(CoreOptions.MANIFEST_ROW_ID_INDEX_MAX_BYTES); - maxPartitions = options.get(CoreOptions.MANIFEST_INDEX_MAX_PARTITIONS); - maxPartitionBytes = options.get(CoreOptions.MANIFEST_INDEX_MAX_PARTITION_BYTES); - maxBucketPairs = options.get(CoreOptions.MANIFEST_INDEX_MAX_BUCKET_PAIRS); + write = options.get(CoreOptions.MANIFEST_SIDECAR_WRITE); + read = options.get(CoreOptions.MANIFEST_SIDECAR_READ); + maxRanges = options.get(CoreOptions.MANIFEST_SIDECAR_MAX_RANGES); + maxBytes = options.get(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES); + maxPartitions = options.get(CoreOptions.MANIFEST_SIDECAR_MAX_PARTITIONS); + maxPartitionBytes = options.get(CoreOptions.MANIFEST_SIDECAR_MAX_PARTITION_BYTES); + maxBucketPairs = options.get(CoreOptions.MANIFEST_SIDECAR_MAX_BUCKET_PAIRS); checkArgument( maxBucketPairs > 0 && maxBucketPairs <= 1048576, - "Invalid manifest.index.max-bucket-pairs"); + "Invalid manifest.sidecar.max-bucket-pairs"); checkArgument( maxPartitions > 0 && maxPartitions <= 1048576, - "Invalid manifest.index.max-partitions"); + "Invalid manifest.sidecar.max-partitions"); checkArgument( maxPartitionBytes > 0 && maxPartitionBytes <= 64 * 1024 * 1024, - "Invalid manifest.index.max-partition-bytes"); + "Invalid manifest.sidecar.max-partition-bytes"); checkArgument( maxRanges > 0 && maxRanges <= 1048576, - "manifest.row-id-index.max-ranges must be in [1, 1048576]"); + "manifest.sidecar.max-ranges must be in [1, 1048576]"); checkArgument( maxBytes >= 128 && maxBytes <= 64 * 1024 * 1024, - "manifest.row-id-index.max-bytes must be in [128, 67108864]"); + "manifest.sidecar.max-bytes must be in [128, 67108864]"); } } @@ -710,7 +710,7 @@ private static ByteBuffer payload(ByteBuffer in) throws IOException { return result; } - /** Bounded, bulk index reads. Null means read the original manifest. */ + /** Bounded, bulk sidecar reads. Null means read the original manifest. */ @Nullable public static Selection read( FileIO io, @@ -743,13 +743,13 @@ public static Selection read( @Nullable RowType partitionType, @Nullable BucketFilter bucketFilter, Settings settings) { - String indexFileName = fileName(manifest); - if (indexFileName == null) { + String sidecarFileName = fileName(manifest); + if (sidecarFileName == null) { return null; } try { byte[] data; - try (InputStream in = io.newInputStream(new Path(path.getParent(), indexFileName))) { + try (InputStream in = io.newInputStream(new Path(path.getParent(), sidecarFileName))) { ByteArrayOutputStream out = new ByteArrayOutputStream(); byte[] buffer = new byte[Math.min(READ_BUFFER_BYTES, settings.maxBytes + 1)]; int n; @@ -799,14 +799,14 @@ public static Selection read( if (Thread.currentThread().isInterrupted()) { throw interrupted(failure); } - LOG.debug("Cannot use row-id block index for {}; reading manifest", path, failure); + LOG.debug("Cannot use manifest sidecar for {}; reading manifest", path, failure); return null; } } private static UncheckedIOException interrupted(Throwable failure) { InterruptedIOException interrupted = - new InterruptedIOException("Interrupted reading row-id index"); + new InterruptedIOException("Interrupted reading manifest sidecar"); interrupted.initCause(failure); return new UncheckedIOException(interrupted); } @@ -904,7 +904,7 @@ public void close() throws IOException { private static void require(boolean valid) throws IOException { if (!valid) { throw new IOException( - "Invalid, unsupported, mismatched or over-budget manifest row-id block index"); + "Invalid, unsupported, mismatched or over-budget manifest sidecar"); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreScan.java b/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreScan.java index b8989021678f..e40d27e789b8 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreScan.java @@ -29,7 +29,7 @@ import org.apache.paimon.manifest.ManifestEntrySerializer; import org.apache.paimon.manifest.ManifestFile; import org.apache.paimon.manifest.ManifestFileMeta; -import org.apache.paimon.manifest.ManifestRowIdIndex; +import org.apache.paimon.manifest.ManifestSidecar; import org.apache.paimon.manifest.PartitionEntry; import org.apache.paimon.manifest.SimpleFileEntry; import org.apache.paimon.operation.metrics.ScanMetrics; @@ -501,7 +501,7 @@ private List readManifest( ManifestFile manifestFile = manifestFileFactory.create(); BucketFilter bucketFilter = createBucketFilter(); - ManifestRowIdIndex.Selection selected = + ManifestSidecar.Selection selected = manifestFile.selectBlocks( manifest, rowRangeIndex, manifestsReader.partitionFilter(), bucketFilter); if (selected != null && selected.blocks().isEmpty()) { diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java index 1aabacee89c5..02a71a7a37ee 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java @@ -43,7 +43,7 @@ import java.util.List; import java.util.Properties; -import static org.apache.paimon.manifest.ManifestRowIdIndexTest.meta; +import static org.apache.paimon.manifest.ManifestSidecarTest.meta; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; @@ -54,12 +54,11 @@ /** Independent partition/row-ID payloads and conservative resource degradation. */ class ManifestBlockIndexTest { private final RowType type = RowType.of(DataTypes.INT(), DataTypes.STRING()); - private final ManifestRowIdIndex.Settings defaults = - new ManifestRowIdIndex.Settings(new Options()); + private final ManifestSidecar.Settings defaults = new ManifestSidecar.Settings(new Options()); private byte[] fixture(String field) throws IOException { Properties p = new Properties(); - try (java.io.InputStream in = getClass().getResourceAsStream("/manifest-block-index.txt")) { + try (java.io.InputStream in = getClass().getResourceAsStream("/manifest-sidecar.txt")) { p.load(in); } return Base64.getDecoder().decode(p.getProperty(field)); @@ -93,7 +92,7 @@ void jointGoldenPreservesTuplesNullsAndDerivedOrdinals() throws Exception { assertThat(a).isEqualTo(fixture("partitionA")); assertThat(b).isEqualTo(fixture("partitionB")); byte[] header = fixture("avroHeader"); - ManifestRowIdIndex.Builder builder = new ManifestRowIdIndex.Builder(defaults, header); + ManifestSidecar.Builder builder = new ManifestSidecar.Builder(defaults, header); builder.beginBlock(header.length, 100, 3); builder.add(0L, 10, a); builder.add(5L, 5, a); @@ -111,21 +110,19 @@ void jointGoldenPreservesTuplesNullsAndDerivedOrdinals() throws Exception { assertThat(data).isEqualTo(fixture("indexWithPartitions")); ManifestFileMeta meta = meta("manifest-golden", header.length + 400, 7); PartitionPredicate filter = spy(part(7)); - assertThat( - ManifestRowIdIndex.select(data, meta, query(20), filter, type, defaults) - .blocks()) + assertThat(ManifestSidecar.select(data, meta, query(20), filter, type, defaults).blocks()) .extracting(block -> block.firstRecord) .containsExactly(0L, 5L); verify(filter, times(2)).test(any(BinaryRow.class)); PartitionPredicate nullFilter = PartitionPredicate.fromPredicate(type, new PredicateBuilder(type).isNull(1)); - assertThat(ManifestRowIdIndex.select(data, meta, null, nullFilter, type, defaults).blocks()) + assertThat(ManifestSidecar.select(data, meta, null, nullFilter, type, defaults).blocks()) .hasSize(3); - assertThat(ManifestRowIdIndex.select(data, meta, null, part(99), type, defaults).blocks()) + assertThat(ManifestSidecar.select(data, meta, null, part(99), type, defaults).blocks()) .isEmpty(); // Missing partition payloads cannot be pruned by dictionary misses. assertThat( - ManifestRowIdIndex.select( + ManifestSidecar.select( fixture("index"), meta, null, part(99), type, defaults) .blocks()) .hasSize(3); @@ -134,10 +131,10 @@ void jointGoldenPreservesTuplesNullsAndDerivedOrdinals() throws Exception { @Test void unavailableDimensionsAreIndependentAndDoNotPoisonLaterBlocks() throws Exception { Options options = new Options(); - options.set(CoreOptions.MANIFEST_INDEX_MAX_PARTITIONS, 1); - ManifestRowIdIndex.Settings settings = new ManifestRowIdIndex.Settings(options); + options.set(CoreOptions.MANIFEST_SIDECAR_MAX_PARTITIONS, 1); + ManifestSidecar.Settings settings = new ManifestSidecar.Settings(options); byte[] header = fixture("avroHeader"); - ManifestRowIdIndex.Builder builder = new ManifestRowIdIndex.Builder(settings, header); + ManifestSidecar.Builder builder = new ManifestSidecar.Builder(settings, header); builder.beginBlock(header.length, 100, 1); builder.add(null, 10, partition(7, "left")); builder.endBlock(); @@ -149,17 +146,13 @@ void unavailableDimensionsAreIndependentAndDoNotPoisonLaterBlocks() throws Excep builder.endBlock(); byte[] data = builder.serialize("m", header.length + 300, 3); ManifestFileMeta meta = meta("m", header.length + 300, 3); - assertThat(ManifestRowIdIndex.select(data, meta, null, part(9), type, settings).blocks()) + assertThat(ManifestSidecar.select(data, meta, null, part(9), type, settings).blocks()) .extracting(block -> block.firstRecord) .containsExactly(1L); - assertThat( - ManifestRowIdIndex.select(data, meta, query(999), part(7), type, settings) - .blocks()) + assertThat(ManifestSidecar.select(data, meta, query(999), part(7), type, settings).blocks()) .extracting(block -> block.firstRecord) .containsExactly(0L); - assertThat( - ManifestRowIdIndex.select(data, meta, query(200), part(9), type, settings) - .blocks()) + assertThat(ManifestSidecar.select(data, meta, query(200), part(9), type, settings).blocks()) .extracting(block -> block.firstRecord) .containsExactly(1L); } @@ -167,11 +160,11 @@ void unavailableDimensionsAreIndependentAndDoNotPoisonLaterBlocks() throws Excep @Test void coarseningContinuesThroughTheEntireBlockAndDetectsUnknownRows() throws Exception { Options options = new Options(); - options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_MAX_RANGES, 1); - ManifestRowIdIndex.Settings settings = new ManifestRowIdIndex.Settings(options); + options.set(CoreOptions.MANIFEST_SIDECAR_MAX_RANGES, 1); + ManifestSidecar.Settings settings = new ManifestSidecar.Settings(options); byte[] header = fixture("avroHeader"); for (boolean unknown : new boolean[] {false, true}) { - ManifestRowIdIndex.Builder builder = new ManifestRowIdIndex.Builder(settings, header); + ManifestSidecar.Builder builder = new ManifestSidecar.Builder(settings, header); builder.beginBlock(header.length, 100, 4); builder.add(100L, 10, partition(7, "left")); builder.add(300L, 10, partition(7, "left")); @@ -181,14 +174,12 @@ void coarseningContinuesThroughTheEntireBlockAndDetectsUnknownRows() throws Exce byte[] data = builder.serialize("m", header.length + 100, 4); ManifestFileMeta meta = meta("m", header.length + 100, 4); for (long point : new long[] {10, 100, 200, Long.MAX_VALUE}) { - assertThat(ManifestRowIdIndex.select(data, meta, query(point), settings).blocks()) + assertThat(ManifestSidecar.select(data, meta, query(point), settings).blocks()) .hasSize(1); } - assertThat(ManifestRowIdIndex.select(data, meta, query(0), settings).blocks()) + assertThat(ManifestSidecar.select(data, meta, query(0), settings).blocks()) .hasSize(unknown ? 1 : 0); - assertThat( - ManifestRowIdIndex.select(data, meta, null, part(9), type, settings) - .blocks()) + assertThat(ManifestSidecar.select(data, meta, null, part(9), type, settings).blocks()) .isEmpty(); } } @@ -241,7 +232,7 @@ void unknownUnsignedEncodingsSkipOnlyTheirDimensionAndMalformedPayloadsFail() th byte[] data = good.clone(); data[first[1]] = (byte) 200; assertThat( - ManifestRowIdIndex.select( + ManifestSidecar.select( checksum(data), meta, query(0), part(99), type, defaults) .blocks()) .extracting(block -> block.firstRecord) @@ -249,7 +240,7 @@ void unknownUnsignedEncodingsSkipOnlyTheirDimensionAndMalformedPayloadsFail() th data = good.clone(); data[first[2]] = (byte) 201; assertThat( - ManifestRowIdIndex.select( + ManifestSidecar.select( checksum(data), meta, query(16), part(7), type, defaults) .blocks()) .extracting(block -> block.firstRecord) @@ -258,22 +249,20 @@ void unknownUnsignedEncodingsSkipOnlyTheirDimensionAndMalformedPayloadsFail() th byte[] bad = good.clone(); bad[position] = 0; // encoding 0 cannot have payload bytes checksum(bad); - assertThatThrownBy(() -> ManifestRowIdIndex.select(bad, meta, query(0), defaults)) + assertThatThrownBy(() -> ManifestSidecar.select(bad, meta, query(0), defaults)) .isInstanceOf(IOException.class); byte[] invalidLength = good.clone(); ByteBuffer.wrap(invalidLength).putInt(position + 1, -1); checksum(invalidLength); assertThatThrownBy( - () -> - ManifestRowIdIndex.select( - invalidLength, meta, query(0), defaults)) + () -> ManifestSidecar.select(invalidLength, meta, query(0), defaults)) .isInstanceOf(IOException.class); } // A checksummed directory with missing bytes/entries must still be rejected. byte[] bad = good.clone(); ByteBuffer.wrap(bad).putLong(first[0] + 16, 2); checksum(bad); - assertThatThrownBy(() -> ManifestRowIdIndex.select(bad, meta, query(0), defaults)) + assertThatThrownBy(() -> ManifestSidecar.select(bad, meta, query(0), defaults)) .isInstanceOf(IOException.class); byte[] badRange = good.clone(); // Row payload begins after its encoding and length, then the range-count integer. @@ -281,13 +270,13 @@ invalidLength, meta, query(0), defaults)) checksum(badRange); assertThatThrownBy( () -> - ManifestRowIdIndex.select( + ManifestSidecar.select( badRange, meta, query(999), part(99), type, defaults)) .isInstanceOf(IOException.class); byte[] badId = good.clone(); ByteBuffer.wrap(badId).putInt(first[1] + 9, 999); checksum(badId); - assertThatThrownBy(() -> ManifestRowIdIndex.select(badId, meta, query(999), defaults)) + assertThatThrownBy(() -> ManifestSidecar.select(badId, meta, query(999), defaults)) .isInstanceOf(IOException.class); } @@ -296,7 +285,7 @@ void bucketPayloadGoldenAndTotalBucketsArePreserved() throws Exception { byte[] header = fixture("avroHeader"); byte[] a = partition(7, "left"); byte[] b = partition(9, null); - ManifestRowIdIndex.Builder builder = new ManifestRowIdIndex.Builder(defaults, header); + ManifestSidecar.Builder builder = new ManifestSidecar.Builder(defaults, header); builder.beginBlock(header.length, 100, 3); builder.add(0L, 10, a, 1, 4); builder.add(5L, 5, a, 1, 4); @@ -314,14 +303,12 @@ void bucketPayloadGoldenAndTotalBucketsArePreserved() throws Exception { assertThat(data).isEqualTo(fixture("indexWithBuckets")); ManifestFileMeta meta = meta("manifest-golden", header.length + 400, 7); BucketFilter bucket = BucketFilter.create(false, 1, null, null); - assertThat( - ManifestRowIdIndex.select(data, meta, null, null, type, bucket, defaults) - .blocks()) + assertThat(ManifestSidecar.select(data, meta, null, null, type, bucket, defaults).blocks()) .extracting(block -> block.firstRecord) .containsExactly(0L); // Existing row-id and partition coverage remains independently usable. assertThat( - ManifestRowIdIndex.select( + ManifestSidecar.select( data, meta, query(0), @@ -344,9 +331,7 @@ public boolean mayContain(int min, int max, int total) { } }; BucketFilter filter = BucketFilter.create(false, null, null, totalAware); - assertThat( - ManifestRowIdIndex.select(data, meta, null, null, type, filter, defaults) - .blocks()) + assertThat(ManifestSidecar.select(data, meta, null, null, type, filter, defaults).blocks()) .extracting(block -> block.firstRecord) .containsExactly(3L); BucketFilter partitionAware = @@ -358,13 +343,13 @@ public boolean mayContain(int min, int max, int total) { throw new AssertionError("Needs actual entry partition"); }); assertThat( - ManifestRowIdIndex.select( + ManifestSidecar.select( data, meta, null, null, type, partitionAware, defaults) .blocks()) .hasSize(3); for (String unavailable : new String[] {"index", "indexWithPartitions"}) { assertThat( - ManifestRowIdIndex.select( + ManifestSidecar.select( fixture(unavailable), meta, null, @@ -380,8 +365,8 @@ public boolean mayContain(int min, int max, int total) { @Test void unknownInvalidOrOverBudgetBucketPayloadIsNull() throws Exception { Options options = new Options(); - options.set(CoreOptions.MANIFEST_INDEX_MAX_BUCKET_PAIRS, 1); - ManifestRowIdIndex.Settings settings = new ManifestRowIdIndex.Settings(options); + options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BUCKET_PAIRS, 1); + ManifestSidecar.Settings settings = new ManifestSidecar.Settings(options); byte[] header = fixture("avroHeader"); for (Integer[] pair : Arrays.asList( @@ -390,7 +375,7 @@ void unknownInvalidOrOverBudgetBucketPayloadIsNull() throws Exception { new Integer[] {4, 4}, new Integer[] {0, 0}, new Integer[] {2, 8})) { - ManifestRowIdIndex.Builder builder = new ManifestRowIdIndex.Builder(settings, header); + ManifestSidecar.Builder builder = new ManifestSidecar.Builder(settings, header); builder.beginBlock(header.length, 100, 2); builder.add(100L, 10, partition(7, "left"), 1, 4); builder.add(200L, 10, partition(7, "left"), pair[0], pair[1]); @@ -402,7 +387,7 @@ void unknownInvalidOrOverBudgetBucketPayloadIsNull() throws Exception { assertThat(ByteBuffer.wrap(data).getInt(positions(data).get(0)[3])).isEqualTo(-1); ManifestFileMeta meta = meta("m", header.length + 200, 3); assertThat( - ManifestRowIdIndex.select( + ManifestSidecar.select( data, meta, null, @@ -414,7 +399,7 @@ void unknownInvalidOrOverBudgetBucketPayloadIsNull() throws Exception { .extracting(block -> block.firstRecord) .containsExactly(0L); assertThat( - ManifestRowIdIndex.select( + ManifestSidecar.select( data, meta, query(999), @@ -447,7 +432,7 @@ void malformedBucketPayloadInvalidatesTheContainer() throws Exception { checksum(bad); assertThatThrownBy( () -> - ManifestRowIdIndex.select( + ManifestSidecar.select( bad, meta, query(999), part(99), type, defaults)) .isInstanceOf(IOException.class); } @@ -456,10 +441,10 @@ bad, meta, query(999), part(99), type, defaults)) @Test void tightByteBudgetKeepsAllDescriptorsOrOmitsTheWholeFile() throws Exception { Options options = new Options(); - options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_MAX_BYTES, 280); - ManifestRowIdIndex.Settings settings = new ManifestRowIdIndex.Settings(options); + options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES, 280); + ManifestSidecar.Settings settings = new ManifestSidecar.Settings(options); byte[] header = fixture("avroHeader"); - ManifestRowIdIndex.Builder builder = new ManifestRowIdIndex.Builder(settings, header); + ManifestSidecar.Builder builder = new ManifestSidecar.Builder(settings, header); for (int i = 0; i < 3; i++) { builder.beginBlock(header.length + 100L * i, 100, 1); builder.add(i * 100L, 10, partition(7, "left")); @@ -468,7 +453,7 @@ void tightByteBudgetKeepsAllDescriptorsOrOmitsTheWholeFile() throws Exception { byte[] data = builder.serialize("m", header.length + 300, 3); assertThat(data.length).isLessThanOrEqualTo(280); assertThat( - ManifestRowIdIndex.select( + ManifestSidecar.select( data, meta("m", header.length + 300, 3), query(999), diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaSerializerTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaSerializerTest.java index 62bb75edd313..0b2dc24baf09 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaSerializerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaSerializerTest.java @@ -55,7 +55,7 @@ void testExtraFiles() throws IOException { null, Collections.emptyList(), Arrays.asList("extra-1", "extra-2"), - Arrays.asList("partition-index", "manifest" + ManifestRowIdIndex.SUFFIX))) { + Arrays.asList("partition-index", "manifest" + ManifestSidecar.SUFFIX))) { ManifestFileMeta meta = new ManifestFileMeta( original.fileName(), diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java index ba0d8d478cbf..8fab7489d7bd 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java @@ -1278,8 +1278,8 @@ private static int indexOf(byte[] bytes, byte[] target, int from, int limit) { @Test void testRowIdSidecarRollingRawRewriteAndDelete() throws Exception { Options options = new Options(); - options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_WRITE, true); - options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_READ, true); + options.set(CoreOptions.MANIFEST_SIDECAR_WRITE, true); + options.set(CoreOptions.MANIFEST_SIDECAR_READ, true); ManifestFile manifests = createManifestFile(tempDir.toString(), 1, options); List entries = new ArrayList<>(); for (int i = 0; i < 2200; i++) { @@ -1296,8 +1296,8 @@ void testRowIdSidecarRollingRawRewriteAndDelete() throws Exception { assertThat(metas.size()).isGreaterThan(1); for (ManifestFileMeta meta : metas) { - assertThat(ManifestRowIdIndex.fileName(meta)) - .isEqualTo(meta.fileName() + ManifestRowIdIndex.SUFFIX); + assertThat(ManifestSidecar.fileName(meta)) + .isEqualTo(meta.fileName() + ManifestSidecar.SUFFIX); List actual = manifests.read(meta.fileName()); for (ManifestEntry entry : Arrays.asList(actual.get(0), actual.get(actual.size() - 1))) { @@ -1321,7 +1321,7 @@ void testRowIdSidecarRollingRawRewriteAndDelete() throws Exception { assertThat( java.nio.file.Files.exists( tempDir.resolve("manifest") - .resolve(meta.fileName() + ManifestRowIdIndex.SUFFIX))) + .resolve(meta.fileName() + ManifestSidecar.SUFFIX))) .isTrue(); } ManifestFileMeta source = metas.get(0); @@ -1332,8 +1332,8 @@ void testRowIdSidecarRollingRawRewriteAndDelete() throws Exception { } rewrite.close(); ManifestFileMeta rewritten = rewrite.result().get(0); - assertThat(ManifestRowIdIndex.fileName(rewritten)) - .isEqualTo(rewritten.fileName() + ManifestRowIdIndex.SUFFIX); + assertThat(ManifestSidecar.fileName(rewritten)) + .isEqualTo(rewritten.fileName() + ManifestSidecar.SUFFIX); assertThat(manifests.read(rewritten.fileName())) .isEqualTo(manifests.read(source.fileName())); long outside = metas.get(metas.size() - 1).maxRowId(); @@ -1347,14 +1347,14 @@ void testRowIdSidecarRollingRawRewriteAndDelete() throws Exception { assertThat( java.nio.file.Files.exists( tempDir.resolve("manifest") - .resolve(rewritten.fileName() + ManifestRowIdIndex.SUFFIX))) + .resolve(rewritten.fileName() + ManifestSidecar.SUFFIX))) .isFalse(); for (ManifestFileMeta meta : metas) { manifests.delete(meta); assertThat( java.nio.file.Files.exists( tempDir.resolve("manifest") - .resolve(meta.fileName() + ManifestRowIdIndex.SUFFIX))) + .resolve(meta.fileName() + ManifestSidecar.SUFFIX))) .isFalse(); } } @@ -1362,8 +1362,8 @@ void testRowIdSidecarRollingRawRewriteAndDelete() throws Exception { @Test void testReadsOnlySelectedBlocksAndPreservesPhysicalOrdinals() throws Exception { Options options = new Options(); - options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_WRITE, true); - options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_READ, true); + options.set(CoreOptions.MANIFEST_SIDECAR_WRITE, true); + options.set(CoreOptions.MANIFEST_SIDECAR_READ, true); RecordingFileIO fileIO = new RecordingFileIO(); ManifestFile.Factory factory = createManifestFileFactory(tempDir.toString(), Long.MAX_VALUE, options, fileIO); @@ -1386,7 +1386,7 @@ void testReadsOnlySelectedBlocksAndPreservesPhysicalOrdinals() throws Exception new Range(1000000000L, 1000000000L), new Range(3000000000L, 3000000000L))); - ManifestRowIdIndex.Selection selected = manifests.selectBlocks(meta, query); + ManifestSidecar.Selection selected = manifests.selectBlocks(meta, query); assertThat(selected.blocks()).hasSize(2); fileIO.reset(); @@ -1402,7 +1402,7 @@ void testReadsOnlySelectedBlocksAndPreservesPhysicalOrdinals() throws Exception java.util.function.Function.identity(), selected); List expected = new ArrayList<>(); - for (ManifestRowIdIndex.Block block : selected.blocks()) { + for (ManifestSidecar.Block block : selected.blocks()) { expected.addAll( entries.subList( (int) block.firstRecord, @@ -1425,8 +1425,8 @@ void testReadsOnlySelectedBlocksAndPreservesPhysicalOrdinals() throws Exception @Test void testScannerPreservesDeletesAndColumnGroups() { Options options = new Options(); - options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_WRITE, true); - options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_READ, true); + options.set(CoreOptions.MANIFEST_SIDECAR_WRITE, true); + options.set(CoreOptions.MANIFEST_SIDECAR_READ, true); RecordingFileIO fileIO = new RecordingFileIO(); ManifestFile.Factory factory = createManifestFileFactory(tempDir.toString(), Long.MAX_VALUE, options, fileIO); @@ -1485,7 +1485,7 @@ void testScannerPreservesDeletesAndColumnGroups() { assertThat(result).containsExactly(live); assertThat( fileIO.opened.stream() - .filter(path -> !path.getName().endsWith(ManifestRowIdIndex.SUFFIX)) + .filter(path -> !path.getName().endsWith(ManifestSidecar.SUFFIX)) .map(Path::getName)) .containsExactlyInAnyOrder(metas.get(0).fileName(), metas.get(1).fileName()); } @@ -1493,14 +1493,14 @@ void testScannerPreservesDeletesAndColumnGroups() { @Test void testSidecarWriteFailureAbortsAllRollingOutputs() { Options options = new Options(); - options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_WRITE, true); + options.set(CoreOptions.MANIFEST_SIDECAR_WRITE, true); AtomicInteger indexes = new AtomicInteger(); FileIO failing = new LocalFileIO() { @Override public PositionOutputStream newOutputStream(Path path, boolean overwrite) throws IOException { - if (path.getName().endsWith(ManifestRowIdIndex.SUFFIX) + if (path.getName().endsWith(ManifestSidecar.SUFFIX) && indexes.incrementAndGet() == 2) { throw new IOException("sidecar write failed"); } @@ -1533,8 +1533,8 @@ public PositionOutputStream newOutputStream(Path path, boolean overwrite) @Test void testBucketOnlyPlanningAndRawRewriteUseNullableBucketPayload() throws Exception { Options options = new Options(); - options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_WRITE, true); - options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_READ, true); + options.set(CoreOptions.MANIFEST_SIDECAR_WRITE, true); + options.set(CoreOptions.MANIFEST_SIDECAR_READ, true); RecordingFileIO io = new RecordingFileIO(); ManifestFile.Factory factory = createManifestFileFactory(tempDir.toString(), Long.MAX_VALUE, options, io); @@ -1585,8 +1585,8 @@ void testBucketOnlyPlanningAndRawRewriteUseNullableBucketPayload() throws Except @Test void testPartitionOnlyPlanningUsesBlocksWithoutRowIds() { Options options = new Options(); - options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_WRITE, true); - options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_READ, true); + options.set(CoreOptions.MANIFEST_SIDECAR_WRITE, true); + options.set(CoreOptions.MANIFEST_SIDECAR_READ, true); RecordingFileIO io = new RecordingFileIO(); ManifestFile.Factory factory = createManifestFileFactory(tempDir.toString(), Long.MAX_VALUE, options, io); @@ -1634,18 +1634,18 @@ void testPartitionOnlyPlanningUsesBlocksWithoutRowIds() { @Test void testUnknownRowIdKeepsPartitionIndexAndNoQueryDoesNotReadSidecar() { Options options = new Options(); - options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_WRITE, true); - options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_READ, true); + options.set(CoreOptions.MANIFEST_SIDECAR_WRITE, true); + options.set(CoreOptions.MANIFEST_SIDECAR_READ, true); RecordingFileIO fileIO = new RecordingFileIO(); ManifestFile manifests = createManifestFileFactory(tempDir.toString(), Long.MAX_VALUE, options, fileIO) .create(); ManifestFileMeta meta = manifests.write(Collections.singletonList(gen.next())).get(0); - assertThat(ManifestRowIdIndex.fileName(meta)).isNotNull(); + assertThat(ManifestSidecar.fileName(meta)).isNotNull(); assertThat( java.nio.file.Files.exists( tempDir.resolve("manifest") - .resolve(meta.fileName() + ManifestRowIdIndex.SUFFIX))) + .resolve(meta.fileName() + ManifestSidecar.SUFFIX))) .isTrue(); fileIO.reset(); @@ -1656,8 +1656,8 @@ void testUnknownRowIdKeepsPartitionIndexAndNoQueryDoesNotReadSidecar() { @Test void testExplicitIndexReferenceAndNullDoesNotProbe() throws Exception { Options options = new Options(); - options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_WRITE, true); - options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_READ, true); + options.set(CoreOptions.MANIFEST_SIDECAR_WRITE, true); + options.set(CoreOptions.MANIFEST_SIDECAR_READ, true); RecordingFileIO io = new RecordingFileIO(); ManifestFile manifests = createManifestFileFactory(tempDir.toString(), Long.MAX_VALUE, options, io).create(); @@ -1670,7 +1670,7 @@ void testExplicitIndexReferenceAndNullDoesNotProbe() throws Exception { original.totalBuckets(), original.file().newFirstRowId(100L)); ManifestFileMeta written = manifests.write(Collections.singletonList(entry)).get(0); - assertThat(ManifestRowIdIndex.fileName(written)).isNotNull(); + assertThat(ManifestSidecar.fileName(written)).isNotNull(); RowRangeIndex query = RowRangeIndex.create(Collections.singletonList(new Range(0, 0))); io.reset(); for (List extraFiles : @@ -1686,11 +1686,11 @@ void testExplicitIndexReferenceAndNullDoesNotProbe() throws Exception { assertThat( java.nio.file.Files.exists( tempDir.resolve("manifest") - .resolve(ManifestRowIdIndex.fileName(written)))) + .resolve(ManifestSidecar.fileName(written)))) .isTrue(); - String explicitName = "custom-index-name" + ManifestRowIdIndex.SUFFIX; + String explicitName = "custom-index-name" + ManifestSidecar.SUFFIX; java.nio.file.Files.move( - tempDir.resolve("manifest").resolve(ManifestRowIdIndex.fileName(written)), + tempDir.resolve("manifest").resolve(ManifestSidecar.fileName(written)), tempDir.resolve("manifest").resolve(explicitName)); String otherName = "other-partition-index"; java.nio.file.Path otherPath = tempDir.resolve("manifest").resolve(otherName); @@ -1716,7 +1716,7 @@ void testCommitCleanerDeletesExplicitIndexReferences() throws Exception { CommitCleaner cleaner = new CommitCleaner(lists, manifests, mock(IndexManifestFile.class)); for (int mode = 0; mode < 2; mode++) { ManifestFileMeta meta = manifests.write(Collections.singletonList(gen.next())).get(0); - String indexName = "commit-index-" + mode + ManifestRowIdIndex.SUFFIX; + String indexName = "commit-index-" + mode + ManifestSidecar.SUFFIX; Path indexPath = new Path(tempDir.toString(), "manifest/" + indexName); LocalFileIO.create().newOutputStream(indexPath, false).close(); String otherName = "commit-other-" + mode; @@ -1846,7 +1846,7 @@ private ManifestFile.Factory createManifestFileFactory( pathFactory, suggestedFileSize, cache) - .withRowIdIndexOptions(options); + .withSidecarOptions(options); } @Test diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestIndexTestUtils.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestIndexTestUtils.java index 74a82c789136..32a94c9a5bf7 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestIndexTestUtils.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestIndexTestUtils.java @@ -80,7 +80,7 @@ public static void registerIndexReferences(FileStore store, long snapshotId) List indexed = new ArrayList<>(); for (ManifestFileMeta meta : lists.read(value.asText())) { // Deliberately use a name which cannot be derived by appending the sidecar suffix. - String name = "index-for-" + meta.fileName() + ManifestRowIdIndex.SUFFIX; + String name = "index-for-" + meta.fileName() + ManifestSidecar.SUFFIX; Path index = store.pathFactory().toManifestFilePath(name); if (!io.exists(index)) { // GC treats index bytes as opaque; unsupported/partial files are still owned. diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestRowIdIndexTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java similarity index 86% rename from paimon-core/src/test/java/org/apache/paimon/manifest/ManifestRowIdIndexTest.java rename to paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java index 6d17e3bd868c..159987f7e72c 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestRowIdIndexTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java @@ -57,25 +57,23 @@ import static org.mockito.Mockito.when; /** Cross-language format, physical block positions, completeness and allocation bounds. */ -class ManifestRowIdIndexTest { +class ManifestSidecarTest { @TempDir java.nio.file.Path temp; - private final ManifestRowIdIndex.Settings settings = - new ManifestRowIdIndex.Settings(new Options()); + private final ManifestSidecar.Settings settings = new ManifestSidecar.Settings(new Options()); static ManifestFileMeta meta(String name, long size, long entries) { ManifestFileMeta meta = mock(ManifestFileMeta.class); when(meta.fileName()).thenReturn(name); when(meta.fileSize()).thenReturn(size); when(meta.extraFiles()) - .thenReturn(Collections.singletonList(name + ManifestRowIdIndex.SUFFIX)); + .thenReturn(Collections.singletonList(name + ManifestSidecar.SUFFIX)); when(meta.numAddedFiles()).thenReturn(entries); return meta; } private Properties fixture() throws IOException { Properties properties = new Properties(); - try (java.io.InputStream input = - getClass().getResourceAsStream("/manifest-block-index.txt")) { + try (java.io.InputStream input = getClass().getResourceAsStream("/manifest-sidecar.txt")) { properties.load(input); } return properties; @@ -96,7 +94,7 @@ private ManifestFileMeta goldenMeta() throws IOException { @Test void crossLanguageFormatAndBlockOrdinals() throws Exception { byte[] header = header(); - ManifestRowIdIndex.Builder builder = new ManifestRowIdIndex.Builder(settings, header); + ManifestSidecar.Builder builder = new ManifestSidecar.Builder(settings, header); builder.beginBlock(header.length, 100, 3); builder.add(0L, 10); builder.add(5L, 5); @@ -133,26 +131,26 @@ void crossLanguageFormatAndBlockOrdinals() throws Exception { }) { assertThat(select(data, meta, point).blocks()).as("row %s", point).isEmpty(); } - ManifestRowIdIndex.Selection selected = select(data, meta, 20); + ManifestSidecar.Selection selected = select(data, meta, 20); assertThat(selected.blocks()).extracting(b -> b.firstRecord).containsExactly(0L, 5L); assertThat(selected.blocks()) .extracting(b -> b.offset) .containsExactly((long) header.length, header.length + 300L); assertThat(selected.blocks()).extracting(b -> b.length).containsExactly(100L, 100L); - ManifestRowIdIndex.Selection gap = select(data, meta, 16); + ManifestSidecar.Selection gap = select(data, meta, 16); assertThat(gap.blocks()).isEmpty(); RowRangeIndex query = RowRangeIndex.create(Arrays.asList(new Range(10, 19), new Range(25, 40))); - assertThat(ManifestRowIdIndex.select(data, meta, query, settings).blocks()).isEmpty(); + assertThat(ManifestSidecar.select(data, meta, query, settings).blocks()).isEmpty(); assertThat(query.ranges()).containsExactly(new Range(10, 19), new Range(25, 40)); } @Test void minMaxSkipsExactIntersectionChecksAndHandlesOneInterval() throws Exception { byte[] header = header(); - ManifestRowIdIndex.Builder builder = new ManifestRowIdIndex.Builder(settings, header); + ManifestSidecar.Builder builder = new ManifestSidecar.Builder(settings, header); builder.beginBlock(header.length, 100, 2); builder.add(0L, 10); builder.add(20L, 10); @@ -168,8 +166,7 @@ void minMaxSkipsExactIntersectionChecksAndHandlesOneInterval() throws Exception ManifestFileMeta meta = meta("m", header.length + 300, 5); RowRangeIndex outside = spy(RowRangeIndex.create(Collections.singletonList(new Range(50, 59)))); - ManifestRowIdIndex.Selection none = - ManifestRowIdIndex.select(data, meta, outside, settings); + ManifestSidecar.Selection none = ManifestSidecar.select(data, meta, outside, settings); assertThat(none.blocks()).isEmpty(); // Only the three envelopes are tested; no individual interval intersection is evaluated. @@ -183,7 +180,7 @@ void minMaxSkipsExactIntersectionChecksAndHandlesOneInterval() throws Exception RowRangeIndex.create( Collections.singletonList( new Range((1L << 32) + 9, (1L << 32) + 9)))); - ManifestRowIdIndex.Selection hit = ManifestRowIdIndex.select(data, meta, one, settings); + ManifestSidecar.Selection hit = ManifestSidecar.select(data, meta, one, settings); assertThat(hit.blocks()).extracting(b -> b.firstRecord).containsExactly(4L); // A one-interval block needs no second intersection check after its envelope matches. @@ -199,14 +196,14 @@ void malformedIntervalsStillFallbackAfterMinMaxRejectionOrAnEarlyHit() throws Ex byte[] hash = MessageDigest.getInstance("SHA-256").digest(Arrays.copyOf(data, data.length - 32)); System.arraycopy(hash, 0, data, data.length - 32, 32); - Files.write(temp.resolve("manifest-golden" + ManifestRowIdIndex.SUFFIX), data); + Files.write(temp.resolve("manifest-golden" + ManifestSidecar.SUFFIX), data); ManifestFileMeta meta = goldenMeta(); for (long point : new long[] {30, 0}) { RowRangeIndex query = RowRangeIndex.create(Collections.singletonList(new Range(point, point))); assertThat( - ManifestRowIdIndex.read( + ManifestSidecar.read( LocalFileIO.create(), new Path(temp.toString(), "manifest-golden"), meta, @@ -219,7 +216,7 @@ void malformedIntervalsStillFallbackAfterMinMaxRejectionOrAnEarlyHit() throws Ex @Test void hugeRangesAreNotExpandedAndInvalidCoverageRetainsBlocks() throws Exception { byte[] header = header(); - ManifestRowIdIndex.Builder builder = new ManifestRowIdIndex.Builder(settings, header); + ManifestSidecar.Builder builder = new ManifestSidecar.Builder(settings, header); builder.beginBlock(header.length, 100, 2); builder.add(0L, Long.MAX_VALUE); builder.add(Long.MAX_VALUE, 1); @@ -229,7 +226,7 @@ void hugeRangesAreNotExpandedAndInvalidCoverageRetainsBlocks() throws Exception assertThat(select(data, meta("m", header.length + 100, 2), Long.MAX_VALUE).blocks()) .hasSize(1); for (Long first : Arrays.asList(null, -1L, Long.MAX_VALUE)) { - builder = new ManifestRowIdIndex.Builder(settings, header); + builder = new ManifestSidecar.Builder(settings, header); builder.beginBlock(header.length, 100, 1); builder.add(first, 2); builder.endBlock(); @@ -242,7 +239,7 @@ void hugeRangesAreNotExpandedAndInvalidCoverageRetainsBlocks() throws Exception .hasSize(1); } for (long count : new long[] {0, -1}) { - builder = new ManifestRowIdIndex.Builder(settings, header); + builder = new ManifestSidecar.Builder(settings, header); builder.beginBlock(header.length, 100, 1); builder.add(0L, count); builder.endBlock(); @@ -255,8 +252,8 @@ void hugeRangesAreNotExpandedAndInvalidCoverageRetainsBlocks() throws Exception .hasSize(1); } Options options = new Options(); - options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_MAX_RANGES, 1); - builder = new ManifestRowIdIndex.Builder(new ManifestRowIdIndex.Settings(options), header); + options.set(CoreOptions.MANIFEST_SIDECAR_MAX_RANGES, 1); + builder = new ManifestSidecar.Builder(new ManifestSidecar.Settings(options), header); builder.beginBlock(header.length, 100, 2); builder.add(0L, 1); builder.add(10L, 1); @@ -268,28 +265,26 @@ void hugeRangesAreNotExpandedAndInvalidCoverageRetainsBlocks() throws Exception 5) .blocks()) .hasSize(1); - options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_MAX_BYTES, 128); - builder = new ManifestRowIdIndex.Builder(new ManifestRowIdIndex.Settings(options), header); + options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES, 128); + builder = new ManifestSidecar.Builder(new ManifestSidecar.Settings(options), header); assertThat(builder.serialize("m", 1, 2)).isNull(); } @Test void corruptMissingIncompleteAndMismatchedIndexesFallback() throws Exception { Path manifest = new Path(temp.toString(), "manifest-golden"); - java.nio.file.Path index = temp.resolve("manifest-golden" + ManifestRowIdIndex.SUFFIX); + java.nio.file.Path index = temp.resolve("manifest-golden" + ManifestSidecar.SUFFIX); ManifestFileMeta meta = goldenMeta(); RowRangeIndex query = RowRangeIndex.create(Collections.singletonList(new Range(11, 11))); - assertThat(ManifestRowIdIndex.read(LocalFileIO.create(), manifest, meta, query, settings)) + assertThat(ManifestSidecar.read(LocalFileIO.create(), manifest, meta, query, settings)) .isNull(); byte[] good = golden(); for (int position : new int[] {0, 9, 11, 15, 16, 55, 63, 67, 75, good.length - 1}) { byte[] bad = good.clone(); bad[position] ^= 2; Files.write(index, bad); - assertThat( - ManifestRowIdIndex.read( - LocalFileIO.create(), manifest, meta, query, settings)) + assertThat(ManifestSidecar.read(LocalFileIO.create(), manifest, meta, query, settings)) .isNull(); } // A valid checksum cannot make an unsupported container version readable. @@ -300,21 +295,20 @@ void corruptMissingIncompleteAndMismatchedIndexesFallback() throws Exception { MessageDigest.getInstance("SHA-256") .digest(Arrays.copyOf(bad, bad.length - 32)); System.arraycopy(hash, 0, bad, bad.length - 32, 32); - assertThatThrownBy(() -> ManifestRowIdIndex.select(bad, meta, query, settings)) + assertThatThrownBy(() -> ManifestSidecar.select(bad, meta, query, settings)) .isInstanceOf(IOException.class); } Files.write(index, Arrays.copyOf(good, good.length - 1)); - assertThat(ManifestRowIdIndex.read(LocalFileIO.create(), manifest, meta, query, settings)) + assertThat(ManifestSidecar.read(LocalFileIO.create(), manifest, meta, query, settings)) .isNull(); Files.write(index, good); assertThat( - ManifestRowIdIndex.read( - LocalFileIO.create(), manifest, meta, query, settings) + ManifestSidecar.read(LocalFileIO.create(), manifest, meta, query, settings) .blocks()) .isEmpty(); assertThatThrownBy( () -> - ManifestRowIdIndex.select( + ManifestSidecar.select( good, meta("other", meta.fileSize(), 7), query, settings)) .isInstanceOf(IOException.class); } @@ -333,7 +327,7 @@ public org.apache.paimon.fs.SeekableInputStream newInputStream(Path path) throw new java.net.SocketTimeoutException("timeout"); } }; - assertThat(ManifestRowIdIndex.read(timedOut, path, manifest, query, settings)).isNull(); + assertThat(ManifestSidecar.read(timedOut, path, manifest, query, settings)).isNull(); assertThat(Thread.currentThread().isInterrupted()).isFalse(); LocalFileIO interrupted = new LocalFileIO() { @@ -346,7 +340,7 @@ public org.apache.paimon.fs.SeekableInputStream newInputStream(Path path) try { assertThatThrownBy( () -> - ManifestRowIdIndex.read( + ManifestSidecar.read( interrupted, path, manifest, query, settings)) .isInstanceOf(java.io.UncheckedIOException.class); assertThat(Thread.currentThread().isInterrupted()).isTrue(); @@ -360,7 +354,7 @@ public org.apache.paimon.fs.SeekableInputStream newInputStream(Path path) { throw new AssertionError("fatal"); } }; - assertThatThrownBy(() -> ManifestRowIdIndex.read(failed, path, manifest, query, settings)) + assertThatThrownBy(() -> ManifestSidecar.read(failed, path, manifest, query, settings)) .isInstanceOf(AssertionError.class); } @@ -403,7 +397,7 @@ public void close() throws IOException { try { assertThatThrownBy( () -> - ManifestRowIdIndex.read( + ManifestSidecar.read( fileIO, new Path(temp.toString(), "m"), manifest, @@ -437,7 +431,7 @@ public org.apache.paimon.fs.SeekableInputStream newInputStream(Path path) } }; assertThat( - ManifestRowIdIndex.read( + ManifestSidecar.read( fileIO, new Path(temp.toString(), "m"), meta("m", 1, 1), @@ -451,7 +445,7 @@ public org.apache.paimon.fs.SeekableInputStream newInputStream(Path path) void indexReadsUseBoundedBulkRequests() throws Exception { byte[] header = header(); for (int blockCount : new int[] {5000, 25000}) { - ManifestRowIdIndex.Builder builder = new ManifestRowIdIndex.Builder(settings, header); + ManifestSidecar.Builder builder = new ManifestSidecar.Builder(settings, header); for (int blockNumber = 0; blockNumber < blockCount; blockNumber++) { builder.beginBlock(header.length + blockNumber * 100L, 100, 1); builder.add((long) blockNumber, 1); @@ -463,9 +457,9 @@ void indexReadsUseBoundedBulkRequests() throws Exception { CountingInput stream = new CountingInput(data, Integer.MAX_VALUE); Path path = new Path(temp.toString(), meta.fileName()); FileIO io = mock(FileIO.class); - when(io.newInputStream(ManifestRowIdIndex.path(path))).thenReturn(stream); - ManifestRowIdIndex.Selection actual = - ManifestRowIdIndex.read( + when(io.newInputStream(ManifestSidecar.path(path))).thenReturn(stream); + ManifestSidecar.Selection actual = + ManifestSidecar.read( io, path, meta, @@ -483,19 +477,19 @@ void indexReadsUseBoundedBulkRequests() throws Exception { void indexShortReadsAndExactBudget() throws Exception { byte[] data = golden(); Options options = new Options(); - options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_MAX_BYTES, data.length); + options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES, data.length); Path path = new Path(temp.toString(), "manifest-golden"); for (int maxRead : new int[] {Integer.MAX_VALUE, 7}) { CountingInput stream = new CountingInput(data, maxRead); FileIO io = mock(FileIO.class); - when(io.newInputStream(ManifestRowIdIndex.path(path))).thenReturn(stream); - ManifestRowIdIndex.Selection actual = - ManifestRowIdIndex.read( + when(io.newInputStream(ManifestSidecar.path(path))).thenReturn(stream); + ManifestSidecar.Selection actual = + ManifestSidecar.read( io, path, goldenMeta(), RowRangeIndex.create(Collections.singletonList(new Range(20, 20))), - new ManifestRowIdIndex.Settings(options)); + new ManifestSidecar.Settings(options)); assertThat(actual.blocks()) .extracting(block -> block.firstRecord) .containsExactly(0L, 5L); @@ -506,18 +500,18 @@ void indexShortReadsAndExactBudget() throws Exception { @Test void indexOverBudgetStopsAfterOneExtraByte() throws Exception { Options options = new Options(); - options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_MAX_BYTES, 128); + options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES, 128); Path path = new Path(temp.toString(), "manifest-golden"); CountingInput stream = new CountingInput(golden(), Integer.MAX_VALUE); FileIO io = mock(FileIO.class); - when(io.newInputStream(ManifestRowIdIndex.path(path))).thenReturn(stream); + when(io.newInputStream(ManifestSidecar.path(path))).thenReturn(stream); assertThat( - ManifestRowIdIndex.read( + ManifestSidecar.read( io, path, goldenMeta(), RowRangeIndex.create(Collections.singletonList(new Range(20, 20))), - new ManifestRowIdIndex.Settings(options))) + new ManifestSidecar.Settings(options))) .isNull(); assertThat(stream.readLengths).containsExactly(129); assertThat(stream.closed).isTrue(); @@ -530,8 +524,8 @@ void adjacentBlocksShareReadsForSingleByteConsumers() throws Exception { for (int position = 0; position < body.length; position++) { body[position] = (byte) position; } - ManifestRowIdIndex.Selection selected = - ManifestRowIdIndex.select( + ManifestSidecar.Selection selected = + ManifestSidecar.select( golden(), goldenMeta(), RowRangeIndex.create( @@ -546,7 +540,7 @@ void adjacentBlocksShareReadsForSingleByteConsumers() throws Exception { FileIO io = mock(FileIO.class); when(io.newInputStream(path)).thenReturn(stream); ByteArrayOutputStream actual = new ByteArrayOutputStream(); - try (InputStream input = ManifestRowIdIndex.openManifest(io, path, selected)) { + try (InputStream input = ManifestSidecar.openManifest(io, path, selected)) { int value; while ((value = input.read()) != -1) { actual.write(value); @@ -571,8 +565,7 @@ void blockReadsSkipGapsAndEmptySelections() throws Exception { when(io.newInputStream(path)).thenReturn(stream); byte[] actual; try (InputStream input = - ManifestRowIdIndex.openManifest( - io, path, select(golden(), goldenMeta(), point))) { + ManifestSidecar.openManifest(io, path, select(golden(), goldenMeta(), point))) { actual = IOUtils.readFully(input, false); } if (point == 20) { @@ -592,7 +585,7 @@ io, path, select(golden(), goldenMeta(), point))) { @Test void largeBlockSpansUseBoundedReads() throws Exception { byte[] header = header(); - ManifestRowIdIndex.Builder builder = new ManifestRowIdIndex.Builder(settings, header); + ManifestSidecar.Builder builder = new ManifestSidecar.Builder(settings, header); long offset = header.length; for (int length : new int[] {512 * 1024, 512 * 1024, 257}) { builder.beginBlock(offset, length, 1); @@ -607,7 +600,7 @@ void largeBlockSpansUseBoundedReads() throws Exception { Path path = new Path(temp.toString(), "manifest-large"); when(io.newInputStream(path)).thenReturn(stream); try (InputStream input = - ManifestRowIdIndex.openManifest( + ManifestSidecar.openManifest( io, path, select(data, meta("manifest-large", offset, 3), 20))) { assertThat(IOUtils.readFully(input, false)).isEqualTo(manifest); } @@ -620,8 +613,8 @@ io, path, select(data, meta("manifest-large", offset, 3), 20))) { void blockShortReadsAndTruncation() throws Exception { byte[] header = header(); Path path = new Path(temp.toString(), "manifest-golden"); - ManifestRowIdIndex.Selection selected = - ManifestRowIdIndex.select( + ManifestSidecar.Selection selected = + ManifestSidecar.select( golden(), goldenMeta(), RowRangeIndex.create( @@ -632,7 +625,7 @@ void blockShortReadsAndTruncation() throws Exception { CountingInput stream = new CountingInput(manifest, 7); FileIO io = mock(FileIO.class); when(io.newInputStream(path)).thenReturn(stream); - try (InputStream input = ManifestRowIdIndex.openManifest(io, path, selected)) { + try (InputStream input = ManifestSidecar.openManifest(io, path, selected)) { if (bodyLength == 400) { assertThat(IOUtils.readFully(input, false)).isEqualTo(manifest); } else { @@ -679,9 +672,9 @@ public void close() throws IOException { } } - private ManifestRowIdIndex.Selection select(byte[] data, ManifestFileMeta meta, long point) + private ManifestSidecar.Selection select(byte[] data, ManifestFileMeta meta, long point) throws IOException { - return ManifestRowIdIndex.select( + return ManifestSidecar.select( data, meta, RowRangeIndex.create(Collections.singletonList(new Range(point, point))), diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/ExpireSnapshotsTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/ExpireSnapshotsTest.java index 30c9f47b9aee..957e4b73177c 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/ExpireSnapshotsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/ExpireSnapshotsTest.java @@ -39,7 +39,7 @@ import org.apache.paimon.manifest.ManifestEntry; import org.apache.paimon.manifest.ManifestFileMeta; import org.apache.paimon.manifest.ManifestIndexTestUtils; -import org.apache.paimon.manifest.ManifestRowIdIndex; +import org.apache.paimon.manifest.ManifestSidecar; import org.apache.paimon.mergetree.compact.DeduplicateMergeFunction; import org.apache.paimon.options.ExpireConfig; import org.apache.paimon.options.MemorySize; @@ -907,7 +907,7 @@ void testSidecarsFollowSnapshotAndTagRetention() throws Exception { manifest.getParent(), "index-for-" + manifest.getName() - + ManifestRowIdIndex.SUFFIX))) + + ManifestSidecar.SUFFIX))) .as("sidecar for %s", manifest) .isEqualTo(retained); } @@ -921,7 +921,7 @@ void testSidecarsFollowSnapshotAndTagRetention() throws Exception { assertThat( fileIO.exists( store.pathFactory() - .toManifestFilePath(ManifestRowIdIndex.fileName(meta)))) + .toManifestFilePath(ManifestSidecar.fileName(meta)))) .isTrue(); } } diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/LocalOrphanFilesCleanTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/LocalOrphanFilesCleanTest.java index 08b66a0a59db..49e22ee551e1 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/LocalOrphanFilesCleanTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/LocalOrphanFilesCleanTest.java @@ -34,7 +34,7 @@ import org.apache.paimon.manifest.ManifestFileMeta; import org.apache.paimon.manifest.ManifestIndexTestUtils; import org.apache.paimon.manifest.ManifestList; -import org.apache.paimon.manifest.ManifestRowIdIndex; +import org.apache.paimon.manifest.ManifestSidecar; import org.apache.paimon.mergetree.compact.ConcatRecordReader; import org.apache.paimon.options.Options; import org.apache.paimon.reader.ReaderSupplier; @@ -174,13 +174,13 @@ void testOrphanCleanupProtectsReferencedSidecars() throws Exception { .manifestListFactory() .create() .readDataManifests(table.snapshotManager().latestSnapshot())) { - Path sidecar = new Path(manifestDir, ManifestRowIdIndex.fileName(meta)); + Path sidecar = new Path(manifestDir, ManifestSidecar.fileName(meta)); sidecars.add(sidecar); - Path guessed = new Path(manifestDir, meta.fileName() + ManifestRowIdIndex.SUFFIX); + Path guessed = new Path(manifestDir, meta.fileName() + ManifestSidecar.SUFFIX); fileIO.newOutputStream(guessed, false).close(); unreferenced.add(guessed); } - Path orphan = new Path(manifestDir, "manifest-orphan" + ManifestRowIdIndex.SUFFIX); + Path orphan = new Path(manifestDir, "manifest-orphan" + ManifestSidecar.SUFFIX); fileIO.newOutputStream(orphan, false).close(); new LocalOrphanFilesClean(table, System.currentTimeMillis() + 2000).clean(); assertThat(fileIO.exists(orphan)).isFalse(); diff --git a/paimon-core/src/test/resources/manifest-block-index.txt b/paimon-core/src/test/resources/manifest-sidecar.txt similarity index 84% rename from paimon-core/src/test/resources/manifest-block-index.txt rename to paimon-core/src/test/resources/manifest-sidecar.txt index 8f0c7fbea6c3..3b01e938aace 100644 --- a/paimon-core/src/test/resources/manifest-block-index.txt +++ b/paimon-core/src/test/resources/manifest-sidecar.txt @@ -18,6 +18,6 @@ avroHeader=T2JqAQQUYXZyby5jb2RlYwhudWxsFmF2cm8uc2NoZW1hDCJsb25nIgAAAAAAAAAAAAAAAAAAAAAA partitionA=AAAAAgAAAAAAAAAABwAAAAAAAABsZWZ0AAAAhA== partitionB=AAAAAgACAAAAAAAACQAAAAAAAAAAAAAAAAAAAA== -index=UEFJTVJJRFgAAAABL0eWubkHdL6ioNdIPj3jGtp9TAk83wbtAsmc+xJVwM8AAAAAAAAByQAAAAAAAAAHAAAAOU9iagEEFGF2cm8uY29kZWMIbnVsbBZhdnJvLnNjaGVtYQwibG9uZyIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAADkAAAAAAAAAZAAAAAAAAAADAAAAAAABAAAAJAAAAAIAAAAAAAAAAAAAAAAAAAAJAAAAAAAAABQAAAAAAAAAGP////8AAAAAAAAAnQAAAAAAAADIAAAAAAAAAAIAAAAAAAEAAAAkAAAAAgAAAAD////+AAAAAQAAAAIAAAeBzDhsZQAAB4HMOGxl/////wAAAAAAAAFlAAAAAAAAAGQAAAAAAAAAAgAAAAAAAQAAACQAAAACAAAAAAAAABQAAAAAAAAAGH//////////f///////////////B9Fp1oyxbV6UmfRT5d+Fv/MxGHAmXjE2Bs5JxBGcn+E= -indexWithPartitions=UEFJTVJJRFgAAAABL0eWubkHdL6ioNdIPj3jGtp9TAk83wbtAsmc+xJVwM8AAAAAAAAByQAAAAAAAAAHAAAAOU9iagEEFGF2cm8uY29kZWMIbnVsbBZhdnJvLnNjaGVtYQwibG9uZyIAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAcAAAAAgAAAAAAAAAABwAAAAAAAABsZWZ0AAAAhAAAABwAAAACAAIAAAAAAAAJAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAA5AAAAAAAAAGQAAAAAAAAAAwEAAAAMAAAAAgAAAAAAAAABAQAAACQAAAACAAAAAAAAAAAAAAAAAAAACQAAAAAAAAAUAAAAAAAAABj/////AAAAAAAAAJ0AAAAAAAAAyAAAAAAAAAACAQAAAAwAAAACAAAAAAAAAAEBAAAAJAAAAAIAAAAA/////gAAAAEAAAACAAAHgcw4bGUAAAeBzDhsZf////8AAAAAAAABZQAAAAAAAABkAAAAAAAAAAIBAAAADAAAAAIAAAAAAAAAAQEAAAAkAAAAAgAAAAAAAAAUAAAAAAAAABh//////////3///////////////1Ap6GHiQax4WqX0VrJqQ5tMmn3ZtQkNqJckeYuATS+A -indexWithBuckets=UEFJTVJJRFgAAAABL0eWubkHdL6ioNdIPj3jGtp9TAk83wbtAsmc+xJVwM8AAAAAAAAByQAAAAAAAAAHAAAAOU9iagEEFGF2cm8uY29kZWMIbnVsbBZhdnJvLnNjaGVtYQwibG9uZyIAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAcAAAAAgAAAAAAAAAABwAAAAAAAABsZWZ0AAAAhAAAABwAAAACAAIAAAAAAAAJAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAA5AAAAAAAAAGQAAAAAAAAAAwEAAAAMAAAAAgAAAAAAAAABAQAAACQAAAACAAAAAAAAAAAAAAAAAAAACQAAAAAAAAAUAAAAAAAAABgAAAAUAAAAAgAAAAEAAAAEAAAAAQAAAAgAAAAAAAAAnQAAAAAAAADIAAAAAAAAAAIBAAAADAAAAAIAAAAAAAAAAQEAAAAkAAAAAgAAAAD////+AAAAAQAAAAIAAAeBzDhsZQAAB4HMOGxlAAAAFAAAAAIAAAACAAAABAAAAAIAAAAIAAAAAAAAAWUAAAAAAAAAZAAAAAAAAAACAQAAAAwAAAACAAAAAAAAAAEBAAAAJAAAAAIAAAAAAAAAFAAAAAAAAAAYf/////////9//////////wAAABQAAAACAAAAAAAAAAEAAAADAAAABKZ7n0UVSe+qqinQITnc3OYtZWHTXxbNPJmCRW2/eyB4 +index=UEFJTVNDQVIAAAABL0eWubkHdL6ioNdIPj3jGtp9TAk83wbtAsmc+xJVwM8AAAAAAAAByQAAAAAAAAAHAAAAOU9iagEEFGF2cm8uY29kZWMIbnVsbBZhdnJvLnNjaGVtYQwibG9uZyIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAADkAAAAAAAAAZAAAAAAAAAADAAAAAAABAAAAJAAAAAIAAAAAAAAAAAAAAAAAAAAJAAAAAAAAABQAAAAAAAAAGP////8AAAAAAAAAnQAAAAAAAADIAAAAAAAAAAIAAAAAAAEAAAAkAAAAAgAAAAD////+AAAAAQAAAAIAAAeBzDhsZQAAB4HMOGxl/////wAAAAAAAAFlAAAAAAAAAGQAAAAAAAAAAgAAAAAAAQAAACQAAAACAAAAAAAAABQAAAAAAAAAGH//////////f///////////////NP18llqvAb/4bxohLXD7cDFPAPeAb5ypZyhxRnTV1o8= +indexWithPartitions=UEFJTVNDQVIAAAABL0eWubkHdL6ioNdIPj3jGtp9TAk83wbtAsmc+xJVwM8AAAAAAAAByQAAAAAAAAAHAAAAOU9iagEEFGF2cm8uY29kZWMIbnVsbBZhdnJvLnNjaGVtYQwibG9uZyIAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAcAAAAAgAAAAAAAAAABwAAAAAAAABsZWZ0AAAAhAAAABwAAAACAAIAAAAAAAAJAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAA5AAAAAAAAAGQAAAAAAAAAAwEAAAAMAAAAAgAAAAAAAAABAQAAACQAAAACAAAAAAAAAAAAAAAAAAAACQAAAAAAAAAUAAAAAAAAABj/////AAAAAAAAAJ0AAAAAAAAAyAAAAAAAAAACAQAAAAwAAAACAAAAAAAAAAEBAAAAJAAAAAIAAAAA/////gAAAAEAAAACAAAHgcw4bGUAAAeBzDhsZf////8AAAAAAAABZQAAAAAAAABkAAAAAAAAAAIBAAAADAAAAAIAAAAAAAAAAQEAAAAkAAAAAgAAAAAAAAAUAAAAAAAAABh//////////3///////////////yuqJytq34abij//eVySjsXqKxL4O5cNwbD/cAdeORFh +indexWithBuckets=UEFJTVNDQVIAAAABL0eWubkHdL6ioNdIPj3jGtp9TAk83wbtAsmc+xJVwM8AAAAAAAAByQAAAAAAAAAHAAAAOU9iagEEFGF2cm8uY29kZWMIbnVsbBZhdnJvLnNjaGVtYQwibG9uZyIAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAcAAAAAgAAAAAAAAAABwAAAAAAAABsZWZ0AAAAhAAAABwAAAACAAIAAAAAAAAJAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAA5AAAAAAAAAGQAAAAAAAAAAwEAAAAMAAAAAgAAAAAAAAABAQAAACQAAAACAAAAAAAAAAAAAAAAAAAACQAAAAAAAAAUAAAAAAAAABgAAAAUAAAAAgAAAAEAAAAEAAAAAQAAAAgAAAAAAAAAnQAAAAAAAADIAAAAAAAAAAIBAAAADAAAAAIAAAAAAAAAAQEAAAAkAAAAAgAAAAD////+AAAAAQAAAAIAAAeBzDhsZQAAB4HMOGxlAAAAFAAAAAIAAAACAAAABAAAAAIAAAAIAAAAAAAAAWUAAAAAAAAAZAAAAAAAAAACAQAAAAwAAAACAAAAAAAAAAEBAAAAJAAAAAIAAAAAAAAAFAAAAAAAAAAYf/////////9//////////wAAABQAAAACAAAAAAAAAAEAAAADAAAABExlWmcJvax6T8wD69ZnRY9t/6uzT9JbUsxIZ4DoIV/4 diff --git a/paimon-python/pypaimon/common/options/core_options.py b/paimon-python/pypaimon/common/options/core_options.py index d6aafc67be2d..b0fed410fad1 100644 --- a/paimon-python/pypaimon/common/options/core_options.py +++ b/paimon-python/pypaimon/common/options/core_options.py @@ -297,44 +297,44 @@ class CoreOptions: .with_description("The parallelism for scanning manifest files.") ) - MANIFEST_ROW_ID_INDEX_WRITE: ConfigOption[bool] = ( - ConfigOptions.key("manifest.row-id-index.write") + MANIFEST_SIDECAR_WRITE: ConfigOption[bool] = ( + ConfigOptions.key("manifest.sidecar.write") .boolean_type() .default_value(False) ) - MANIFEST_ROW_ID_INDEX_READ: ConfigOption[bool] = ( - ConfigOptions.key("manifest.row-id-index.read") + MANIFEST_SIDECAR_READ: ConfigOption[bool] = ( + ConfigOptions.key("manifest.sidecar.read") .boolean_type() .default_value(False) ) - MANIFEST_ROW_ID_INDEX_MAX_RANGES: ConfigOption[int] = ( - ConfigOptions.key("manifest.row-id-index.max-ranges") + MANIFEST_SIDECAR_MAX_RANGES: ConfigOption[int] = ( + ConfigOptions.key("manifest.sidecar.max-ranges") .int_type() .default_value(131072) ) - MANIFEST_ROW_ID_INDEX_MAX_BYTES: ConfigOption[int] = ( - ConfigOptions.key("manifest.row-id-index.max-bytes") + MANIFEST_SIDECAR_MAX_BYTES: ConfigOption[int] = ( + ConfigOptions.key("manifest.sidecar.max-bytes") .int_type() .default_value(8388608) ) - MANIFEST_INDEX_MAX_PARTITIONS: ConfigOption[int] = ( - ConfigOptions.key("manifest.index.max-partitions") + MANIFEST_SIDECAR_MAX_PARTITIONS: ConfigOption[int] = ( + ConfigOptions.key("manifest.sidecar.max-partitions") .int_type() .default_value(65536) ) - MANIFEST_INDEX_MAX_BUCKET_PAIRS: ConfigOption[int] = ( - ConfigOptions.key("manifest.index.max-bucket-pairs") + MANIFEST_SIDECAR_MAX_BUCKET_PAIRS: ConfigOption[int] = ( + ConfigOptions.key("manifest.sidecar.max-bucket-pairs") .int_type() .default_value(4096) ) - MANIFEST_INDEX_MAX_PARTITION_BYTES: ConfigOption[int] = ( - ConfigOptions.key("manifest.index.max-partition-bytes") + MANIFEST_SIDECAR_MAX_PARTITION_BYTES: ConfigOption[int] = ( + ConfigOptions.key("manifest.sidecar.max-partition-bytes") .int_type() .default_value(1048576) ) diff --git a/paimon-python/pypaimon/manifest/manifest_file_manager.py b/paimon-python/pypaimon/manifest/manifest_file_manager.py index f79cce283abb..bf7660923833 100644 --- a/paimon-python/pypaimon/manifest/manifest_file_manager.py +++ b/paimon-python/pypaimon/manifest/manifest_file_manager.py @@ -19,8 +19,8 @@ from io import BytesIO from typing import Callable, List, Optional -from pypaimon.manifest.row_id_index import ( - Settings, SUFFIX, Query, build_from_entries, read_index, read_selected_bytes, +from pypaimon.manifest.manifest_sidecar import ( + Settings, SUFFIX, Query, build_from_entries, read_sidecar, read_selected_bytes, ) import fastavro @@ -73,9 +73,10 @@ def read_entries_parallel(self, manifest_files: List[ManifestFileMeta], manifest def _process_single_manifest(manifest_file: ManifestFileMeta): path = f"{self.manifest_path}/{manifest_file.file_name}" selected = None - if settings.read and (query is not None or index_partition_filter is not None or early_entry_filter is not None): - selected = read_index(self.file_io, path, manifest_file, query, settings, - index_partition_filter, self.partition_keys_fields, early_entry_filter) + if settings.read and (query is not None or index_partition_filter is not None + or early_entry_filter is not None): + selected = read_sidecar(self.file_io, path, manifest_file, query, settings, + index_partition_filter, self.partition_keys_fields, early_entry_filter) if selected is not None and not selected.blocks: return [] return self.read( @@ -364,7 +365,7 @@ def delete(self, manifest: ManifestFileMeta): def _flush(self, file_name: str, avro_bytes: bytes, entries) -> ManifestFileMeta: manifest_path = f"{self.manifest_path}/{file_name}" - index_file_name = None + sidecar_file_name = None try: with self.file_io.new_output_stream(manifest_path) as output_stream: output_stream.write(avro_bytes) @@ -372,16 +373,16 @@ def _flush(self, file_name: str, avro_bytes: bytes, entries) -> ManifestFileMeta if settings.write: data = build_from_entries(avro_bytes, entries, file_name, settings) if data is not None: - index_file_name = file_name + SUFFIX - with self.file_io.new_output_stream(f"{self.manifest_path}/{index_file_name}") as output_stream: + sidecar_file_name = file_name + SUFFIX + with self.file_io.new_output_stream(f"{self.manifest_path}/{sidecar_file_name}") as output_stream: output_stream.write(data) # Publish the reference only after both objects close successfully. return self._build_meta(file_name, entries, len(avro_bytes), - [index_file_name] if index_file_name is not None else None) + [sidecar_file_name] if sidecar_file_name is not None else None) except BaseException as e: self.file_io.delete_quietly(manifest_path) - if index_file_name is not None: - self.file_io.delete_quietly(f"{self.manifest_path}/{index_file_name}") + if sidecar_file_name is not None: + self.file_io.delete_quietly(f"{self.manifest_path}/{sidecar_file_name}") if not isinstance(e, Exception) or isinstance(e, InterruptedError): raise raise RuntimeError(f"Failed to write manifest file: {e}") from e diff --git a/paimon-python/pypaimon/manifest/row_id_index.py b/paimon-python/pypaimon/manifest/manifest_sidecar.py similarity index 93% rename from paimon-python/pypaimon/manifest/row_id_index.py rename to paimon-python/pypaimon/manifest/manifest_sidecar.py index 09b4437e3921..44cd7e3f39a0 100644 --- a/paimon-python/pypaimon/manifest/row_id_index.py +++ b/paimon-python/pypaimon/manifest/manifest_sidecar.py @@ -33,8 +33,8 @@ from pypaimon.table.row.generic_row import GenericRowSerializer, GenericRowDeserializer LOG = logging.getLogger(__name__) -SUFFIX = '.row-id-index' -MAGIC = b'PAIMRIDX' +SUFFIX = '.avro.sidecar' +MAGIC = b'PAIMSCAR' FORMAT_VERSION = 1 MAX_ROW_ID = (1 << 63) - 1 MAX_AVRO_HEADER = 1024 * 1024 @@ -60,26 +60,26 @@ class Settings: def __post_init__(self): if not 1 <= self.max_bucket_pairs <= 1048576: - raise ValueError('Invalid manifest.index.max-bucket-pairs') + raise ValueError('Invalid manifest.sidecar.max-bucket-pairs') if not 1 <= self.max_partitions <= 1048576: - raise ValueError('Invalid manifest.index.max-partitions') + raise ValueError('Invalid manifest.sidecar.max-partitions') if not 1 <= self.max_partition_bytes <= 64 * 1024 * 1024: - raise ValueError('Invalid manifest.index.max-partition-bytes') + raise ValueError('Invalid manifest.sidecar.max-partition-bytes') if not 1 <= self.max_ranges <= 1048576: - raise ValueError('manifest.row-id-index.max-ranges must be in [1, 1048576]') + raise ValueError('manifest.sidecar.max-ranges must be in [1, 1048576]') if not 128 <= self.max_bytes <= 64 * 1024 * 1024: - raise ValueError('manifest.row-id-index.max-bytes must be in [128, 67108864]') + raise ValueError('manifest.sidecar.max-bytes must be in [128, 67108864]') @classmethod def from_options(cls, options): return cls( - options.options.get(CoreOptions.MANIFEST_ROW_ID_INDEX_WRITE), - options.options.get(CoreOptions.MANIFEST_ROW_ID_INDEX_READ), - options.options.get(CoreOptions.MANIFEST_ROW_ID_INDEX_MAX_RANGES), - options.options.get(CoreOptions.MANIFEST_ROW_ID_INDEX_MAX_BYTES), - options.options.get(CoreOptions.MANIFEST_INDEX_MAX_PARTITIONS), - options.options.get(CoreOptions.MANIFEST_INDEX_MAX_PARTITION_BYTES), - options.options.get(CoreOptions.MANIFEST_INDEX_MAX_BUCKET_PAIRS)) + options.options.get(CoreOptions.MANIFEST_SIDECAR_WRITE), + options.options.get(CoreOptions.MANIFEST_SIDECAR_READ), + options.options.get(CoreOptions.MANIFEST_SIDECAR_MAX_RANGES), + options.options.get(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES), + options.options.get(CoreOptions.MANIFEST_SIDECAR_MAX_PARTITIONS), + options.options.get(CoreOptions.MANIFEST_SIDECAR_MAX_PARTITION_BYTES), + options.options.get(CoreOptions.MANIFEST_SIDECAR_MAX_BUCKET_PAIRS)) @dataclass(frozen=True) @@ -324,7 +324,7 @@ def build_from_entries(avro_bytes, entries, name, settings): def _require(condition): if not condition: - raise ValueError('Invalid, unsupported, mismatched or over-budget manifest row-id block index') + raise ValueError('Invalid, unsupported, mismatched or over-budget manifest sidecar') def select(data, manifest, query, settings, partition_filter=None, partition_fields=None, bucket_filter=None): @@ -449,18 +449,18 @@ def select(data, manifest, query, settings, partition_filter=None, partition_fie return Selection(header, tuple(selected)) -def index_file_name(manifest): +def sidecar_file_name(manifest): return next((name for name in manifest.extra_files or [] if name.endswith(SUFFIX)), None) -def read_index(file_io, manifest_path, manifest, query, settings, partition_filter=None, partition_fields=None, - bucket_filter=None): - name = index_file_name(manifest) +def read_sidecar(file_io, manifest_path, manifest, query, settings, partition_filter=None, partition_fields=None, + bucket_filter=None): + name = sidecar_file_name(manifest) if name is None: return None - index_path = manifest_path.rsplit('/', 1)[0] + '/' + name + sidecar_path = manifest_path.rsplit('/', 1)[0] + '/' + name try: - with file_io.new_input_stream(index_path) as stream: + with file_io.new_input_stream(sidecar_path) as stream: data = bytearray() while True: chunk = stream.read(min(READ_BUFFER_BYTES, settings.max_bytes + 1 - len(data))) @@ -485,7 +485,7 @@ def read_index(file_io, manifest_path, manifest, query, settings, partition_filt pending.append(cause.__cause__) if cause.__context__ is not None: pending.append(cause.__context__) - LOG.debug('Cannot use row-id block index for %s; reading manifest: %s', manifest_path, error) + LOG.debug('Cannot use manifest sidecar for %s; reading manifest: %s', manifest_path, error) return None diff --git a/paimon-python/pypaimon/tests/manifest/manifest_block_index_test.py b/paimon-python/pypaimon/tests/manifest/manifest_block_index_test.py index 2d3e14d2e244..595c8a28141f 100644 --- a/paimon-python/pypaimon/tests/manifest/manifest_block_index_test.py +++ b/paimon-python/pypaimon/tests/manifest/manifest_block_index_test.py @@ -25,10 +25,10 @@ import pytest -from pypaimon.manifest.row_id_index import Builder, Settings, select +from pypaimon.manifest.manifest_sidecar import Builder, Settings, select from pypaimon.schema.data_types import AtomicType, DataField from pypaimon.table.row.generic_row import GenericRow, GenericRowSerializer -from pypaimon.tests.manifest.row_id_index_test import avro_header, golden, golden_meta +from pypaimon.tests.manifest.manifest_sidecar_test import avro_header, golden, golden_meta from pypaimon.utils.range import Range FIELDS = [DataField(0, 'p', AtomicType('INT')), DataField(1, 'q', AtomicType('STRING'))] @@ -43,7 +43,7 @@ def part(p): def fixture(key): - path = Path(__file__).resolve().parents[4] / 'paimon-core/src/test/resources/manifest-block-index.txt' + path = Path(__file__).resolve().parents[4] / 'paimon-core/src/test/resources/manifest-sidecar.txt' value = next(line.split('=', 1)[1] for line in path.read_text().splitlines() if line.startswith(key + '=')) return base64.b64decode(value) diff --git a/paimon-python/pypaimon/tests/manifest/row_id_index_test.py b/paimon-python/pypaimon/tests/manifest/manifest_sidecar_test.py similarity index 93% rename from paimon-python/pypaimon/tests/manifest/row_id_index_test.py rename to paimon-python/pypaimon/tests/manifest/manifest_sidecar_test.py index d04dac145631..591cbfad8e92 100644 --- a/paimon-python/pypaimon/tests/manifest/row_id_index_test.py +++ b/paimon-python/pypaimon/tests/manifest/manifest_sidecar_test.py @@ -33,9 +33,9 @@ from pypaimon.common.options.core_options import CoreOptions from pypaimon.globalindex.global_index_result import GlobalIndexResult -from pypaimon.manifest.row_id_index import ( - Block, Builder, Selection, Settings, SUFFIX, MAX_ROW_ID, Query, select, read_index, - read_selected_bytes, index_file_name, +from pypaimon.manifest.manifest_sidecar import ( + Block, Builder, Selection, Settings, SUFFIX, MAX_ROW_ID, Query, select, read_sidecar, + read_selected_bytes, sidecar_file_name, ) from pypaimon.manifest.schema.manifest_entry import ManifestEntry from pypaimon.manifest.manifest_list_manager import ManifestListManager @@ -50,7 +50,7 @@ def fixture(): path = (Path(__file__).resolve().parents[4] / 'paimon-core/src/test/resources' / - 'manifest-block-index.txt') + 'manifest-sidecar.txt') return dict(line.split('=', 1) for line in path.read_text().splitlines() if line.startswith(('index=', 'avroHeader='))) @@ -116,7 +116,7 @@ def close(self): raise self.failure -class RowIdIndexReadTest(unittest.TestCase): +class ManifestSidecarReadTest(unittest.TestCase): def test_index_reads_use_bounded_bulk_requests(self): header = avro_header() for block_count in (5000, 25000): @@ -133,8 +133,8 @@ def test_index_reads_use_bounded_bulk_requests(self): extra_files=['manifest-large' + SUFFIX]) stream = CountingInput(data) file_io = SimpleNamespace(new_input_stream=lambda path: stream) - actual = read_index(file_io, '/manifest/manifest-large', meta, - [Range(0, 0)], Settings()) + actual = read_sidecar(file_io, '/manifest/manifest-large', meta, + [Range(0, 0)], Settings()) self.assertEqual(actual, select(data, meta, [Range(0, 0)], Settings())) self.assertEqual(len(stream.reads), (len(data) + (1 << 20) - 1) // (1 << 20)) self.assertLessEqual(max(stream.requests), 1 << 20) @@ -148,8 +148,8 @@ def test_index_short_reads_and_exact_budget(self): stream = CountingInput(data, max_read) file_io = SimpleNamespace(new_input_stream=lambda path: stream) settings = Settings(max_bytes=len(data)) - actual = read_index(file_io, '/manifest/manifest-golden', meta, - [Range(20, 20)], settings) + actual = read_sidecar(file_io, '/manifest/manifest-golden', meta, + [Range(20, 20)], settings) self.assertEqual(actual, select(data, meta, [Range(20, 20)], settings)) self.assertTrue(stream.closed) @@ -158,8 +158,8 @@ def test_index_over_budget_stops_after_one_extra_byte(self): meta.extra_files = [meta.file_name + SUFFIX] stream = CountingInput(data) file_io = SimpleNamespace(new_input_stream=lambda path: stream) - self.assertIsNone(read_index(file_io, '/manifest/manifest-golden', meta, - [Range(20, 20)], Settings(max_bytes=128))) + self.assertIsNone(read_sidecar(file_io, '/manifest/manifest-golden', meta, + [Range(20, 20)], Settings(max_bytes=128))) self.assertEqual(stream.reads, [(0, 129)]) self.assertTrue(stream.closed) @@ -208,7 +208,7 @@ def test_block_short_reads_and_truncation(self): self.assertTrue(stream.closed) -class RowIdIndexFormatTest(unittest.TestCase): +class ManifestSidecarFormatTest(unittest.TestCase): def test_cross_language_and_block_ordinals(self): data, meta, header = golden(), golden_meta(), avro_header() for point in (0, 9, 20, 24, (1 << 32) - 2, 1 << 32, (1 << 32) + 2, @@ -321,11 +321,11 @@ def test_invalid_envelopes(self): select(data, meta, [Range(10, 10)], Settings()) -class RowIdIndexScanTest(existing.ManifestEntryIdentifierTest): +class ManifestSidecarScanTest(existing.ManifestEntryIdentifierTest): def setUp(self): super().setUp() - self.table.options.options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_WRITE, True) - self.table.options.options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_READ, True) + self.table.options.options.set(CoreOptions.MANIFEST_SIDECAR_WRITE, True) + self.table.options.options.set(CoreOptions.MANIFEST_SIDECAR_READ, True) def entry(self, name, first, count=10, kind=0): return ManifestEntry(kind, self._create_file_meta('unused').min_key, 0, 1, @@ -346,7 +346,7 @@ def test_bucket_point_lookup_with_rescale_and_delete_entries(self): metadata = self.write_meta('buckets', entries) results = [] for enabled in (False, True): - self.table.options.options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_READ, enabled) + self.table.options.options.set(CoreOptions.MANIFEST_SIDECAR_READ, enabled) scanner = FileScanner(self.table, lambda: ([metadata], None)) scanner._bucket_selector = selector with patch('pypaimon.manifest.manifest_file_manager.read_selected_bytes', @@ -373,7 +373,7 @@ def test_partition_only_and_conjunctive_planning_keep_entry_and_delete_filters(s schema = Schema.from_pyarrow_schema( pa.schema([('p', pa.int32()), ('q', pa.string()), ('value', pa.string())]), partition_keys=['p', 'q'], - options={'manifest.row-id-index.write': 'true', 'manifest.row-id-index.read': 'true'}) + options={'manifest.sidecar.write': 'true', 'manifest.sidecar.read': 'true'}) self.catalog.create_table('default.partition_block_index', schema, False) self.table = self.catalog.get_table('default.partition_block_index') self.manifest_file_manager = ManifestFileManager(self.table) @@ -389,7 +389,7 @@ def entry(name, first, p, kind=0): predicate = Predicate('equal', 0, 'p', [1]) results = [] for enabled in (False, True): - self.table.options.options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_READ, enabled) + self.table.options.options.set(CoreOptions.MANIFEST_SIDECAR_READ, enabled) scanner = FileScanner(self.table, lambda: ([manifest], None), partition_predicate=predicate) with patch('pypaimon.manifest.manifest_file_manager.read_selected_bytes', wraps=read_selected_bytes) as read_blocks: @@ -416,12 +416,12 @@ def entry(name, first, p, kind=0): def test_explicit_reference_and_null_does_not_probe(self): manager = self.manifest_file_manager written = self.write_meta('explicit', [self.entry('data.parquet', 100)]) - self.assertEqual(index_file_name(written), written.file_name + SUFFIX) - index_path = Path(manager.manifest_path, index_file_name(written)) + self.assertEqual(sidecar_file_name(written), written.file_name + SUFFIX) + index_path = Path(manager.manifest_path, sidecar_file_name(written)) explicit_path = index_path.with_name('independent-index' + SUFFIX) index_path.rename(explicit_path) other_path = index_path.with_name('other-partition-index') - other_path.write_bytes(b'not a row-id index') + other_path.write_bytes(b'not a manifest sidecar') indexed = replace(written, extra_files=[other_path.name, explicit_path.name]) with patch.object(self.table.file_io, 'new_input_stream', wraps=self.table.file_io.new_input_stream) as opened: @@ -445,14 +445,14 @@ def test_explicit_reference_and_null_does_not_probe(self): def test_manifest_list_index_reference_compatibility(self): indexed = self.write_meta('indexed', [self.entry('data.parquet', 100)]) indexed = replace(indexed, extra_files=['other-index'] + indexed.extra_files) - self.table.options.options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_WRITE, False) + self.table.options.options.set(CoreOptions.MANIFEST_SIDECAR_WRITE, False) unindexed = self.write_meta('legacy-entry', [self.entry('old.parquet', None)]) - self.assertIsNone(index_file_name(unindexed)) + self.assertIsNone(sidecar_file_name(unindexed)) lists = ManifestListManager(self.table) lists.write('references', [indexed, unindexed]) actual = lists.read('references') self.assertEqual([meta.extra_files for meta in actual], [indexed.extra_files, None]) - self.assertEqual([index_file_name(meta) for meta in actual], [index_file_name(indexed), None]) + self.assertEqual([sidecar_file_name(meta) for meta in actual], [sidecar_file_name(indexed), None]) self.assertEqual([meta.file_name for meta in actual], [indexed.file_name, unindexed.file_name]) data = Path(lists.manifest_path, 'references').read_bytes() @@ -465,7 +465,7 @@ def test_manifest_list_index_reference_compatibility(self): [indexed.file_name, unindexed.file_name]) with self.table.file_io.new_output_stream(str(Path(lists.manifest_path, 'old-list'))) as stream: fastavro.writer(stream, legacy_schema, legacy_records) - self.assertTrue(all(index_file_name(meta) is None for meta in lists.read('old-list'))) + self.assertTrue(all(sidecar_file_name(meta) is None for meta in lists.read('old-list'))) def test_skips_blocks_inside_a_matching_manifest(self): entries = [self.entry('file-%d.parquet' % i, i * 1000) for i in range(4000)] @@ -473,7 +473,7 @@ def test_skips_blocks_inside_a_matching_manifest(self): outputs = [] reader = fastavro.reader for enabled in (False, True): - self.table.options.options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_READ, enabled) + self.table.options.options.set(CoreOptions.MANIFEST_SIDECAR_READ, enabled) scanner = FileScanner(self.table, lambda: ([meta], None)) scanner.with_global_index_result(GlobalIndexResult.from_ranges([Range(2000005, 2000005)])) decoded = [] @@ -509,16 +509,16 @@ def test_actual_global_index_scanner_72_to_2(self): metas.append(self.write_meta('manifest-%d' % i, entries)) results = [] for enabled in (False, True): - self.table.options.options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_READ, enabled) + self.table.options.options.set(CoreOptions.MANIFEST_SIDECAR_READ, enabled) scanner = FileScanner(self.table, lambda: (metas, None)) scanner.with_global_index_result(GlobalIndexResult.from_ranges([Range(50, 50)])) manager = scanner.manifest_file_manager with patch.object(manager, 'read', wraps=manager.read) as read_body, \ - patch('pypaimon.manifest.manifest_file_manager.read_index', wraps=read_index) as read_sidecar: + patch('pypaimon.manifest.manifest_file_manager.read_sidecar', wraps=read_sidecar) as read_metadata: entries, _ = scanner._create_data_evolution_split_generator() results.append(sorted(e.file.file_name for e in entries)) self.assertEqual(len(read_body.call_args_list), 2 if enabled else 72) - self.assertEqual(len(read_sidecar.call_args_list), 72 if enabled else 0) + self.assertEqual(len(read_metadata.call_args_list), 72 if enabled else 0) self.assertEqual(results, [['hit.blob', 'hit.parquet']] * 2) def test_delete_union_no_resurrection_and_no_query_no_index_io(self): @@ -529,7 +529,7 @@ def test_delete_union_no_resurrection_and_no_query_no_index_io(self): self.write_meta('gap', [self.entry('lo', 0), self.entry('hi', 100)])] manager = self.manifest_file_manager for enabled in (False, True): - self.table.options.options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_READ, enabled) + self.table.options.options.set(CoreOptions.MANIFEST_SIDECAR_READ, enabled) with patch.object(manager, 'read', wraps=manager.read) as read_body: entries = manager.read_entries_parallel(metas[:2], row_ranges=[Range(50, 50)]) self.assertEqual(entries, []) @@ -552,7 +552,7 @@ def test_delete_union_no_resurrection_and_no_query_no_index_io(self): self.assertIsNone(read_body.call_args[1]['selected_blocks']) with patch.object(self.table.file_io, 'new_input_stream', side_effect=InterruptedError('stop')): with self.assertRaises(InterruptedError): - read_index(self.table.file_io, path, metas[0], [Range(0, 0)], Settings()) + read_sidecar(self.table.file_io, path, metas[0], [Range(0, 0)], Settings()) def test_sidecar_cancellation_during_open(self): self._check_sidecar_cancellation('open') @@ -601,7 +601,7 @@ def _check_sidecar_io_failure(self, phase, failure, cancelled, close_failure=Non manager = self.manifest_file_manager meta = self.write_meta('failure-' + phase + '-' + type(failure).__name__, [self.entry('data.parquet', 100)]) - index_path = str(Path(manager.manifest_path, index_file_name(meta))) + index_path = str(Path(manager.manifest_path, sidecar_file_name(meta))) body_path = str(Path(manager.manifest_path, meta.file_name)) stream = (FailingIndexInput(Path(index_path).read_bytes(), failure, phase, close_failure) if phase != 'open' else None) @@ -652,7 +652,7 @@ def test_rolling_merge_limits_and_abort_cleanup(self): for meta in outputs: self.assertTrue(Path(manager.manifest_path, meta.file_name + SUFFIX).exists()) for meta in metas: - self.assertIsNotNone(index_file_name(meta)) + self.assertIsNotNone(sidecar_file_name(meta)) manager.delete(meta) self.assertFalse(Path(manager.manifest_path, meta.file_name + SUFFIX).exists()) original = self.table.file_io.new_output_stream @@ -667,11 +667,11 @@ def fail(path): self.assertFalse(Path(manager.manifest_path, 'failed').exists()) self.assertFalse(Path(manager.manifest_path, 'failed' + SUFFIX).exists()) unknown = manager.write('unknown', [self.entry('legacy', None)]) - self.assertIsNotNone(index_file_name(unknown)) - data = Path(manager.manifest_path, index_file_name(unknown)).read_bytes() + self.assertIsNotNone(sidecar_file_name(unknown)) + data = Path(manager.manifest_path, sidecar_file_name(unknown)).read_bytes() self.assertEqual(len(select(data, unknown, [Range(100, 100)], Settings()).blocks), 1) - self.table.options.options.set(CoreOptions.MANIFEST_ROW_ID_INDEX_MAX_RANGES, 1) + self.table.options.options.set(CoreOptions.MANIFEST_SIDECAR_MAX_RANGES, 1) huge = manager.write('huge', [self.entry('one', 0, 10), self.entry('two', 100, 10)]) - self.assertIsNotNone(index_file_name(huge)) - data = Path(manager.manifest_path, index_file_name(huge)).read_bytes() + self.assertIsNotNone(sidecar_file_name(huge)) + data = Path(manager.manifest_path, sidecar_file_name(huge)).read_bytes() self.assertEqual(len(select(data, huge, [Range(50, 50)], Settings()).blocks), 1) From 06c44021c784bd85585d2c2995e8cb42bb275c68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Mon, 14 Sep 2026 11:39:10 +0800 Subject: [PATCH 09/23] [core] Unify encoding for manifest sidecar payloads --- docs/docs/concepts/spec/manifest.md | 18 +++++---- .../paimon/manifest/ManifestSidecar.java | 24 ++++++----- .../manifest/ManifestBlockIndexTest.java | 40 +++++++++++++++---- .../src/test/resources/manifest-sidecar.txt | 6 +-- .../pypaimon/manifest/manifest_sidecar.py | 25 ++++++------ .../manifest/manifest_block_index_test.py | 28 +++++++++---- 6 files changed, 91 insertions(+), 50 deletions(-) diff --git a/docs/docs/concepts/spec/manifest.md b/docs/docs/concepts/spec/manifest.md index f8467a8349fa..24fa03341b61 100644 --- a/docs/docs/concepts/spec/manifest.md +++ b/docs/docs/concepts/spec/manifest.md @@ -105,8 +105,9 @@ blocks[] // original physical order rowIdEncoding : byte rowIdPayloadLength : int rowIdPayload : bytes - bucketPayloadLength : int // -1 means null (unavailable) - bucketPayload : bytes // omitted when null + bucketEncoding : byte + bucketPayloadLength : int + bucketPayload : bytes checksum : 32 bytes // SHA-256 of all preceding bytes ``` @@ -117,10 +118,11 @@ serialized tuple. Partition predicates are evaluated once per dictionary entry. | Dimension | Encoding | Payload | | --- | --- | --- | -| Either | `0` | Unavailable; payload length must be zero. | +| Any | `0` | Unavailable; payload length must be zero. | | Partition | `1` | Positive `partitionIdCount: int` followed by sorted unique dictionary IDs (`int`). | | Row ID | `1` | Positive `rangeCount: int` followed by sorted disjoint inclusive `(start: long, end: long)` pairs. Coverage may conservatively include gaps. | -| Either | Other nonzero ID | Skip exactly the bounded payload length; treat only this dimension as unavailable. | +| Bucket | `1` | Positive `pairCount: int` followed by sorted unique `(bucket: int, totalBuckets: int)` pairs. | +| Any | Other nonzero ID | Skip exactly the bounded payload length; treat only this dimension as unavailable. | Payload lengths exclude the encoding and length fields. Invalid lengths, known-payload framing, dictionary references, interval order, checksums or physical coverage invalidate @@ -128,18 +130,18 @@ the container. Byte spans must cover the entire original manifest after its head record counts must sum to the manifest entry count. Readers continue validating blocks and known payloads even when a predicate has already rejected a block. -The nullable bucket payload contains a positive `pairCount: int` followed by that many +Bucket encoding 1 contains a positive `pairCount: int` followed by that many `(bucket: int, totalBuckets: int)` pairs. Pairs are sorted by bucket, then totalBuckets, and deduplicated. They preserve bucket-count changes between writes; the bucket number alone is not sufficient for point lookup after rescaling. A valid pair satisfies `0 <= bucket < totalBuckets`. Missing, invalid, negative/synthetic or over-budget bucket -metadata makes that block's bucket payload null. Partition and row-ID coverage remain -independently usable; no mutual-exclusion restriction is imposed. +metadata makes that block's bucket coverage unavailable (encoding 0, length 0). Partition +and row-ID coverage remain independently usable; no mutual-exclusion restriction is imposed. Readers test bucket-only queries using the existing bucket-selection logic, including the total-bucket count. Java uses conservative partition-independent bounds for `ManifestBucketFilter`; arbitrary partition-dependent callbacks remain at the entry -filter stage. A null bucket payload cannot exclude a block. Malformed payload lengths, +filter stage. An unavailable bucket payload cannot exclude a block. Malformed payload lengths, pair counts, ordering or values invalidate the container rather than excluding a block. All entries contribute, including ADD, DELETE and every file format/column group. diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java index 8bb02b5cb000..450d46f2e83f 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java @@ -68,7 +68,7 @@ public final class ManifestSidecar { private static final long MAGIC = 0x5041494d53434152L; private static final int FORMAT_VERSION = 1; private static final int HEADER_BYTES = 60; - private static final int BLOCK_BYTES = 38; + private static final int BLOCK_BYTES = 39; private static final int MAX_BLOCKS = 131072; private static final byte[] EMPTY = new byte[0]; private static final int DIGEST_BYTES = 32; @@ -461,8 +461,7 @@ public byte[] serialize(String name, long fileSize, long entryCount) throws IOEx out.writeLong(block.block.recordCount); writePayload(out, block.partitions); writePayload(out, block.rowIds); - out.writeInt(block.buckets.length == 0 ? -1 : block.buckets.length); - out.write(block.buckets); + writePayload(out, block.buckets); } out.write(digest(buffer.toByteArray())); return buffer.toByteArray(); @@ -666,21 +665,24 @@ public static Selection select( } } } + require(in.remaining() >= 5); + int bucketEncoding = Byte.toUnsignedInt(in.get()); + ByteBuffer bucketPayload = payload(in); boolean bucketHit = true; - require(in.remaining() >= 4); - int bucketLength = in.getInt(); - if (bucketLength != -1) { - require(bucketLength >= 4 && bucketLength <= in.remaining()); - int pairs = in.getInt(); + if (bucketEncoding == 0) { + require(!bucketPayload.hasRemaining()); + } else if (bucketEncoding == 1) { + require(bucketPayload.remaining() >= 4); + int pairs = bucketPayload.getInt(); require( pairs > 0 && pairs <= settings.maxBucketPairs - && bucketLength == 4L + 8L * pairs); + && bucketPayload.remaining() == 8L * pairs); bucketHit = bucketFilter == null; long previous = -1; for (int j = 0; j < pairs; j++) { - int bucket = in.getInt(); - int totalBuckets = in.getInt(); + int bucket = bucketPayload.getInt(); + int totalBuckets = bucketPayload.getInt(); require(bucket >= 0 && totalBuckets > bucket); long pair = ((long) bucket << 32) | totalBuckets; require(pair > previous); diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java index 02a71a7a37ee..5c805a6c242e 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java @@ -208,10 +208,9 @@ private List positions(byte[] data) { length = in.getInt(); in.position(in.position() + length); int bucket = in.position(); + in.get(); length = in.getInt(); - if (length >= 0) { - in.position(in.position() + length); - } + in.position(in.position() + length); result.add(new int[] {block, partition, row, bucket}); } return result; @@ -226,7 +225,7 @@ private byte[] checksum(byte[] data) throws Exception { @Test void unknownUnsignedEncodingsSkipOnlyTheirDimensionAndMalformedPayloadsFail() throws Exception { - byte[] good = fixture("indexWithPartitions"); + byte[] good = fixture("indexWithBuckets"); int[] first = positions(good).get(0); ManifestFileMeta meta = meta("manifest-golden", fixture("avroHeader").length + 400, 7); byte[] data = good.clone(); @@ -245,13 +244,36 @@ void unknownUnsignedEncodingsSkipOnlyTheirDimensionAndMalformedPayloadsFail() th .blocks()) .extracting(block -> block.firstRecord) .containsExactly(0L); - for (int position : new int[] {first[1], first[2]}) { + data = good.clone(); + data[first[3]] = (byte) 202; + // Unknown encodings skip their payload without decoding even an invalid pair count. + ByteBuffer.wrap(data).putInt(first[3] + 5, 0); + checksum(data); + BucketFilter noBucket = BucketFilter.create(false, 99, null, null); + assertThat( + ManifestSidecar.select( + data, meta, query(20), part(7), type, noBucket, defaults) + .blocks()) + .extracting(block -> block.firstRecord) + .containsExactly(0L); + assertThat( + ManifestSidecar.select( + data, meta, query(999), part(7), type, noBucket, defaults) + .blocks()) + .isEmpty(); + assertThat( + ManifestSidecar.select( + data, meta, query(20), part(99), type, noBucket, defaults) + .blocks()) + .isEmpty(); + for (int position : new int[] {first[1], first[2], first[3]}) { byte[] bad = good.clone(); bad[position] = 0; // encoding 0 cannot have payload bytes checksum(bad); assertThatThrownBy(() -> ManifestSidecar.select(bad, meta, query(0), defaults)) .isInstanceOf(IOException.class); byte[] invalidLength = good.clone(); + invalidLength[position] = (byte) 255; ByteBuffer.wrap(invalidLength).putInt(position + 1, -1); checksum(invalidLength); assertThatThrownBy( @@ -363,7 +385,7 @@ public boolean mayContain(int min, int max, int total) { } @Test - void unknownInvalidOrOverBudgetBucketPayloadIsNull() throws Exception { + void unknownInvalidOrOverBudgetBucketPayloadIsUnavailable() throws Exception { Options options = new Options(); options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BUCKET_PAIRS, 1); ManifestSidecar.Settings settings = new ManifestSidecar.Settings(options); @@ -384,7 +406,9 @@ void unknownInvalidOrOverBudgetBucketPayloadIsNull() throws Exception { builder.add(300L, 10, partition(7, "left"), 1, 4); builder.endBlock(); byte[] data = builder.serialize("m", header.length + 200, 3); - assertThat(ByteBuffer.wrap(data).getInt(positions(data).get(0)[3])).isEqualTo(-1); + int bucket = positions(data).get(0)[3]; + assertThat(data[bucket]).isZero(); + assertThat(ByteBuffer.wrap(data).getInt(bucket + 1)).isZero(); ManifestFileMeta meta = meta("m", header.length + 200, 3); assertThat( ManifestSidecar.select( @@ -415,7 +439,7 @@ void unknownInvalidOrOverBudgetBucketPayloadIsNull() throws Exception { @Test void malformedBucketPayloadInvalidatesTheContainer() throws Exception { byte[] good = fixture("indexWithBuckets"); - int payload = positions(good).get(0)[3]; + int payload = positions(good).get(0)[3] + 1; ManifestFileMeta meta = meta("manifest-golden", fixture("avroHeader").length + 400, 7); for (int[] mutation : new int[][] { diff --git a/paimon-core/src/test/resources/manifest-sidecar.txt b/paimon-core/src/test/resources/manifest-sidecar.txt index 3b01e938aace..f8a914f451d8 100644 --- a/paimon-core/src/test/resources/manifest-sidecar.txt +++ b/paimon-core/src/test/resources/manifest-sidecar.txt @@ -18,6 +18,6 @@ avroHeader=T2JqAQQUYXZyby5jb2RlYwhudWxsFmF2cm8uc2NoZW1hDCJsb25nIgAAAAAAAAAAAAAAAAAAAAAA partitionA=AAAAAgAAAAAAAAAABwAAAAAAAABsZWZ0AAAAhA== partitionB=AAAAAgACAAAAAAAACQAAAAAAAAAAAAAAAAAAAA== -index=UEFJTVNDQVIAAAABL0eWubkHdL6ioNdIPj3jGtp9TAk83wbtAsmc+xJVwM8AAAAAAAAByQAAAAAAAAAHAAAAOU9iagEEFGF2cm8uY29kZWMIbnVsbBZhdnJvLnNjaGVtYQwibG9uZyIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAADkAAAAAAAAAZAAAAAAAAAADAAAAAAABAAAAJAAAAAIAAAAAAAAAAAAAAAAAAAAJAAAAAAAAABQAAAAAAAAAGP////8AAAAAAAAAnQAAAAAAAADIAAAAAAAAAAIAAAAAAAEAAAAkAAAAAgAAAAD////+AAAAAQAAAAIAAAeBzDhsZQAAB4HMOGxl/////wAAAAAAAAFlAAAAAAAAAGQAAAAAAAAAAgAAAAAAAQAAACQAAAACAAAAAAAAABQAAAAAAAAAGH//////////f///////////////NP18llqvAb/4bxohLXD7cDFPAPeAb5ypZyhxRnTV1o8= -indexWithPartitions=UEFJTVNDQVIAAAABL0eWubkHdL6ioNdIPj3jGtp9TAk83wbtAsmc+xJVwM8AAAAAAAAByQAAAAAAAAAHAAAAOU9iagEEFGF2cm8uY29kZWMIbnVsbBZhdnJvLnNjaGVtYQwibG9uZyIAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAcAAAAAgAAAAAAAAAABwAAAAAAAABsZWZ0AAAAhAAAABwAAAACAAIAAAAAAAAJAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAA5AAAAAAAAAGQAAAAAAAAAAwEAAAAMAAAAAgAAAAAAAAABAQAAACQAAAACAAAAAAAAAAAAAAAAAAAACQAAAAAAAAAUAAAAAAAAABj/////AAAAAAAAAJ0AAAAAAAAAyAAAAAAAAAACAQAAAAwAAAACAAAAAAAAAAEBAAAAJAAAAAIAAAAA/////gAAAAEAAAACAAAHgcw4bGUAAAeBzDhsZf////8AAAAAAAABZQAAAAAAAABkAAAAAAAAAAIBAAAADAAAAAIAAAAAAAAAAQEAAAAkAAAAAgAAAAAAAAAUAAAAAAAAABh//////////3///////////////yuqJytq34abij//eVySjsXqKxL4O5cNwbD/cAdeORFh -indexWithBuckets=UEFJTVNDQVIAAAABL0eWubkHdL6ioNdIPj3jGtp9TAk83wbtAsmc+xJVwM8AAAAAAAAByQAAAAAAAAAHAAAAOU9iagEEFGF2cm8uY29kZWMIbnVsbBZhdnJvLnNjaGVtYQwibG9uZyIAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAcAAAAAgAAAAAAAAAABwAAAAAAAABsZWZ0AAAAhAAAABwAAAACAAIAAAAAAAAJAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAA5AAAAAAAAAGQAAAAAAAAAAwEAAAAMAAAAAgAAAAAAAAABAQAAACQAAAACAAAAAAAAAAAAAAAAAAAACQAAAAAAAAAUAAAAAAAAABgAAAAUAAAAAgAAAAEAAAAEAAAAAQAAAAgAAAAAAAAAnQAAAAAAAADIAAAAAAAAAAIBAAAADAAAAAIAAAAAAAAAAQEAAAAkAAAAAgAAAAD////+AAAAAQAAAAIAAAeBzDhsZQAAB4HMOGxlAAAAFAAAAAIAAAACAAAABAAAAAIAAAAIAAAAAAAAAWUAAAAAAAAAZAAAAAAAAAACAQAAAAwAAAACAAAAAAAAAAEBAAAAJAAAAAIAAAAAAAAAFAAAAAAAAAAYf/////////9//////////wAAABQAAAACAAAAAAAAAAEAAAADAAAABExlWmcJvax6T8wD69ZnRY9t/6uzT9JbUsxIZ4DoIV/4 +index=UEFJTVNDQVIAAAABL0eWubkHdL6ioNdIPj3jGtp9TAk83wbtAsmc+xJVwM8AAAAAAAAByQAAAAAAAAAHAAAAOU9iagEEFGF2cm8uY29kZWMIbnVsbBZhdnJvLnNjaGVtYQwibG9uZyIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAADkAAAAAAAAAZAAAAAAAAAADAAAAAAABAAAAJAAAAAIAAAAAAAAAAAAAAAAAAAAJAAAAAAAAABQAAAAAAAAAGAAAAAAAAAAAAAAAAJ0AAAAAAAAAyAAAAAAAAAACAAAAAAABAAAAJAAAAAIAAAAA/////gAAAAEAAAACAAAHgcw4bGUAAAeBzDhsZQAAAAAAAAAAAAAAAWUAAAAAAAAAZAAAAAAAAAACAAAAAAABAAAAJAAAAAIAAAAAAAAAFAAAAAAAAAAYf/////////9//////////wAAAAAAvl+JhwByNVfzaEZmgXTZEMYB85Mh4bR1hf61USimKec= +indexWithPartitions=UEFJTVNDQVIAAAABL0eWubkHdL6ioNdIPj3jGtp9TAk83wbtAsmc+xJVwM8AAAAAAAAByQAAAAAAAAAHAAAAOU9iagEEFGF2cm8uY29kZWMIbnVsbBZhdnJvLnNjaGVtYQwibG9uZyIAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAcAAAAAgAAAAAAAAAABwAAAAAAAABsZWZ0AAAAhAAAABwAAAACAAIAAAAAAAAJAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAA5AAAAAAAAAGQAAAAAAAAAAwEAAAAMAAAAAgAAAAAAAAABAQAAACQAAAACAAAAAAAAAAAAAAAAAAAACQAAAAAAAAAUAAAAAAAAABgAAAAAAAAAAAAAAACdAAAAAAAAAMgAAAAAAAAAAgEAAAAMAAAAAgAAAAAAAAABAQAAACQAAAACAAAAAP////4AAAABAAAAAgAAB4HMOGxlAAAHgcw4bGUAAAAAAAAAAAAAAAFlAAAAAAAAAGQAAAAAAAAAAgEAAAAMAAAAAgAAAAAAAAABAQAAACQAAAACAAAAAAAAABQAAAAAAAAAGH//////////f/////////8AAAAAACeaHd6e41hdWskZ6DqWibQKE2TYT04EVjoN1YBmh9VF +indexWithBuckets=UEFJTVNDQVIAAAABL0eWubkHdL6ioNdIPj3jGtp9TAk83wbtAsmc+xJVwM8AAAAAAAAByQAAAAAAAAAHAAAAOU9iagEEFGF2cm8uY29kZWMIbnVsbBZhdnJvLnNjaGVtYQwibG9uZyIAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAcAAAAAgAAAAAAAAAABwAAAAAAAABsZWZ0AAAAhAAAABwAAAACAAIAAAAAAAAJAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAA5AAAAAAAAAGQAAAAAAAAAAwEAAAAMAAAAAgAAAAAAAAABAQAAACQAAAACAAAAAAAAAAAAAAAAAAAACQAAAAAAAAAUAAAAAAAAABgBAAAAFAAAAAIAAAABAAAABAAAAAEAAAAIAAAAAAAAAJ0AAAAAAAAAyAAAAAAAAAACAQAAAAwAAAACAAAAAAAAAAEBAAAAJAAAAAIAAAAA/////gAAAAEAAAACAAAHgcw4bGUAAAeBzDhsZQEAAAAUAAAAAgAAAAIAAAAEAAAAAgAAAAgAAAAAAAABZQAAAAAAAABkAAAAAAAAAAIBAAAADAAAAAIAAAAAAAAAAQEAAAAkAAAAAgAAAAAAAAAUAAAAAAAAABh//////////3//////////AQAAABQAAAACAAAAAAAAAAEAAAADAAAABGCAy0ixcqKywlCau1+E/XLEorySmCmL1zl5q9/dwdBu diff --git a/paimon-python/pypaimon/manifest/manifest_sidecar.py b/paimon-python/pypaimon/manifest/manifest_sidecar.py index 44cd7e3f39a0..c4b72b45b871 100644 --- a/paimon-python/pypaimon/manifest/manifest_sidecar.py +++ b/paimon-python/pypaimon/manifest/manifest_sidecar.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -"""Independent partition, row-id and nullable bucket coverage for each Avro block.""" +"""Independent partition, row-id and bucket coverage for each Avro block.""" import hashlib import logging @@ -42,7 +42,7 @@ HEADER = struct.Struct('>8sI32sqq') BLOCK = struct.Struct('>qqq') MAX_BLOCKS = 131072 -BLOCK_BYTES = 38 +BLOCK_BYTES = 39 PAIR = struct.Struct('>qq') LONG = struct.Struct('>q') _PROPAGATED_ERRORS = (InterruptedError, CancelledError, ArrowCancelled, MemoryError, RecursionError) @@ -290,11 +290,9 @@ def serialize(self, name, file_size, entry_count): data.extend(struct.pack('>I', len(self.blocks))) for block, partitions, row_ids, buckets in self.blocks: data.extend(BLOCK.pack(block.offset, block.length, block.record_count)) - for payload in (partitions, row_ids): + for payload in (partitions, row_ids, buckets): data.extend(struct.pack('>BI', 1 if payload else 0, len(payload))) data.extend(payload) - data.extend(struct.pack('>i', len(buckets) if buckets else -1)) - data.extend(buckets) return bytes(data) + hashlib.sha256(data).digest() @@ -424,14 +422,17 @@ def select(data, manifest, query, settings, partition_filter=None, partition_fie if candidate and not row_hit: row_hit = ranges == 1 or query.intersects(start, end) offset += payload_length + _require(offset + 5 <= limit) + bucket_encoding, payload_length = struct.unpack_from('>BI', data, offset) + offset += 5 + _require(payload_length <= limit - offset) bucket_hit = True - _require(offset + 4 <= limit) - bucket_length, = struct.unpack_from('>i', data, offset) - offset += 4 - if bucket_length != -1: - _require(4 <= bucket_length <= limit - offset) + if bucket_encoding == 0: + _require(payload_length == 0) + elif bucket_encoding == 1: + _require(payload_length >= 4) pairs, = struct.unpack_from('>I', data, offset) - _require(0 < pairs <= settings.max_bucket_pairs and bucket_length == 4 + 8 * pairs) + _require(0 < pairs <= settings.max_bucket_pairs and payload_length == 4 + 8 * pairs) bucket_hit = bucket_filter is None previous = (-1, -1) for j in range(pairs): @@ -440,7 +441,7 @@ def select(data, manifest, query, settings, partition_filter=None, partition_fie previous = (bucket, total_buckets) if not bucket_hit: bucket_hit = bucket_filter(bucket, total_buckets) - offset += bucket_length + offset += payload_length if partition_hit and row_hit and bucket_hit: selected.append(Block(file_offset, length, first_record, count)) next_offset = file_offset + length diff --git a/paimon-python/pypaimon/tests/manifest/manifest_block_index_test.py b/paimon-python/pypaimon/tests/manifest/manifest_block_index_test.py index 595c8a28141f..0efad7e1c33e 100644 --- a/paimon-python/pypaimon/tests/manifest/manifest_block_index_test.py +++ b/paimon-python/pypaimon/tests/manifest/manifest_block_index_test.py @@ -128,8 +128,8 @@ def positions(data): r = p + 5 + struct.unpack_from('>I', data, p + 1)[0] offset = r + 5 + struct.unpack_from('>I', data, r + 1)[0] bucket = offset - length, = struct.unpack_from('>i', data, offset) - offset += 4 + max(0, length) + length, = struct.unpack_from('>I', data, offset + 1) + offset += 5 + length result.append((block, p, r, bucket)) return result @@ -140,8 +140,8 @@ def checksum(data): def test_unsigned_unknown_encodings_skip_only_one_payload_and_validate_lengths(): - good = fixture('indexWithPartitions') - block, p, r, _ = positions(good)[0] + good = fixture('indexWithBuckets') + block, p, r, b = positions(good)[0] data = bytearray(good) data[p] = 200 selected = select(checksum(data), golden_meta(), [Range(0, 0)], Settings(), part(99), FIELDS) @@ -150,11 +150,23 @@ def test_unsigned_unknown_encodings_skip_only_one_payload_and_validate_lengths() data[r] = 201 selected = select(checksum(data), golden_meta(), [Range(16, 16)], Settings(), part(7), FIELDS) assert [b.first_record for b in selected.blocks] == [0] - for position in (p, r): + data = bytearray(good) + data[b] = 202 + # Skip unknown payloads without decoding even an invalid pair count. + struct.pack_into('>I', data, b + 5, 0) + checksum(data) + no_bucket = lambda bucket, total: False + selected = select(data, golden_meta(), [Range(20, 20)], Settings(), part(7), FIELDS, no_bucket) + assert [block.first_record for block in selected.blocks] == [0] + assert not select(data, golden_meta(), [Range(999, 999)], Settings(), part(7), FIELDS, no_bucket).blocks + assert not select(data, golden_meta(), [Range(20, 20)], Settings(), part(99), FIELDS, no_bucket).blocks + for position in (p, r, b): data = bytearray(good) data[position] = 0 with pytest.raises(ValueError): select(checksum(data), golden_meta(), None, Settings()) + data = bytearray(good) + data[position] = 255 struct.pack_into('>i', data, position + 1, -1) with pytest.raises(ValueError): select(checksum(data), golden_meta(), None, Settings()) @@ -221,7 +233,7 @@ def test_randomized_budget_degradation_has_no_false_negatives(): assert i * 6 in ordinals -def test_bucket_payload_golden_rescale_and_nullable_payloads(): +def test_bucket_payload_golden_rescale_and_unavailable_payloads(): a, b, header = partition(7, 'left'), partition(9, None), avro_header() builder = Builder(Settings(), header) for offset, length, values in [ @@ -257,7 +269,7 @@ def test_bucket_budget_and_unknown_pairs_degrade_only_bucket_payload(pair): builder.add(300, 10, partition(7, 'left'), 1, 4) builder.end_block() data = builder.serialize('m', len(header) + 200, 3) - assert struct.unpack_from('>i', data, positions(data)[0][3])[0] == -1 + assert struct.unpack_from('>BI', data, positions(data)[0][3]) == (0, 0) metadata = meta('m', len(header) + 200, 3) selected = select(data, metadata, None, settings, bucket_filter=lambda bucket, total: False) assert [b.first_record for b in selected.blocks] == [0] @@ -266,7 +278,7 @@ def test_bucket_budget_and_unknown_pairs_degrade_only_bucket_payload(pair): def test_malformed_bucket_payload_invalidates_the_container(): good = fixture('indexWithBuckets') - payload = positions(good)[0][3] + payload = positions(good)[0][3] + 1 for offset, value in [(payload, -2), (payload, (1 << 31) - 1), (payload, 0), (payload + 4, 0), (payload + 8, -1), (payload + 12, 1), (payload + 16, 0)]: bad = bytearray(good) From 95620cb8a25153c28eb98263bdba6595ada68f86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Mon, 14 Sep 2026 12:07:52 +0800 Subject: [PATCH 10/23] [core] Preserve manifest cache and explain statistics with sidecars --- docs/docs/concepts/spec/manifest.md | 3 ++ .../apache/paimon/manifest/ManifestFile.java | 2 +- .../paimon/manifest/ManifestSidecar.java | 10 +++- .../paimon/manifest/ManifestFileTest.java | 46 +++++++++++++++++-- .../manifest/manifest_file_manager.py | 7 +-- .../pypaimon/read/scanner/file_scanner.py | 5 +- .../tests/manifest/manifest_sidecar_test.py | 38 +++++++++++++++ 7 files changed, 97 insertions(+), 14 deletions(-) diff --git a/docs/docs/concepts/spec/manifest.md b/docs/docs/concepts/spec/manifest.md index 24fa03341b61..b024d3d0dcff 100644 --- a/docs/docs/concepts/spec/manifest.md +++ b/docs/docs/concepts/spec/manifest.md @@ -174,6 +174,9 @@ therefore reads row-ID payload bytes too; payload lengths save decoding work for encodings, not storage I/O. Selected compressed blocks are read by byte range with adjacent spans coalesced. Existing immutable manifests are not backfilled by enabling the write option. +Java selections covering every block can reuse the full-manifest cache; partial selections +bypass it. PyPaimon explain scans disable sidecar pruning to preserve complete entry counters. + Selected blocks still pass through entry filtering and ADD/DELETE reconciliation. Snapshot, tag, changelog, orphan-file and failed-commit cleanup retain or remove the sidecar through its extra-file reference together with the owning manifest. diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java index 94498fc0fa40..3fe5609a5671 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java @@ -167,7 +167,7 @@ public List read( try { Path path = pathFactory.toPath(fileName); // A partial manifest must never enter the cache under the full manifest's key. - if (cache != null && selected == null) { + if (cache != null && (selected == null || selected.isFullManifest())) { ManifestEntryFilters filters = new ManifestEntryFilters( partitionFilter, bucketFilter, readFilter, readTFilter); diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java index 450d46f2e83f..d20cba79f1eb 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java @@ -148,15 +148,21 @@ public Block(long offset, long length, long firstRecord, long recordCount) { public static final class Selection { private final byte[] header; private final List blocks; + private final boolean fullManifest; - private Selection(byte[] header, List blocks) { + private Selection(byte[] header, List blocks, boolean fullManifest) { this.header = header; this.blocks = Collections.unmodifiableList(blocks); + this.fullManifest = fullManifest; } public List blocks() { return blocks; } + + boolean isFullManifest() { + return fullManifest; + } } /** @@ -699,7 +705,7 @@ public static Selection select( firstRecord += records; } require(!in.hasRemaining() && nextOffset == manifest.fileSize() && firstRecord == entries); - return new Selection(header, selected); + return new Selection(header, selected, selected.size() == count); } private static ByteBuffer payload(ByteBuffer in) throws IOException { diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java index 8fab7489d7bd..2357c5b9c044 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java @@ -1366,7 +1366,12 @@ void testReadsOnlySelectedBlocksAndPreservesPhysicalOrdinals() throws Exception options.set(CoreOptions.MANIFEST_SIDECAR_READ, true); RecordingFileIO fileIO = new RecordingFileIO(); ManifestFile.Factory factory = - createManifestFileFactory(tempDir.toString(), Long.MAX_VALUE, options, fileIO); + createManifestFileFactory( + tempDir.toString(), + Long.MAX_VALUE, + options, + fileIO, + new SegmentsCache<>(1024, MemorySize.ofMebiBytes(16), Long.MAX_VALUE)); ManifestFile manifests = factory.create(); List entries = new ArrayList<>(); for (int i = 0; i < 4000; i++) { @@ -1418,8 +1423,43 @@ void testReadsOnlySelectedBlocksAndPreservesPhysicalOrdinals() throws Exception .collect(Collectors.toList())); assertThat(fileIO.opened) .containsExactly(new Path(tempDir.toString(), "manifest/" + meta.fileName())); - // Block selections cannot populate the ordinary full-manifest read cache. - assertThat(manifests.read(meta.fileName())).containsExactlyElementsOf(entries); + ManifestSidecar.Selection allBlocks = + manifests.selectBlocks( + meta, + RowRangeIndex.create( + Collections.singletonList(new Range(0, Long.MAX_VALUE)))); + fileIO.reset(); + assertThat( + factory.create() + .read( + meta.fileName(), + meta.fileSize(), + null, + null, + row -> true, + entry -> true, + java.util.function.Function.identity(), + allBlocks)) + .containsExactlyInAnyOrderElementsOf(entries); + // The preceding partial read must not populate the full-manifest cache. + assertThat(fileIO.opened) + .containsExactly(new Path(tempDir.toString(), "manifest/" + meta.fileName())); + + fileIO.reset(); + assertThat( + factory.create() + .read( + meta.fileName(), + meta.fileSize(), + null, + null, + row -> true, + entry -> true, + java.util.function.Function.identity(), + allBlocks)) + .containsExactlyInAnyOrderElementsOf(entries); + assertThat(manifests.read(meta.fileName())).containsExactlyInAnyOrderElementsOf(entries); + assertThat(fileIO.opened).isEmpty(); } @Test diff --git a/paimon-python/pypaimon/manifest/manifest_file_manager.py b/paimon-python/pypaimon/manifest/manifest_file_manager.py index bf7660923833..bd2cb09cfccd 100644 --- a/paimon-python/pypaimon/manifest/manifest_file_manager.py +++ b/paimon-python/pypaimon/manifest/manifest_file_manager.py @@ -62,21 +62,18 @@ def read_entries_parallel(self, manifest_files: List[ManifestFileMeta], manifest early_record_filter: Optional[Callable[[dict], bool]] = None, partition_filter=None, row_ranges=None, - index_partition_filter=None, ) -> List[ManifestEntry]: settings = Settings.from_options(self.table.options) query = Query(row_ranges) if settings.read and row_ranges is not None else None - if index_partition_filter is None: - index_partition_filter = partition_filter def _process_single_manifest(manifest_file: ManifestFileMeta): path = f"{self.manifest_path}/{manifest_file.file_name}" selected = None - if settings.read and (query is not None or index_partition_filter is not None + if settings.read and (query is not None or partition_filter is not None or early_entry_filter is not None): selected = read_sidecar(self.file_io, path, manifest_file, query, settings, - index_partition_filter, self.partition_keys_fields, early_entry_filter) + partition_filter, self.partition_keys_fields, early_entry_filter) if selected is not None and not selected.blocks: return [] return self.read( diff --git a/paimon-python/pypaimon/read/scanner/file_scanner.py b/paimon-python/pypaimon/read/scanner/file_scanner.py index 134a1e1c7a20..0c794d678dee 100755 --- a/paimon-python/pypaimon/read/scanner/file_scanner.py +++ b/paimon-python/pypaimon/read/scanner/file_scanner.py @@ -583,7 +583,7 @@ def read_manifest_entries(self, manifest_files: List[ManifestFileMeta], self.scan_stats.manifest_files_after_partition += len(manifest_files) # Force single-threaded so we can mutate stats without locking. max_workers = 1 - # Disable both early filters in explain mode (scan_stats) so all entries + # Disable early entry filters and sidecar pruning in explain mode so all entries # flow through _filter_manifest_entry for accurate funnel counting. early_row_filter = None if self.scan_stats is not None \ else _build_early_row_range_filter(row_ranges) @@ -597,8 +597,7 @@ def read_manifest_entries(self, manifest_files: List[ManifestFileMeta], early_entry_filter=self._build_early_bucket_filter(), early_record_filter=early_row_filter, partition_filter=partition_filter, - row_ranges=row_ranges, - index_partition_filter=self.partition_key_predicate, + row_ranges=row_ranges if self.scan_stats is None else None, ) def _build_early_bucket_filter(self): diff --git a/paimon-python/pypaimon/tests/manifest/manifest_sidecar_test.py b/paimon-python/pypaimon/tests/manifest/manifest_sidecar_test.py index 591cbfad8e92..531390893272 100644 --- a/paimon-python/pypaimon/tests/manifest/manifest_sidecar_test.py +++ b/paimon-python/pypaimon/tests/manifest/manifest_sidecar_test.py @@ -42,6 +42,7 @@ from pypaimon.manifest.manifest_file_manager import ManifestFileManager from pypaimon.manifest.schema.manifest_file_meta import MANIFEST_FILE_META_SCHEMA from pypaimon.read.scanner.file_scanner import FileScanner +from pypaimon.read.scan_stats import ScanStats from pypaimon.tests.manifest import manifest_entry_identifier_test as existing from pypaimon.schema.schema import Schema from pypaimon.table.row.generic_row import GenericRow @@ -413,6 +414,43 @@ def entry(name, first, p, kind=0): metas = [self.write_meta('adds', add), self.write_meta('deletes', [replace(e, kind=1) for e in add])] self.assertEqual(scanner.read_manifest_entries(metas, row_ranges=[Range(105, 105)]), []) + def test_explain_keeps_complete_entry_counts_with_sidecar_enabled(self): + import pyarrow as pa + from pypaimon.common.predicate import Predicate + + schema = Schema.from_pyarrow_schema( + pa.schema([('p', pa.int32()), ('q', pa.string()), ('value', pa.string())]), + partition_keys=['p', 'q'], + options={'manifest.sidecar.write': 'true', 'manifest.sidecar.read': 'true'}) + self.catalog.create_table('default.explain_sidecar', schema, False) + self.table = self.catalog.get_table('default.explain_sidecar') + self.manifest_file_manager = ManifestFileManager(self.table) + fields = self.table.partition_keys_fields + entries = [replace(self.entry('part-%d.parquet' % i, i * 1000), + partition=GenericRow([i // 1000, None], fields)) for i in range(4000)] + manifest = self.write_meta('explain-manifest', entries) + predicate = Predicate('equal', 0, 'p', [1]) + ranges = [Range(1000000, 1000000)] + for partition_filter, row_ranges in [(predicate, None), (None, ranges), (predicate, ranges)]: + results = [] + for enabled in (False, True): + with self.subTest(partition=partition_filter, row_ranges=row_ranges, sidecar=enabled): + self.table.options.options.set(CoreOptions.MANIFEST_SIDECAR_READ, enabled) + scanner = FileScanner(self.table, lambda: ([manifest], None), + partition_predicate=partition_filter) + scanner.scan_stats = ScanStats() + with patch('pypaimon.manifest.manifest_file_manager.read_sidecar', + wraps=read_sidecar) as read_metadata: + actual = scanner.read_manifest_entries([manifest], row_ranges=row_ranges) + read_metadata.assert_not_called() + stats = scanner.scan_stats + self.assertEqual(stats.entries_potential_total, 4000) + self.assertEqual(stats.entries_total, 4000) + self.assertEqual(stats.entries_after_partition, 1000 if partition_filter else 4000) + self.assertEqual(stats.partition_keys_before, {(p, None) for p in range(4)}) + results.append([entry.file.file_name for entry in actual]) + self.assertEqual(results[0], results[1]) + def test_explicit_reference_and_null_does_not_probe(self): manager = self.manifest_file_manager written = self.write_meta('explicit', [self.entry('data.parquet', 100)]) From 533ebe000ed2bb2a069386a8489a19f0361abe41 Mon Sep 17 00:00:00 2001 From: YeJunHao <41894543+leaves12138@users.noreply.github.com> Date: Mon, 14 Sep 2026 14:22:12 +0800 Subject: [PATCH 11/23] [core] Avoid redundant manifest sidecar interval decoding Reuse the first row-id interval in Java and Python selectors while preserving complete validation and fallback for malformed sidecars. Add boundary, empty-query, corruption and decode-count regression tests. --- .../paimon/manifest/ManifestSidecar.java | 17 ++-- .../paimon/manifest/ManifestSidecarTest.java | 78 ++++++++++++++----- .../pypaimon/manifest/manifest_sidecar.py | 16 ++-- .../tests/manifest/manifest_sidecar_test.py | 37 +++++++-- 4 files changed, 108 insertions(+), 40 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java index d20cba79f1eb..5bcc03f77e83 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java @@ -655,19 +655,22 @@ public static Selection select( && ranges <= settings.maxRanges - totalRanges && rowPayload.remaining() == 16L * ranges); totalRanges += ranges; - long min = rowPayload.getLong(rowPayload.position()); - long max = rowPayload.getLong(rowPayload.limit() - 8); - require(min >= 0 && max >= min); + long min = rowPayload.getLong(); + long firstEnd = rowPayload.getLong(); + long max = ranges == 1 ? firstEnd : rowPayload.getLong(rowPayload.limit() - 8); + require(min >= 0 && firstEnd >= min && max >= firstEnd); boolean candidate = query == null || query.intersects(min, max); - rowHit = query == null; - long previous = -1; - for (int j = 0; j < ranges; j++) { + rowHit = + query == null + || (candidate && (ranges == 1 || query.intersects(min, firstEnd))); + long previous = firstEnd; + for (int rangeIndex = 1; rangeIndex < ranges; rangeIndex++) { long start = rowPayload.getLong(); long end = rowPayload.getLong(); require(start >= 0 && end >= start && start > previous); previous = end; if (candidate && !rowHit) { - rowHit = ranges == 1 || query.intersects(start, end); + rowHit = query.intersects(start, end); } } } diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java index 159987f7e72c..f43824e219de 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java @@ -187,29 +187,69 @@ void minMaxSkipsExactIntersectionChecksAndHandlesOneInterval() throws Exception verify(one, times(3)).intersects(anyLong(), anyLong()); } + @Test + void singleIntervalHandlesBoundariesAndAbsentQueries() throws Exception { + byte[] header = header(); + for (Range range : + Arrays.asList( + new Range(0, 0), + new Range(42, 51), + new Range(Long.MAX_VALUE, Long.MAX_VALUE))) { + ManifestSidecar.Builder builder = new ManifestSidecar.Builder(settings, header); + builder.beginBlock(header.length, 100, 1); + builder.add(range.from, range.to - range.from + 1); + builder.endBlock(); + byte[] data = builder.serialize("m", header.length + 100, 1); + ManifestFileMeta meta = meta("m", header.length + 100, 1); + assertThat(ManifestSidecar.select(data, meta, null, settings).blocks()).hasSize(1); + assertThat( + ManifestSidecar.select( + data, + meta, + RowRangeIndex.create(Collections.emptyList()), + settings) + .blocks()) + .isEmpty(); + for (long point : new long[] {range.from, range.to}) { + RowRangeIndex query = + spy( + RowRangeIndex.create( + Collections.singletonList(new Range(point, point)))); + assertThat(ManifestSidecar.select(data, meta, query, settings).blocks()).hasSize(1); + verify(query).intersects(range.from, range.to); + } + long missing = range.from > 0 ? range.from - 1 : range.to + 1; + assertThat(select(data, meta, missing).blocks()).isEmpty(); + } + } + @Test void malformedIntervalsStillFallbackAfterMinMaxRejectionOrAnEarlyHit() throws Exception { - byte[] data = golden(); int firstBlockIntervals = 60 + 4 + header().length + 4 + 4 + 24 + 5 + 5 + 4; - // Make the second interval overlap the first, keeping the envelope unchanged. - ByteBuffer.wrap(data).putLong(firstBlockIntervals + 16, 9L); - byte[] hash = - MessageDigest.getInstance("SHA-256").digest(Arrays.copyOf(data, data.length - 32)); - System.arraycopy(hash, 0, data, data.length - 32, 32); - Files.write(temp.resolve("manifest-golden" + ManifestSidecar.SUFFIX), data); ManifestFileMeta meta = goldenMeta(); - - for (long point : new long[] {30, 0}) { - RowRangeIndex query = - RowRangeIndex.create(Collections.singletonList(new Range(point, point))); - assertThat( - ManifestSidecar.read( - LocalFileIO.create(), - new Path(temp.toString(), "manifest-golden"), - meta, - query, - settings)) - .isNull(); + for (long[] mutation : new long[][] {{0, -1}, {8, -1}, {8, 30}, {16, 9}, {24, 19}}) { + byte[] data = golden(); + ByteBuffer.wrap(data).putLong(firstBlockIntervals + (int) mutation[0], mutation[1]); + byte[] hash = + MessageDigest.getInstance("SHA-256") + .digest(Arrays.copyOf(data, data.length - 32)); + System.arraycopy(hash, 0, data, data.length - 32, 32); + Files.write(temp.resolve("manifest-golden" + ManifestSidecar.SUFFIX), data); + for (RowRangeIndex query : + Arrays.asList( + RowRangeIndex.create(Collections.singletonList(new Range(30, 30))), + RowRangeIndex.create(Collections.singletonList(new Range(0, 0))), + RowRangeIndex.create(Collections.emptyList()), + null)) { + assertThat( + ManifestSidecar.read( + LocalFileIO.create(), + new Path(temp.toString(), "manifest-golden"), + meta, + query, + settings)) + .isNull(); + } } } diff --git a/paimon-python/pypaimon/manifest/manifest_sidecar.py b/paimon-python/pypaimon/manifest/manifest_sidecar.py index c4b72b45b871..5a26b4751646 100644 --- a/paimon-python/pypaimon/manifest/manifest_sidecar.py +++ b/paimon-python/pypaimon/manifest/manifest_sidecar.py @@ -409,18 +409,18 @@ def select(data, manifest, query, settings, partition_filter=None, partition_fie ranges, = struct.unpack_from('>I', data, offset) _require(0 < ranges <= settings.max_ranges - total_ranges and 4 + 16 * ranges == payload_length) total_ranges += ranges - min_row_id, = LONG.unpack_from(data, offset + 4) - max_row_id, = LONG.unpack_from(data, offset + payload_length - 8) - _require(min_row_id >= 0 and max_row_id >= min_row_id) + min_row_id, first_end = PAIR.unpack_from(data, offset + 4) + max_row_id = first_end if ranges == 1 else LONG.unpack_from(data, offset + payload_length - 8)[0] + _require(min_row_id >= 0 and first_end >= min_row_id and max_row_id >= first_end) candidate = query is None or query.intersects(min_row_id, max_row_id) - row_hit = query is None - previous = -1 - for j in range(ranges): - start, end = PAIR.unpack_from(data, offset + 4 + 16 * j) + row_hit = query is None or (candidate and (ranges == 1 or query.intersects(min_row_id, first_end))) + previous = first_end + for range_position in range(1, ranges): + start, end = PAIR.unpack_from(data, offset + 4 + 16 * range_position) _require(start >= 0 and end >= start and start > previous) previous = end if candidate and not row_hit: - row_hit = ranges == 1 or query.intersects(start, end) + row_hit = query.intersects(start, end) offset += payload_length _require(offset + 5 <= limit) bucket_encoding, payload_length = struct.unpack_from('>BI', data, offset) diff --git a/paimon-python/pypaimon/tests/manifest/manifest_sidecar_test.py b/paimon-python/pypaimon/tests/manifest/manifest_sidecar_test.py index 531390893272..4bc274571f49 100644 --- a/paimon-python/pypaimon/tests/manifest/manifest_sidecar_test.py +++ b/paimon-python/pypaimon/tests/manifest/manifest_sidecar_test.py @@ -33,6 +33,7 @@ from pypaimon.common.options.core_options import CoreOptions from pypaimon.globalindex.global_index_result import GlobalIndexResult +from pypaimon.manifest import manifest_sidecar from pypaimon.manifest.manifest_sidecar import ( Block, Builder, Selection, Settings, SUFFIX, MAX_ROW_ID, Query, select, read_sidecar, read_selected_bytes, sidecar_file_name, @@ -264,14 +265,38 @@ def test_minmax_skips_exact_checks_and_one_interval_is_already_exact(self): check.assert_any_call(100, 209) check.assert_any_call(1 << 32, (1 << 32) + 9) + def test_single_interval_decodes_pair_once(self): + header = avro_header() + for first, count in [(0, 1), (42, 10), (MAX_ROW_ID, 1)]: + builder = Builder(Settings(), header) + builder.begin_block(len(header), 100, 1) + builder.add(first, count) + builder.end_block() + data = builder.serialize('m', len(header) + 100, 1) + meta = SimpleNamespace(file_name='m', file_size=len(header) + 100, + num_added_files=1, num_deleted_files=0) + last = first + count - 1 + missing = first - 1 if first > 0 else last + 1 + for ranges, expected in [(None, 1), ([], 0), ([Range(first, first)], 1), + ([Range(last, last)], 1), ([Range(missing, missing)], 0)]: + with self.subTest(first=first, count=count, ranges=ranges), \ + patch.object(manifest_sidecar, 'PAIR', wraps=manifest_sidecar.PAIR) as pairs, \ + patch.object(manifest_sidecar, 'LONG', wraps=manifest_sidecar.LONG) as bounds: + selected = select(data, meta, ranges, Settings()) + self.assertEqual(len(selected.blocks), expected) + pairs.unpack_from.assert_called_once() + bounds.unpack_from.assert_not_called() + def test_rejected_and_early_hit_blocks_still_validate_every_interval(self): - data = bytearray(golden()) first_block_intervals = 60 + 4 + len(avro_header()) + 4 + 4 + 24 + 5 + 5 + 4 - struct.pack_into('>q', data, first_block_intervals + 16, 9) - data[-32:] = hashlib.sha256(data[:-32]).digest() - for point in (30, 0): - with self.assertRaises(ValueError): - select(data, golden_meta(), [Range(point, point)], Settings()) + for relative_offset, value in [(0, -1), (8, -1), (8, 30), (16, 9), (24, 19)]: + data = bytearray(golden()) + struct.pack_into('>q', data, first_block_intervals + relative_offset, value) + data[-32:] = hashlib.sha256(data[:-32]).digest() + for ranges in ([Range(30, 30)], [Range(0, 0)], None, []): + with self.subTest(offset=relative_offset, value=value, ranges=ranges), \ + self.assertRaises(ValueError): + select(data, golden_meta(), ranges, Settings()) def test_coverage_and_budgets(self): header = avro_header() From 56e362be7c6736df20ddba93ab8860522af8ebd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Mon, 14 Sep 2026 14:27:13 +0800 Subject: [PATCH 12/23] Fix minus --- .../java/org/apache/paimon/manifest/ManifestFile.java | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java index 3fe5609a5671..adaa539f72dc 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java @@ -387,15 +387,7 @@ public Path toPath(String fileName) { @Nullable public ManifestSidecar.Selection selectBlocks( ManifestFileMeta manifest, @Nullable RowRangeIndex query) { - return selectBlocks(manifest, query, null); - } - - @Nullable - public ManifestSidecar.Selection selectBlocks( - ManifestFileMeta manifest, - @Nullable RowRangeIndex query, - @Nullable PartitionPredicate partitionFilter) { - return selectBlocks(manifest, query, partitionFilter, null); + return selectBlocks(manifest, query, null, null); } @Nullable From 98b516be6a5110c81ac70adcd8d7d9e3568fc9ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Mon, 14 Sep 2026 14:41:28 +0800 Subject: [PATCH 13/23] [core] Use a byte budget for manifest sidecar metadata --- docs/docs/concepts/spec/manifest.md | 9 +-- .../java/org/apache/paimon/CoreOptions.java | 28 -------- .../paimon/manifest/ManifestSidecar.java | 65 +++-------------- .../manifest/ManifestBlockIndexTest.java | 71 +++++++++++++++---- .../paimon/manifest/ManifestSidecarTest.java | 13 ++-- .../pypaimon/common/options/core_options.py | 24 ------- .../pypaimon/manifest/manifest_sidecar.py | 48 +++---------- .../manifest/manifest_block_index_test.py | 59 ++++++++++----- .../tests/manifest/manifest_sidecar_test.py | 16 ++--- 9 files changed, 139 insertions(+), 194 deletions(-) diff --git a/docs/docs/concepts/spec/manifest.md b/docs/docs/concepts/spec/manifest.md index b024d3d0dcff..6297686db399 100644 --- a/docs/docs/concepts/spec/manifest.md +++ b/docs/docs/concepts/spec/manifest.md @@ -146,18 +146,15 @@ pair counts, ordering or values invalidate the container rather than excluding a All entries contribute, including ADD, DELETE and every file format/column group. Row-ID ranges are never expanded into individual values. If an exact union exceeds its -range budget, it becomes the inclusive `[min,max]` envelope with encoding 1. Processing +available byte budget, it becomes the inclusive `[min,max]` envelope with encoding 1. Processing continues through the end of the block to extend those bounds and detect unknown row IDs. An unknown or invalid row-ID range makes only that block's row-ID payload unavailable. Partition budget exhaustion independently makes that block's partition payload unavailable. The dictionary can consequently be incomplete for the manifest: a dictionary miss never excludes a block with unavailable partition coverage. Later blocks can still use existing IDs. -`manifest.sidecar.max-ranges` bounds stored intervals (default 131072). -`manifest.sidecar.max-partitions` bounds dictionary entries (default 65536), and -`manifest.sidecar.max-partition-bytes` bounds dictionary bytes including length fields -(default 1048576). `manifest.sidecar.max-bucket-pairs` bounds distinct bucket/count pairs -per block (default 4096). `manifest.sidecar.max-bytes` bounds the whole serialized container +`manifest.sidecar.max-bytes` bounds the whole serialized container, including the +partition dictionary and all three payload types (default 8388608); the Avro header is also capped at 1 MiB and the directory at 131072 blocks. Writers discard optional row-ID payloads, bucket payloads, then partition payloads/dictionary if necessary, to fit the complete directory. If the directory itself cannot fit, no sidecar is published. diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java index dba38a078a9b..aa0c8a934c9b 100644 --- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java @@ -524,34 +524,6 @@ public InlineElement getDescription() { .withDescription( "Read optional manifest sidecars for partition, row-id or bucket filters after coarse pruning. Missing or invalid sidecars fall back to manifest reads."); - public static final ConfigOption MANIFEST_SIDECAR_MAX_RANGES = - key("manifest.sidecar.max-ranges") - .intType() - .defaultValue(131072) - .withDescription( - "Maximum disjoint row-id intervals across all Avro blocks in a manifest. On exhaustion, coarsen coverage to min/max or mark row-id coverage unavailable. Range: 1 to 1048576."); - - public static final ConfigOption MANIFEST_SIDECAR_MAX_PARTITIONS = - key("manifest.sidecar.max-partitions") - .intType() - .defaultValue(65536) - .withDescription( - "Maximum partition dictionary entries per manifest sidecar. Further unknown partitions disable partition coverage only for their blocks."); - - public static final ConfigOption MANIFEST_SIDECAR_MAX_BUCKET_PAIRS = - key("manifest.sidecar.max-bucket-pairs") - .intType() - .defaultValue(4096) - .withDescription( - "Maximum distinct bucket and total-bucket pairs per block. Exceeding this budget disables bucket coverage for the block, preserving other sidecar payloads."); - - public static final ConfigOption MANIFEST_SIDECAR_MAX_PARTITION_BYTES = - key("manifest.sidecar.max-partition-bytes") - .intType() - .defaultValue(1048576) - .withDescription( - "Maximum serialized partition dictionary bytes per manifest sidecar. Exceeding this budget preserves independently available row-id coverage."); - public static final ConfigOption MANIFEST_SIDECAR_MAX_BYTES = key("manifest.sidecar.max-bytes") .intType() diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java index 5bcc03f77e83..9145b9ece545 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java @@ -97,32 +97,12 @@ public static String fileName(ManifestFileMeta manifest) { public static final class Settings { public final boolean write; public final boolean read; - public final int maxRanges; public final int maxBytes; - public final int maxPartitions; - public final int maxPartitionBytes; - public final int maxBucketPairs; public Settings(Options options) { write = options.get(CoreOptions.MANIFEST_SIDECAR_WRITE); read = options.get(CoreOptions.MANIFEST_SIDECAR_READ); - maxRanges = options.get(CoreOptions.MANIFEST_SIDECAR_MAX_RANGES); maxBytes = options.get(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES); - maxPartitions = options.get(CoreOptions.MANIFEST_SIDECAR_MAX_PARTITIONS); - maxPartitionBytes = options.get(CoreOptions.MANIFEST_SIDECAR_MAX_PARTITION_BYTES); - maxBucketPairs = options.get(CoreOptions.MANIFEST_SIDECAR_MAX_BUCKET_PAIRS); - checkArgument( - maxBucketPairs > 0 && maxBucketPairs <= 1048576, - "Invalid manifest.sidecar.max-bucket-pairs"); - checkArgument( - maxPartitions > 0 && maxPartitions <= 1048576, - "Invalid manifest.sidecar.max-partitions"); - checkArgument( - maxPartitionBytes > 0 && maxPartitionBytes <= 64 * 1024 * 1024, - "Invalid manifest.sidecar.max-partition-bytes"); - checkArgument( - maxRanges > 0 && maxRanges <= 1048576, - "manifest.sidecar.max-ranges must be in [1, 1048576]"); checkArgument( maxBytes >= 128 && maxBytes <= 64 * 1024 * 1024, "manifest.sidecar.max-bytes must be in [128, 67108864]"); @@ -189,7 +169,6 @@ public static final class Builder { private long max; private int dictionaryBytes; private int optionalBytes; - private int savedRanges; public Builder(Settings settings, @Nullable byte[] header) { this.settings = settings; @@ -288,7 +267,7 @@ public void add( end = Math.max(end, next.getValue()); ranges.remove(next.getKey()); } - if (ranges.size() >= Math.min(settings.maxRanges, settings.maxBytes / 16)) { + if (4L + 16L * (ranges.size() + 1L) > settings.maxBytes - optionalBytes) { coarse = true; ranges.clear(); } else { @@ -311,8 +290,7 @@ private void addBucket(@Nullable Integer bucket, @Nullable Integer totalBuckets) } long pair = ((long) bucket << 32) | totalBuckets; if (!bucketPairs.contains(pair) - && bucketPairs.size() - >= Math.min(settings.maxBucketPairs, settings.maxBytes / 8)) { + && 4L + 8L * (bucketPairs.size() + 1L) > settings.maxBytes - optionalBytes) { bucketAvailable = false; bucketPairs.clear(); } else { @@ -331,10 +309,7 @@ private void addPartition(@Nullable byte[] bytes) { } Integer id = dictionary.get(ByteBuffer.wrap(bytes)); if (id == null) { - if (dictionary.size() >= settings.maxPartitions - || bytes.length + 4L - > Math.min(settings.maxPartitionBytes, settings.maxBytes) - - dictionaryBytes) { + if (bytes.length + 4L > settings.maxBytes - dictionaryBytes) { partitionAvailable = false; partitionIds.clear(); return; @@ -353,23 +328,18 @@ public void endBlock() throws IOException { require(current != null && entriesInBlock == current.recordCount); byte[] rowPayload = EMPTY; byte[] partitionPayload = EMPTY; - int rowCount = 0; if (rowAvailable) { - if (coarse - || ranges.size() > settings.maxRanges - savedRanges - || 4L + 16L * ranges.size() > settings.maxBytes - optionalBytes) { + if (coarse || 4L + 16L * ranges.size() > settings.maxBytes - optionalBytes) { ranges.clear(); ranges.put(min, max); } - if (savedRanges < settings.maxRanges - && 4L + 16L * ranges.size() <= settings.maxBytes - optionalBytes) { + if (4L + 16L * ranges.size() <= settings.maxBytes - optionalBytes) { ByteBuffer out = ByteBuffer.allocate(4 + 16 * ranges.size()); out.putInt(ranges.size()); for (Map.Entry range : ranges.entrySet()) { out.putLong(range.getKey()).putLong(range.getValue()); } rowPayload = out.array(); - rowCount = ranges.size(); optionalBytes += rowPayload.length; } } @@ -395,7 +365,6 @@ public void endBlock() throws IOException { optionalBytes += bucketPayload.length; } blocks.add(new IndexedBlock(current, partitionPayload, rowPayload, bucketPayload)); - savedRanges += rowCount; nextOffset = Math.addExact(current.offset, current.length); nextRecord = Math.addExact(current.firstRecord, current.recordCount); ranges.clear(); @@ -578,21 +547,13 @@ public static Selection select( in.get(header); require(header[0] == 'O' && header[1] == 'b' && header[2] == 'j' && header[3] == 1); int partitions = in.getInt(); - require( - partitions >= 0 - && partitions <= settings.maxPartitions - && partitions <= in.remaining() / 16); + require(partitions >= 0 && partitions <= in.remaining() / 16); boolean[] matches = new boolean[partitions]; Set unique = new java.util.HashSet<>(); - int dictionaryBytes = 0; for (int id = 0; id < partitions; id++) { require(in.remaining() >= 4); int length = in.getInt(); - require( - length >= 12 - && length <= in.remaining() - && length + 4L <= settings.maxPartitionBytes - dictionaryBytes); - dictionaryBytes += 4 + length; + require(length >= 12 && length <= in.remaining()); ByteBuffer encoded = in.slice(); encoded.limit(length); int arity = encoded.getInt(0); @@ -614,7 +575,6 @@ public static Selection select( require(count >= 0 && count <= MAX_BLOCKS && count <= in.remaining() / BLOCK_BYTES); long nextOffset = headerLength; long firstRecord = 0; - int totalRanges = 0; List selected = new ArrayList<>(); for (int i = 0; i < count; i++) { require(in.remaining() >= BLOCK_BYTES); @@ -650,11 +610,7 @@ public static Selection select( } else if (rowEncoding == 1) { require(rowPayload.remaining() >= 4); int ranges = rowPayload.getInt(); - require( - ranges > 0 - && ranges <= settings.maxRanges - totalRanges - && rowPayload.remaining() == 16L * ranges); - totalRanges += ranges; + require(ranges > 0 && rowPayload.remaining() == 16L * ranges); long min = rowPayload.getLong(); long firstEnd = rowPayload.getLong(); long max = ranges == 1 ? firstEnd : rowPayload.getLong(rowPayload.limit() - 8); @@ -683,10 +639,7 @@ public static Selection select( } else if (bucketEncoding == 1) { require(bucketPayload.remaining() >= 4); int pairs = bucketPayload.getInt(); - require( - pairs > 0 - && pairs <= settings.maxBucketPairs - && bucketPayload.remaining() == 8L * pairs); + require(pairs > 0 && bucketPayload.remaining() == 8L * pairs); bucketHit = bucketFilter == null; long previous = -1; for (int j = 0; j < pairs; j++) { diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java index 5c805a6c242e..1ca525c59a0f 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java @@ -131,7 +131,7 @@ void jointGoldenPreservesTuplesNullsAndDerivedOrdinals() throws Exception { @Test void unavailableDimensionsAreIndependentAndDoNotPoisonLaterBlocks() throws Exception { Options options = new Options(); - options.set(CoreOptions.MANIFEST_SIDECAR_MAX_PARTITIONS, 1); + options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES, 512); ManifestSidecar.Settings settings = new ManifestSidecar.Settings(options); byte[] header = fixture("avroHeader"); ManifestSidecar.Builder builder = new ManifestSidecar.Builder(settings, header); @@ -139,7 +139,7 @@ void unavailableDimensionsAreIndependentAndDoNotPoisonLaterBlocks() throws Excep builder.add(null, 10, partition(7, "left")); builder.endBlock(); builder.beginBlock(header.length + 100, 100, 1); - builder.add(200L, 10, partition(9, null)); // dictionary budget exceeded + builder.add(200L, 10, partition(9, String.join("", Collections.nCopies(600, "x")))); builder.endBlock(); builder.beginBlock(header.length + 200, 100, 1); builder.add(300L, 10, partition(7, "left")); // an existing dictionary ID remains usable @@ -160,19 +160,20 @@ void unavailableDimensionsAreIndependentAndDoNotPoisonLaterBlocks() throws Excep @Test void coarseningContinuesThroughTheEntireBlockAndDetectsUnknownRows() throws Exception { Options options = new Options(); - options.set(CoreOptions.MANIFEST_SIDECAR_MAX_RANGES, 1); + options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES, 512); ManifestSidecar.Settings settings = new ManifestSidecar.Settings(options); byte[] header = fixture("avroHeader"); for (boolean unknown : new boolean[] {false, true}) { ManifestSidecar.Builder builder = new ManifestSidecar.Builder(settings, header); - builder.beginBlock(header.length, 100, 4); - builder.add(100L, 10, partition(7, "left")); - builder.add(300L, 10, partition(7, "left")); + builder.beginBlock(header.length, 100, 66); + for (int i = 0; i < 64; i++) { + builder.add(100L + i * 1000L, 10, partition(7, "left")); + } builder.add(10L, 10, partition(7, "left")); builder.add(unknown ? null : Long.MAX_VALUE, 1, partition(7, "left")); builder.endBlock(); - byte[] data = builder.serialize("m", header.length + 100, 4); - ManifestFileMeta meta = meta("m", header.length + 100, 4); + byte[] data = builder.serialize("m", header.length + 100, 66); + ManifestFileMeta meta = meta("m", header.length + 100, 66); for (long point : new long[] {10, 100, 200, Long.MAX_VALUE}) { assertThat(ManifestSidecar.select(data, meta, query(point), settings).blocks()) .hasSize(1); @@ -387,7 +388,7 @@ public boolean mayContain(int min, int max, int total) { @Test void unknownInvalidOrOverBudgetBucketPayloadIsUnavailable() throws Exception { Options options = new Options(); - options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BUCKET_PAIRS, 1); + options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES, 512); ManifestSidecar.Settings settings = new ManifestSidecar.Settings(options); byte[] header = fixture("avroHeader"); for (Integer[] pair : @@ -398,18 +399,22 @@ void unknownInvalidOrOverBudgetBucketPayloadIsUnavailable() throws Exception { new Integer[] {0, 0}, new Integer[] {2, 8})) { ManifestSidecar.Builder builder = new ManifestSidecar.Builder(settings, header); - builder.beginBlock(header.length, 100, 2); + int extraPairs = pair[0] != null && pair[0] == 2 ? 65 : 0; + builder.beginBlock(header.length, 100, 2 + extraPairs); builder.add(100L, 10, partition(7, "left"), 1, 4); builder.add(200L, 10, partition(7, "left"), pair[0], pair[1]); + for (int i = 0; i < extraPairs; i++) { + builder.add(200L, 10, partition(7, "left"), i, 100); + } builder.endBlock(); builder.beginBlock(header.length + 100, 100, 1); builder.add(300L, 10, partition(7, "left"), 1, 4); builder.endBlock(); - byte[] data = builder.serialize("m", header.length + 200, 3); + byte[] data = builder.serialize("m", header.length + 200, 3 + extraPairs); int bucket = positions(data).get(0)[3]; assertThat(data[bucket]).isZero(); assertThat(ByteBuffer.wrap(data).getInt(bucket + 1)).isZero(); - ManifestFileMeta meta = meta("m", header.length + 200, 3); + ManifestFileMeta meta = meta("m", header.length + 200, 3 + extraPairs); assertThat( ManifestSidecar.select( data, @@ -462,6 +467,48 @@ bad, meta, query(999), part(99), type, defaults)) } } + @Test + void payloadsCanExceedFormerLimitsWithinByteBudget() throws Exception { + byte[] header = fixture("avroHeader"); + ManifestSidecar.Builder builder = new ManifestSidecar.Builder(defaults, header); + int blocks = 33; + int entriesPerBlock = 4097; + int entries = blocks * entriesPerBlock; + int blockBytes = 1024 * 1024; + for (int block = 0; block < blocks; block++) { + builder.beginBlock( + header.length + (long) block * blockBytes, blockBytes, entriesPerBlock); + for (int i = 0; i < entriesPerBlock; i++) { + int entry = block * entriesPerBlock + i; + builder.add(entry * 2L, 1, partition(entry, null), i, entriesPerBlock + 1); + } + builder.endBlock(); + } + long fileSize = header.length + (long) blocks * blockBytes; + byte[] data = builder.serialize("m", fileSize, entries); + assertThat(data.length).isLessThanOrEqualTo(defaults.maxBytes); + ManifestFileMeta meta = meta("m", fileSize, entries); + long last = (entries - 1L) * 2; + assertThat(ManifestSidecar.select(data, meta, query(last), defaults).blocks()) + .extracting(block -> block.firstRecord) + .containsExactly((blocks - 1L) * entriesPerBlock); + assertThat(ManifestSidecar.select(data, meta, query(last - 1), defaults).blocks()) + .isEmpty(); + assertThat(ManifestSidecar.select(data, meta, null, part(entries), type, defaults).blocks()) + .isEmpty(); + assertThat( + ManifestSidecar.select( + data, + meta, + null, + null, + type, + BucketFilter.create(false, entriesPerBlock, null, null), + defaults) + .blocks()) + .isEmpty(); + } + @Test void tightByteBudgetKeepsAllDescriptorsOrOmitsTheWholeFile() throws Exception { Options options = new Options(); diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java index f43824e219de..b48e10a34975 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java @@ -292,16 +292,17 @@ void hugeRangesAreNotExpandedAndInvalidCoverageRetainsBlocks() throws Exception .hasSize(1); } Options options = new Options(); - options.set(CoreOptions.MANIFEST_SIDECAR_MAX_RANGES, 1); + options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES, 512); builder = new ManifestSidecar.Builder(new ManifestSidecar.Settings(options), header); - builder.beginBlock(header.length, 100, 2); - builder.add(0L, 1); - builder.add(10L, 1); + builder.beginBlock(header.length, 100, 64); + for (int i = 0; i < 64; i++) { + builder.add(i * 10L, 1); + } builder.endBlock(); assertThat( select( - builder.serialize("m", header.length + 100, 2), - meta("m", header.length + 100, 2), + builder.serialize("m", header.length + 100, 64), + meta("m", header.length + 100, 64), 5) .blocks()) .hasSize(1); diff --git a/paimon-python/pypaimon/common/options/core_options.py b/paimon-python/pypaimon/common/options/core_options.py index b0fed410fad1..370dbb4beea9 100644 --- a/paimon-python/pypaimon/common/options/core_options.py +++ b/paimon-python/pypaimon/common/options/core_options.py @@ -309,36 +309,12 @@ class CoreOptions: .default_value(False) ) - MANIFEST_SIDECAR_MAX_RANGES: ConfigOption[int] = ( - ConfigOptions.key("manifest.sidecar.max-ranges") - .int_type() - .default_value(131072) - ) - MANIFEST_SIDECAR_MAX_BYTES: ConfigOption[int] = ( ConfigOptions.key("manifest.sidecar.max-bytes") .int_type() .default_value(8388608) ) - MANIFEST_SIDECAR_MAX_PARTITIONS: ConfigOption[int] = ( - ConfigOptions.key("manifest.sidecar.max-partitions") - .int_type() - .default_value(65536) - ) - - MANIFEST_SIDECAR_MAX_BUCKET_PAIRS: ConfigOption[int] = ( - ConfigOptions.key("manifest.sidecar.max-bucket-pairs") - .int_type() - .default_value(4096) - ) - - MANIFEST_SIDECAR_MAX_PARTITION_BYTES: ConfigOption[int] = ( - ConfigOptions.key("manifest.sidecar.max-partition-bytes") - .int_type() - .default_value(1048576) - ) - MANIFEST_COMPRESSION: ConfigOption[str] = ( ConfigOptions.key("manifest.compression") .string_type() diff --git a/paimon-python/pypaimon/manifest/manifest_sidecar.py b/paimon-python/pypaimon/manifest/manifest_sidecar.py index 5a26b4751646..83e7e2ad9792 100644 --- a/paimon-python/pypaimon/manifest/manifest_sidecar.py +++ b/paimon-python/pypaimon/manifest/manifest_sidecar.py @@ -52,21 +52,9 @@ class Settings: write: bool = False read: bool = False - max_ranges: int = 131072 max_bytes: int = 8 * 1024 * 1024 - max_partitions: int = 65536 - max_partition_bytes: int = 1024 * 1024 - max_bucket_pairs: int = 4096 def __post_init__(self): - if not 1 <= self.max_bucket_pairs <= 1048576: - raise ValueError('Invalid manifest.sidecar.max-bucket-pairs') - if not 1 <= self.max_partitions <= 1048576: - raise ValueError('Invalid manifest.sidecar.max-partitions') - if not 1 <= self.max_partition_bytes <= 64 * 1024 * 1024: - raise ValueError('Invalid manifest.sidecar.max-partition-bytes') - if not 1 <= self.max_ranges <= 1048576: - raise ValueError('manifest.sidecar.max-ranges must be in [1, 1048576]') if not 128 <= self.max_bytes <= 64 * 1024 * 1024: raise ValueError('manifest.sidecar.max-bytes must be in [128, 67108864]') @@ -75,11 +63,7 @@ def from_options(cls, options): return cls( options.options.get(CoreOptions.MANIFEST_SIDECAR_WRITE), options.options.get(CoreOptions.MANIFEST_SIDECAR_READ), - options.options.get(CoreOptions.MANIFEST_SIDECAR_MAX_RANGES), - options.options.get(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES), - options.options.get(CoreOptions.MANIFEST_SIDECAR_MAX_PARTITIONS), - options.options.get(CoreOptions.MANIFEST_SIDECAR_MAX_PARTITION_BYTES), - options.options.get(CoreOptions.MANIFEST_SIDECAR_MAX_BUCKET_PAIRS)) + options.options.get(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES)) @dataclass(frozen=True) @@ -116,7 +100,6 @@ def __init__(self, settings, header): self.dictionary = {} self.dictionary_bytes = 0 self.optional_bytes = 0 - self.saved_ranges = 0 self.blocks = [] self.ranges = [] self.partition_ids = set() @@ -173,7 +156,8 @@ def add(self, first, count, partition=None, bucket=None, total_buckets=None): first = min(first, self.ranges[right][0]) end = max(end, self.ranges[right][1]) right += 1 - if len(self.ranges) - (right - left) >= min(self.settings.max_ranges, self.settings.max_bytes // 16): + if (4 + 16 * (len(self.ranges) - (right - left) + 1) + > self.settings.max_bytes - self.optional_bytes): self.coarse = True self.ranges.clear() else: @@ -189,7 +173,7 @@ def _add_bucket(self, bucket, total_buckets): return pair = (bucket, total_buckets) if (pair not in self.bucket_pairs - and len(self.bucket_pairs) >= min(self.settings.max_bucket_pairs, self.settings.max_bytes // 8)): + and 4 + 8 * (len(self.bucket_pairs) + 1) > self.settings.max_bytes - self.optional_bytes): self.bucket_available = False self.bucket_pairs.clear() else: @@ -205,9 +189,7 @@ def _add_partition(self, partition): partition = bytes(partition) id_ = self.dictionary.get(partition) if id_ is None: - if (len(self.dictionary) >= self.settings.max_partitions - or len(partition) + 4 > min(self.settings.max_partition_bytes, self.settings.max_bytes) - - self.dictionary_bytes): + if len(partition) + 4 > self.settings.max_bytes - self.dictionary_bytes: self.partition_available = False self.partition_ids.clear() return @@ -222,15 +204,12 @@ def end_block(self): block = self.current _require(block is not None and self.entries_in_block == block.record_count) row_payload = partition_payload = b'' - row_count = 0 if self.row_available: - if (self.coarse or len(self.ranges) > self.settings.max_ranges - self.saved_ranges + if (self.coarse or 4 + 16 * len(self.ranges) > self.settings.max_bytes - self.optional_bytes): self.ranges = [(self.min, self.max)] - if (self.saved_ranges < self.settings.max_ranges - and 4 + 16 * len(self.ranges) <= self.settings.max_bytes - self.optional_bytes): + if 4 + 16 * len(self.ranges) <= self.settings.max_bytes - self.optional_bytes: row_payload = struct.pack('>I', len(self.ranges)) + b''.join(PAIR.pack(*r) for r in self.ranges) - row_count = len(self.ranges) self.optional_bytes += len(row_payload) if (self.partition_available and 4 + 4 * len(self.partition_ids) <= self.settings.max_bytes - self.optional_bytes): @@ -244,7 +223,6 @@ def end_block(self): bucket_payload += b''.join(struct.pack('>ii', *pair) for pair in sorted(self.bucket_pairs)) self.optional_bytes += len(bucket_payload) self.blocks.append([block, partition_payload, row_payload, bucket_payload]) - self.saved_ranges += row_count self.next_offset = block.offset + block.length self.next_record = block.first_record + block.record_count self.ranges.clear() @@ -343,16 +321,14 @@ def select(data, manifest, query, settings, partition_filter=None, partition_fie offset += header_length partitions, = struct.unpack_from('>I', data, offset) offset += 4 - _require(partitions <= settings.max_partitions and partitions <= (limit - offset) // 16) + _require(partitions <= (limit - offset) // 16) matches = [] unique = set() - dictionary_bytes = 0 for _ in range(partitions): _require(offset + 4 <= limit) length, = struct.unpack_from('>I', data, offset) offset += 4 - _require(12 <= length <= limit - offset and length + 4 <= settings.max_partition_bytes - dictionary_bytes) - dictionary_bytes += 4 + length + _require(12 <= length <= limit - offset) partition = bytes(data[offset:offset + length]) arity, = struct.unpack_from('>i', partition) _require(arity >= 0 and 4 + ((arity + 71) // 64) * 8 + arity * 8 <= length) @@ -371,7 +347,6 @@ def select(data, manifest, query, settings, partition_filter=None, partition_fie _require(blocks <= MAX_BLOCKS and blocks <= (limit - offset) // BLOCK_BYTES) next_offset = header_length first_record = 0 - total_ranges = 0 selected = [] for _ in range(blocks): _require(offset + BLOCK_BYTES <= limit) @@ -407,8 +382,7 @@ def select(data, manifest, query, settings, partition_filter=None, partition_fie elif row_encoding == 1: _require(payload_length >= 4) ranges, = struct.unpack_from('>I', data, offset) - _require(0 < ranges <= settings.max_ranges - total_ranges and 4 + 16 * ranges == payload_length) - total_ranges += ranges + _require(ranges > 0 and 4 + 16 * ranges == payload_length) min_row_id, first_end = PAIR.unpack_from(data, offset + 4) max_row_id = first_end if ranges == 1 else LONG.unpack_from(data, offset + payload_length - 8)[0] _require(min_row_id >= 0 and first_end >= min_row_id and max_row_id >= first_end) @@ -432,7 +406,7 @@ def select(data, manifest, query, settings, partition_filter=None, partition_fie elif bucket_encoding == 1: _require(payload_length >= 4) pairs, = struct.unpack_from('>I', data, offset) - _require(0 < pairs <= settings.max_bucket_pairs and payload_length == 4 + 8 * pairs) + _require(pairs > 0 and payload_length == 4 + 8 * pairs) bucket_hit = bucket_filter is None previous = (-1, -1) for j in range(pairs): diff --git a/paimon-python/pypaimon/tests/manifest/manifest_block_index_test.py b/paimon-python/pypaimon/tests/manifest/manifest_block_index_test.py index 0efad7e1c33e..c2bccc2c058a 100644 --- a/paimon-python/pypaimon/tests/manifest/manifest_block_index_test.py +++ b/paimon-python/pypaimon/tests/manifest/manifest_block_index_test.py @@ -78,11 +78,10 @@ def test_partition_dictionary_golden_tuples_nulls_and_derived_ordinals(): assert len(select(golden(), golden_meta(), None, Settings(), part(99), FIELDS).blocks) == 3 -@pytest.mark.parametrize('settings', [Settings(max_partitions=1), Settings(max_partition_bytes=32)]) -def test_unavailable_dimensions_are_independent_and_dictionary_misses_keep_unknown_blocks(settings): - header = avro_header() +def test_unavailable_dimensions_are_independent_and_dictionary_misses_keep_unknown_blocks(): + settings, header = Settings(max_bytes=512), avro_header() builder = Builder(settings, header) - for i, (first, p) in enumerate([(None, partition(7, 'left')), (200, partition(9, None)), + for i, (first, p) in enumerate([(None, partition(7, 'left')), (200, partition(9, 'x' * 600)), (300, partition(7, 'left'))]): builder.begin_block(len(header) + 100 * i, 100, 1) builder.add(first, 10, p) @@ -98,14 +97,16 @@ def test_unavailable_dimensions_are_independent_and_dictionary_misses_keep_unkno @pytest.mark.parametrize('unknown', [False, True]) def test_coarse_row_ranges_keep_extending_bounds_and_detect_late_unknowns(unknown): - settings, header = Settings(max_ranges=1), avro_header() + settings, header = Settings(max_bytes=512), avro_header() builder = Builder(settings, header) - builder.begin_block(len(header), 100, 4) - for first, count in [(100, 10), (300, 10), (10, 10), (None if unknown else (1 << 63) - 1, 1)]: + builder.begin_block(len(header), 100, 66) + for i in range(64): + builder.add(100 + i * 1000, 10, partition(7, 'left')) + for first, count in [(10, 10), (None if unknown else (1 << 63) - 1, 1)]: builder.add(first, count, partition(7, 'left')) builder.end_block() - data = builder.serialize('m', len(header) + 100, 4) - metadata = meta('m', len(header) + 100, 4) + data = builder.serialize('m', len(header) + 100, 66) + metadata = meta('m', len(header) + 100, 66) for point in [10, 100, 200, (1 << 63) - 1]: assert len(select(data, metadata, [Range(point, point)], settings).blocks) == 1 assert len(select(data, metadata, [Range(0, 0)], settings).blocks) == int(unknown) @@ -206,10 +207,7 @@ def test_randomized_budget_degradation_has_no_false_negatives(): rng = random.Random(9743) header = avro_header() for _ in range(60): - settings = Settings(max_ranges=rng.choice([1, 4, 100]), - max_partitions=rng.choice([1, 3, 20]), - max_partition_bytes=rng.choice([32, 128, 1024]), - max_bytes=rng.choice([512, 1024, 8192])) + settings = Settings(max_bytes=rng.choice([384, 512, 1024, 8192])) builder = Builder(settings, header) blocks = [] for i in range(5): @@ -259,23 +257,50 @@ def test_bucket_payload_golden_rescale_and_unavailable_payloads(): @pytest.mark.parametrize('pair', [(None, None), (-1, 4), (4, 4), (0, 0), (2, 8)]) def test_bucket_budget_and_unknown_pairs_degrade_only_bucket_payload(pair): - header, settings = avro_header(), Settings(max_bucket_pairs=1) + header, settings = avro_header(), Settings(max_bytes=512) builder = Builder(settings, header) - builder.begin_block(len(header), 100, 2) + extra_pairs = 65 if pair == (2, 8) else 0 + builder.begin_block(len(header), 100, 2 + extra_pairs) builder.add(100, 10, partition(7, 'left'), 1, 4) builder.add(200, 10, partition(7, 'left'), *pair) + for i in range(extra_pairs): + builder.add(200, 10, partition(7, 'left'), i, 100) builder.end_block() builder.begin_block(len(header) + 100, 100, 1) builder.add(300, 10, partition(7, 'left'), 1, 4) builder.end_block() - data = builder.serialize('m', len(header) + 200, 3) + data = builder.serialize('m', len(header) + 200, 3 + extra_pairs) assert struct.unpack_from('>BI', data, positions(data)[0][3]) == (0, 0) - metadata = meta('m', len(header) + 200, 3) + metadata = meta('m', len(header) + 200, 3 + extra_pairs) selected = select(data, metadata, None, settings, bucket_filter=lambda bucket, total: False) assert [b.first_record for b in selected.blocks] == [0] assert not select(data, metadata, [Range(999, 999)], settings, bucket_filter=lambda bucket, total: False).blocks +def test_payloads_can_exceed_former_limits_within_byte_budget(): + header, settings = avro_header(), Settings() + builder = Builder(settings, header) + blocks, entries_per_block, block_bytes = 33, 4097, 1024 * 1024 + entries = blocks * entries_per_block + for block in range(blocks): + builder.begin_block(len(header) + block * block_bytes, block_bytes, entries_per_block) + for i in range(entries_per_block): + entry = block * entries_per_block + i + builder.add(entry * 2, 1, partition(entry, None), i, entries_per_block + 1) + builder.end_block() + file_size = len(header) + blocks * block_bytes + data = builder.serialize('m', file_size, entries) + assert len(data) <= settings.max_bytes + metadata = meta('m', file_size, entries) + last = (entries - 1) * 2 + selected = select(data, metadata, [Range(last, last)], settings) + assert [b.first_record for b in selected.blocks] == [(blocks - 1) * entries_per_block] + assert not select(data, metadata, [Range(last - 1, last - 1)], settings).blocks + assert not select(data, metadata, None, settings, part(entries), FIELDS).blocks + assert not select(data, metadata, None, settings, + bucket_filter=lambda bucket, total: bucket == entries_per_block).blocks + + def test_malformed_bucket_payload_invalidates_the_container(): good = fixture('indexWithBuckets') payload = positions(good)[0][3] + 1 diff --git a/paimon-python/pypaimon/tests/manifest/manifest_sidecar_test.py b/paimon-python/pypaimon/tests/manifest/manifest_sidecar_test.py index 4bc274571f49..af0d7ab33c68 100644 --- a/paimon-python/pypaimon/tests/manifest/manifest_sidecar_test.py +++ b/paimon-python/pypaimon/tests/manifest/manifest_sidecar_test.py @@ -315,14 +315,14 @@ def test_coverage_and_budgets(self): b.add(MAX_ROW_ID, 1) b.end_block() self.assertLess(len(b.serialize('m', len(header) + 100, 2)), 512) - b = Builder(Settings(max_ranges=1), header) - b.begin_block(len(header), 100, 2) - b.add(1, 1) - b.add(1 << 32, 1) + b = Builder(Settings(max_bytes=512), header) + b.begin_block(len(header), 100, 64) + for i in range(64): + b.add(1 + i * 100, 1) b.end_block() - data = b.serialize('m', len(header) + 100, 2) + data = b.serialize('m', len(header) + 100, 64) meta = SimpleNamespace(file_name='m', file_size=len(header) + 100, - num_added_files=2, num_deleted_files=0) + num_added_files=64, num_deleted_files=0) self.assertEqual(len(select(data, meta, [Range(10, 10)], Settings()).blocks), 1) b = Builder(Settings(max_bytes=128), header) self.assertIsNone(b.serialize('m', 1, 1)) @@ -733,8 +733,8 @@ def fail(path): self.assertIsNotNone(sidecar_file_name(unknown)) data = Path(manager.manifest_path, sidecar_file_name(unknown)).read_bytes() self.assertEqual(len(select(data, unknown, [Range(100, 100)], Settings()).blocks), 1) - self.table.options.options.set(CoreOptions.MANIFEST_SIDECAR_MAX_RANGES, 1) - huge = manager.write('huge', [self.entry('one', 0, 10), self.entry('two', 100, 10)]) + huge = manager.write('huge', [self.entry('one', 0, MAX_ROW_ID), + self.entry('two', MAX_ROW_ID, 1)]) self.assertIsNotNone(sidecar_file_name(huge)) data = Path(manager.manifest_path, sidecar_file_name(huge)).read_bytes() self.assertEqual(len(select(data, huge, [Range(50, 50)], Settings()).blocks), 1) From 4215e41f8b60af240138dda7b46c8fead9684121 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Mon, 14 Sep 2026 14:57:14 +0800 Subject: [PATCH 14/23] [core] Derive sidecar memory budget from manifest target size --- docs/docs/concepts/spec/manifest.md | 6 ++-- .../java/org/apache/paimon/CoreOptions.java | 13 +++++--- .../apache/paimon/manifest/ManifestFile.java | 9 +++++- .../paimon/manifest/ManifestSidecar.java | 8 +++-- .../manifest/ManifestBlockIndexTest.java | 9 +++--- .../paimon/manifest/ManifestFileTest.java | 19 ++++++++++++ .../paimon/manifest/ManifestSidecarTest.java | 31 ++++++++++++++++--- .../pypaimon/common/options/core_options.py | 14 +++++++-- .../pypaimon/manifest/manifest_sidecar.py | 15 +++++---- .../tests/manifest/manifest_sidecar_test.py | 27 ++++++++++++++++ 10 files changed, 124 insertions(+), 27 deletions(-) diff --git a/docs/docs/concepts/spec/manifest.md b/docs/docs/concepts/spec/manifest.md index 6297686db399..636a9c7e15c4 100644 --- a/docs/docs/concepts/spec/manifest.md +++ b/docs/docs/concepts/spec/manifest.md @@ -154,8 +154,10 @@ The dictionary can consequently be incomplete for the manifest: a dictionary mis excludes a block with unavailable partition coverage. Later blocks can still use existing IDs. `manifest.sidecar.max-bytes` bounds the whole serialized container, including the -partition dictionary and all three payload types -(default 8388608); the Avro header is also capped at 1 MiB and the directory at 131072 blocks. +partition dictionary and all three payload types. It accepts memory sizes such as +`16 mb` and, when unset, defaults to twice the configured `manifest.target-file-size` +(16 MiB with the default 8 MiB manifest target). An explicit sidecar size overrides this default. +The Avro header is also capped at 1 MiB and the directory at 131072 blocks. Writers discard optional row-ID payloads, bucket payloads, then partition payloads/dictionary if necessary, to fit the complete directory. If the directory itself cannot fit, no sidecar is published. No emitted sidecar omits block descriptors. These are encoded-size bounds; construction diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java index aa0c8a934c9b..12ef2cdd03a2 100644 --- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java @@ -524,12 +524,12 @@ public InlineElement getDescription() { .withDescription( "Read optional manifest sidecars for partition, row-id or bucket filters after coarse pruning. Missing or invalid sidecars fall back to manifest reads."); - public static final ConfigOption MANIFEST_SIDECAR_MAX_BYTES = + public static final ConfigOption MANIFEST_SIDECAR_MAX_BYTES = key("manifest.sidecar.max-bytes") - .intType() - .defaultValue(8388608) + .memoryType() + .noDefaultValue() .withDescription( - "Maximum serialized manifest sidecar bytes, including header and checksum. Optional payloads are dropped before omitting a sidecar whose complete block directory cannot fit. Range: 128 to 67108864."); + "Maximum serialized manifest sidecar size, including header and checksum. Defaults to twice manifest.target-file-size. Optional payloads are dropped before omitting a sidecar whose complete block directory cannot fit."); public static final ConfigOption MANIFEST_COMPRESSION = key("manifest.compression") @@ -3231,6 +3231,11 @@ public MemorySize manifestTargetSize() { return options.get(MANIFEST_TARGET_FILE_SIZE); } + public MemorySize manifestSidecarMaxSize() { + return options.getOptional(MANIFEST_SIDECAR_MAX_BYTES) + .orElseGet(() -> manifestTargetSize().multiply(2)); + } + public MemorySize manifestFullCompactionThresholdSize() { return options.get(MANIFEST_FULL_COMPACTION_FILE_SIZE); } diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java index adaa539f72dc..cd96bd611ab1 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java @@ -18,6 +18,7 @@ package org.apache.paimon.manifest; +import org.apache.paimon.CoreOptions; import org.apache.paimon.annotation.VisibleForTesting; import org.apache.paimon.data.InternalRow; import org.apache.paimon.format.FileFormat; @@ -457,7 +458,13 @@ public Factory( } public Factory withSidecarOptions(Options options) { - sidecarSettings = new ManifestSidecar.Settings(options); + // Disabled sidecars must not constrain the manifest target size. + sidecarSettings = + new ManifestSidecar.Settings( + options.get(CoreOptions.MANIFEST_SIDECAR_READ) + || options.get(CoreOptions.MANIFEST_SIDECAR_WRITE) + ? options + : new Options()); return this; } diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java index 9145b9ece545..5e778e64f1bf 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java @@ -102,10 +102,12 @@ public static final class Settings { public Settings(Options options) { write = options.get(CoreOptions.MANIFEST_SIDECAR_WRITE); read = options.get(CoreOptions.MANIFEST_SIDECAR_READ); - maxBytes = options.get(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES); + long bytes = new CoreOptions(options).manifestSidecarMaxSize().getBytes(); + // Keep room for the extra byte used to detect an over-budget input. checkArgument( - maxBytes >= 128 && maxBytes <= 64 * 1024 * 1024, - "manifest.sidecar.max-bytes must be in [128, 67108864]"); + bytes >= 128 && bytes < Integer.MAX_VALUE, + "manifest.sidecar.max-bytes must be in [128, 2147483646] bytes"); + maxBytes = (int) bytes; } } diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java index 1ca525c59a0f..2c0a1b4d7d80 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java @@ -22,6 +22,7 @@ import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.BinaryRowWriter; import org.apache.paimon.data.BinaryString; +import org.apache.paimon.options.MemorySize; import org.apache.paimon.options.Options; import org.apache.paimon.partition.PartitionPredicate; import org.apache.paimon.predicate.PredicateBuilder; @@ -131,7 +132,7 @@ void jointGoldenPreservesTuplesNullsAndDerivedOrdinals() throws Exception { @Test void unavailableDimensionsAreIndependentAndDoNotPoisonLaterBlocks() throws Exception { Options options = new Options(); - options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES, 512); + options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES, new MemorySize(512)); ManifestSidecar.Settings settings = new ManifestSidecar.Settings(options); byte[] header = fixture("avroHeader"); ManifestSidecar.Builder builder = new ManifestSidecar.Builder(settings, header); @@ -160,7 +161,7 @@ void unavailableDimensionsAreIndependentAndDoNotPoisonLaterBlocks() throws Excep @Test void coarseningContinuesThroughTheEntireBlockAndDetectsUnknownRows() throws Exception { Options options = new Options(); - options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES, 512); + options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES, new MemorySize(512)); ManifestSidecar.Settings settings = new ManifestSidecar.Settings(options); byte[] header = fixture("avroHeader"); for (boolean unknown : new boolean[] {false, true}) { @@ -388,7 +389,7 @@ public boolean mayContain(int min, int max, int total) { @Test void unknownInvalidOrOverBudgetBucketPayloadIsUnavailable() throws Exception { Options options = new Options(); - options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES, 512); + options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES, new MemorySize(512)); ManifestSidecar.Settings settings = new ManifestSidecar.Settings(options); byte[] header = fixture("avroHeader"); for (Integer[] pair : @@ -512,7 +513,7 @@ void payloadsCanExceedFormerLimitsWithinByteBudget() throws Exception { @Test void tightByteBudgetKeepsAllDescriptorsOrOmitsTheWholeFile() throws Exception { Options options = new Options(); - options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES, 280); + options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES, new MemorySize(280)); ManifestSidecar.Settings settings = new ManifestSidecar.Settings(options); byte[] header = fixture("avroHeader"); ManifestSidecar.Builder builder = new ManifestSidecar.Builder(settings, header); diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java index 2357c5b9c044..a1f4b4778ef5 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java @@ -1359,6 +1359,25 @@ void testRowIdSidecarRollingRawRewriteAndDelete() throws Exception { } } + @Test + void testDisabledSidecarsDoNotConstrainManifestTargetSize() { + for (String target : new String[] {"1 bytes", "1 gb"}) { + Options options = new Options(); + options.setString(CoreOptions.MANIFEST_TARGET_FILE_SIZE.key(), target); + ManifestFile manifestFile = + createManifestFileFactory( + tempDir.toString(), + Long.MAX_VALUE, + options, + new RecordingFileIO()) + .create(); + List entries = Collections.singletonList(gen.next()); + ManifestFileMeta meta = manifestFile.write(entries).get(0); + assertThat(ManifestSidecar.fileName(meta)).isNull(); + assertThat(manifestFile.read(meta.fileName())).containsExactlyElementsOf(entries); + } + } + @Test void testReadsOnlySelectedBlocksAndPreservesPhysicalOrdinals() throws Exception { Options options = new Options(); diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java index b48e10a34975..067a59ce4c8a 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java @@ -23,6 +23,7 @@ import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.options.MemorySize; import org.apache.paimon.options.Options; import org.apache.paimon.utils.IOUtils; import org.apache.paimon.utils.Range; @@ -147,6 +148,28 @@ void crossLanguageFormatAndBlockOrdinals() throws Exception { assertThat(query.ranges()).containsExactly(new Range(10, 19), new Range(25, 40)); } + @Test + void settingsUseMemorySizesAndFollowTheManifestTarget() { + Options options = new Options(); + assertThat(new ManifestSidecar.Settings(options).maxBytes).isEqualTo(16 * 1024 * 1024); + options.setString(CoreOptions.MANIFEST_TARGET_FILE_SIZE.key(), "12 mb"); + assertThat(new ManifestSidecar.Settings(options).maxBytes).isEqualTo(24 * 1024 * 1024); + options.setString(CoreOptions.MANIFEST_TARGET_FILE_SIZE.key(), "64 mb"); + assertThat(new ManifestSidecar.Settings(options).maxBytes).isEqualTo(128 * 1024 * 1024); + + options.setString(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES.key(), "512 kb"); + options.setString(CoreOptions.MANIFEST_TARGET_FILE_SIZE.key(), "1 gb"); + assertThat(new ManifestSidecar.Settings(options).maxBytes).isEqualTo(512 * 1024); + options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES, new MemorySize(Integer.MAX_VALUE - 1L)); + assertThat(new ManifestSidecar.Settings(options).maxBytes).isEqualTo(Integer.MAX_VALUE - 1); + for (String value : new String[] {"127 bytes", "2147483647 bytes", "2 gb"}) { + options.setString(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES.key(), value); + assertThatThrownBy(() -> new ManifestSidecar.Settings(options)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("manifest.sidecar.max-bytes"); + } + } + @Test void minMaxSkipsExactIntersectionChecksAndHandlesOneInterval() throws Exception { byte[] header = header(); @@ -292,7 +315,7 @@ void hugeRangesAreNotExpandedAndInvalidCoverageRetainsBlocks() throws Exception .hasSize(1); } Options options = new Options(); - options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES, 512); + options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES, new MemorySize(512)); builder = new ManifestSidecar.Builder(new ManifestSidecar.Settings(options), header); builder.beginBlock(header.length, 100, 64); for (int i = 0; i < 64; i++) { @@ -306,7 +329,7 @@ void hugeRangesAreNotExpandedAndInvalidCoverageRetainsBlocks() throws Exception 5) .blocks()) .hasSize(1); - options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES, 128); + options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES, new MemorySize(128)); builder = new ManifestSidecar.Builder(new ManifestSidecar.Settings(options), header); assertThat(builder.serialize("m", 1, 2)).isNull(); } @@ -518,7 +541,7 @@ void indexReadsUseBoundedBulkRequests() throws Exception { void indexShortReadsAndExactBudget() throws Exception { byte[] data = golden(); Options options = new Options(); - options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES, data.length); + options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES, new MemorySize(data.length)); Path path = new Path(temp.toString(), "manifest-golden"); for (int maxRead : new int[] {Integer.MAX_VALUE, 7}) { CountingInput stream = new CountingInput(data, maxRead); @@ -541,7 +564,7 @@ void indexShortReadsAndExactBudget() throws Exception { @Test void indexOverBudgetStopsAfterOneExtraByte() throws Exception { Options options = new Options(); - options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES, 128); + options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES, new MemorySize(128)); Path path = new Path(temp.toString(), "manifest-golden"); CountingInput stream = new CountingInput(golden(), Integer.MAX_VALUE); FileIO io = mock(FileIO.class); diff --git a/paimon-python/pypaimon/common/options/core_options.py b/paimon-python/pypaimon/common/options/core_options.py index 370dbb4beea9..d6c5193e09ff 100644 --- a/paimon-python/pypaimon/common/options/core_options.py +++ b/paimon-python/pypaimon/common/options/core_options.py @@ -309,10 +309,14 @@ class CoreOptions: .default_value(False) ) - MANIFEST_SIDECAR_MAX_BYTES: ConfigOption[int] = ( + MANIFEST_SIDECAR_MAX_BYTES: ConfigOption[MemorySize] = ( ConfigOptions.key("manifest.sidecar.max-bytes") - .int_type() - .default_value(8388608) + .memory_type() + .no_default_value() + .with_description( + "Maximum serialized manifest sidecar size, including header and checksum. " + "Defaults to twice manifest.target-file-size." + ) ) MANIFEST_COMPRESSION: ConfigOption[str] = ( @@ -1240,6 +1244,10 @@ def manifest_target_size(self, default=None): default = MemorySize.of_bytes(default) if isinstance(default, int) else MemorySize.parse(default) return self.options.get(CoreOptions.MANIFEST_TARGET_FILE_SIZE, default).get_bytes() + def manifest_sidecar_max_size(self): + size = self.options.get(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES) + return size.get_bytes() if size is not None else 2 * self.manifest_target_size() + def manifest_merge_skip_on_write_only(self, default=None): return self.options.get(CoreOptions.MANIFEST_MERGE_SKIP_ON_WRITE_ONLY, default) diff --git a/paimon-python/pypaimon/manifest/manifest_sidecar.py b/paimon-python/pypaimon/manifest/manifest_sidecar.py index 83e7e2ad9792..eea23c4f7c04 100644 --- a/paimon-python/pypaimon/manifest/manifest_sidecar.py +++ b/paimon-python/pypaimon/manifest/manifest_sidecar.py @@ -52,18 +52,21 @@ class Settings: write: bool = False read: bool = False - max_bytes: int = 8 * 1024 * 1024 + max_bytes: int = 2 * CoreOptions.MANIFEST_TARGET_FILE_SIZE.default_value().get_bytes() def __post_init__(self): - if not 128 <= self.max_bytes <= 64 * 1024 * 1024: - raise ValueError('manifest.sidecar.max-bytes must be in [128, 67108864]') + if not 128 <= self.max_bytes < (1 << 31) - 1: + raise ValueError('manifest.sidecar.max-bytes must be in [128, 2147483646] bytes') @classmethod def from_options(cls, options): + write = options.options.get(CoreOptions.MANIFEST_SIDECAR_WRITE) + read = options.options.get(CoreOptions.MANIFEST_SIDECAR_READ) + # Disabled sidecars must not constrain the manifest target size. + if not write and not read: + return cls() return cls( - options.options.get(CoreOptions.MANIFEST_SIDECAR_WRITE), - options.options.get(CoreOptions.MANIFEST_SIDECAR_READ), - options.options.get(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES)) + write, read, options.manifest_sidecar_max_size()) @dataclass(frozen=True) diff --git a/paimon-python/pypaimon/tests/manifest/manifest_sidecar_test.py b/paimon-python/pypaimon/tests/manifest/manifest_sidecar_test.py index af0d7ab33c68..5e2177c842b7 100644 --- a/paimon-python/pypaimon/tests/manifest/manifest_sidecar_test.py +++ b/paimon-python/pypaimon/tests/manifest/manifest_sidecar_test.py @@ -32,6 +32,7 @@ from unittest.mock import patch from pypaimon.common.options.core_options import CoreOptions +from pypaimon.common.options.options import Options from pypaimon.globalindex.global_index_result import GlobalIndexResult from pypaimon.manifest import manifest_sidecar from pypaimon.manifest.manifest_sidecar import ( @@ -211,6 +212,32 @@ def test_block_short_reads_and_truncation(self): class ManifestSidecarFormatTest(unittest.TestCase): + def test_settings_use_memory_sizes_and_follow_the_manifest_target(self): + options = CoreOptions(Options({})) + self.assertEqual(Settings().max_bytes, 16 * 1024 * 1024) + self.assertEqual(Settings.from_options(options).max_bytes, 16 * 1024 * 1024) + options.options.set(CoreOptions.MANIFEST_SIDECAR_READ, True) + options.options.set(CoreOptions.MANIFEST_TARGET_FILE_SIZE, '12 mb') + self.assertEqual(Settings.from_options(options).max_bytes, 24 * 1024 * 1024) + options.options.set(CoreOptions.MANIFEST_TARGET_FILE_SIZE, '64 mb') + self.assertEqual(Settings.from_options(options).max_bytes, 128 * 1024 * 1024) + options.options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES, '512 kb') + options.options.set(CoreOptions.MANIFEST_TARGET_FILE_SIZE, '1 gb') + self.assertEqual(Settings.from_options(options).max_bytes, 512 * 1024) + options.options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES, '2147483646 bytes') + self.assertEqual(Settings.from_options(options).max_bytes, (1 << 31) - 2) + for value in ('127 bytes', '2147483647 bytes', '2 gb'): + with self.subTest(value=value), self.assertRaisesRegex(ValueError, 'manifest.sidecar.max-bytes'): + options.options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES, value) + Settings.from_options(options) + + def test_disabled_sidecars_do_not_constrain_manifest_target_size(self): + for target in ('1 bytes', '1 gb'): + with self.subTest(target=target): + settings = Settings.from_options(CoreOptions(Options({'manifest.target-file-size': target}))) + self.assertFalse(settings.read) + self.assertFalse(settings.write) + def test_cross_language_and_block_ordinals(self): data, meta, header = golden(), golden_meta(), avro_header() for point in (0, 9, 20, 24, (1 << 32) - 2, 1 << 32, (1 << 32) + 2, From e7bbc3e2b9c830d4a456a353e9482f0b049a0e26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Mon, 14 Sep 2026 15:19:02 +0800 Subject: [PATCH 15/23] [core] Short-circuit manifest sidecar predicate matching --- docs/docs/concepts/spec/manifest.md | 8 +- .../paimon/manifest/ManifestSidecar.java | 26 ++++--- .../manifest/ManifestBlockIndexTest.java | 74 +++++++++++++++++++ .../pypaimon/manifest/manifest_sidecar.py | 22 +++--- .../manifest/manifest_block_index_test.py | 50 ++++++++++++- 5 files changed, 153 insertions(+), 27 deletions(-) diff --git a/docs/docs/concepts/spec/manifest.md b/docs/docs/concepts/spec/manifest.md index 636a9c7e15c4..4f192192d646 100644 --- a/docs/docs/concepts/spec/manifest.md +++ b/docs/docs/concepts/spec/manifest.md @@ -164,9 +164,11 @@ No emitted sidecar omits block descriptors. These are encoded-size bounds; const also incurs bounded object/buffer overhead. Query concurrency multiplies per-reader costs. For conjunctive filters a block is retained only if each dimension is either unavailable -or matches. Matches in different dimensions can come from different entries in the block, -so entry filtering and deletion merging remain necessary. Block min/max is derived from -the first/last interval before testing the individual intervals. +or matches. Matching skips absent filters and short-circuits after a dimension rejects a +block, while known payloads remain validated. Matches in different dimensions can come +from different entries in the block, so entry filtering and deletion merging remain +necessary. Block min/max is derived from the first/last interval before testing the +individual intervals. Readers still consume and validate the whole bounded sidecar. A partition-only query therefore reads row-ID payload bytes too; payload lengths save decoding work for unknown diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java index 5e778e64f1bf..0fe56837d350 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java @@ -550,7 +550,7 @@ public static Selection select( require(header[0] == 'O' && header[1] == 'b' && header[2] == 'j' && header[3] == 1); int partitions = in.getInt(); require(partitions >= 0 && partitions <= in.remaining() / 16); - boolean[] matches = new boolean[partitions]; + boolean[] matches = partitionFilter == null ? null : new boolean[partitions]; Set unique = new java.util.HashSet<>(); for (int id = 0; id < partitions; id++) { require(in.remaining() >= 4); @@ -562,9 +562,7 @@ public static Selection select( require(arity >= 0 && 4L + ((arity + 71L) / 64) * 8 + arity * 8L <= length); require(partitionType == null || arity == partitionType.getFieldCount()); require(unique.add(encoded.asReadOnlyBuffer())); - if (partitionFilter == null) { - matches[id] = true; - } else { + if (partitionFilter != null) { byte[] bytes = new byte[length]; encoded.get(bytes); BinaryRow partition = SerializationUtils.deserializeBinaryRow(bytes); @@ -600,7 +598,9 @@ public static Selection select( int id = partitionPayload.getInt(); require(id > previous && id < partitions); previous = id; - partitionHit |= matches[id]; + if (!partitionHit) { + partitionHit = matches[id]; + } } } require(in.remaining() >= 5); @@ -615,12 +615,14 @@ public static Selection select( require(ranges > 0 && rowPayload.remaining() == 16L * ranges); long min = rowPayload.getLong(); long firstEnd = rowPayload.getLong(); - long max = ranges == 1 ? firstEnd : rowPayload.getLong(rowPayload.limit() - 8); - require(min >= 0 && firstEnd >= min && max >= firstEnd); - boolean candidate = query == null || query.intersects(min, max); - rowHit = - query == null - || (candidate && (ranges == 1 || query.intersects(min, firstEnd))); + require(min >= 0 && firstEnd >= min); + boolean candidate = false; + if (partitionHit && query != null) { + long max = ranges == 1 ? firstEnd : rowPayload.getLong(rowPayload.limit() - 8); + require(max >= firstEnd); + candidate = query.intersects(min, max); + rowHit = candidate && (ranges == 1 || query.intersects(min, firstEnd)); + } long previous = firstEnd; for (int rangeIndex = 1; rangeIndex < ranges; rangeIndex++) { long start = rowPayload.getLong(); @@ -642,7 +644,7 @@ public static Selection select( require(bucketPayload.remaining() >= 4); int pairs = bucketPayload.getInt(); require(pairs > 0 && bucketPayload.remaining() == 8L * pairs); - bucketHit = bucketFilter == null; + bucketHit = !partitionHit || !rowHit || bucketFilter == null; long previous = -1; for (int j = 0; j < pairs; j++) { int bucket = bucketPayload.getInt(); diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java index 2c0a1b4d7d80..846c73d50db3 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java @@ -48,9 +48,12 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.verifyNoMoreInteractions; /** Independent partition/row-ID payloads and conservative resource degradation. */ class ManifestBlockIndexTest { @@ -225,6 +228,77 @@ private byte[] checksum(byte[] data) throws Exception { return data; } + @Test + void partitionMissSkipsRowAndBucketMatching() throws Exception { + byte[] data = fixture("indexWithBuckets"); + ManifestFileMeta meta = meta("manifest-golden", fixture("avroHeader").length + 400, 7); + RowRangeIndex rows = spy(query(0)); + BucketFilter buckets = mock(BucketFilter.class); + assertThat( + ManifestSidecar.select(data, meta, rows, part(99), type, buckets, defaults) + .blocks()) + .isEmpty(); + verifyNoInteractions(rows, buckets); + } + + @Test + void rowMissSkipsBucketMatchingWithOrWithoutPartitionFilter() throws Exception { + byte[] data = fixture("indexWithBuckets"); + ManifestFileMeta meta = meta("manifest-golden", fixture("avroHeader").length + 400, 7); + for (PartitionPredicate partition : Arrays.asList(null, part(7))) { + BucketFilter buckets = mock(BucketFilter.class); + assertThat( + ManifestSidecar.select( + data, meta, query(15), partition, type, buckets, + defaults) + .blocks()) + .isEmpty(); + verifyNoInteractions(buckets); + } + } + + @Test + void absentPartitionFilterKeepsRowAndBucketMatching() throws Exception { + byte[] data = fixture("indexWithBuckets"); + ManifestFileMeta meta = meta("manifest-golden", fixture("avroHeader").length + 400, 7); + BucketFilter buckets = spy(BucketFilter.create(false, 1, null, null)); + assertThat( + ManifestSidecar.select(data, meta, query(20), null, type, buckets, defaults) + .blocks()) + .extracting(block -> block.firstRecord) + .containsExactly(0L); + verify(buckets).mayContain(1, 4); + verify(buckets).mayContain(0, 1); + verify(buckets).mayContain(3, 4); + verifyNoMoreInteractions(buckets); + } + + @Test + void absentRowOrBucketFiltersKeepRemainingDimensions() throws Exception { + byte[] data = fixture("indexWithBuckets"); + ManifestFileMeta meta = meta("manifest-golden", fixture("avroHeader").length + 400, 7); + assertThat( + ManifestSidecar.select( + data, + meta, + null, + part(7), + type, + BucketFilter.create(false, 1, null, null), + defaults) + .blocks()) + .extracting(block -> block.firstRecord) + .containsExactly(0L); + assertThat( + ManifestSidecar.select(data, meta, query(20), part(7), type, null, defaults) + .blocks()) + .extracting(block -> block.firstRecord) + .containsExactly(0L, 5L); + assertThat(ManifestSidecar.select(data, meta, null, null, type, null, defaults).blocks()) + .extracting(block -> block.firstRecord) + .containsExactly(0L, 3L, 5L); + } + @Test void unknownUnsignedEncodingsSkipOnlyTheirDimensionAndMalformedPayloadsFail() throws Exception { byte[] good = fixture("indexWithBuckets"); diff --git a/paimon-python/pypaimon/manifest/manifest_sidecar.py b/paimon-python/pypaimon/manifest/manifest_sidecar.py index eea23c4f7c04..66541d4270ea 100644 --- a/paimon-python/pypaimon/manifest/manifest_sidecar.py +++ b/paimon-python/pypaimon/manifest/manifest_sidecar.py @@ -325,7 +325,7 @@ def select(data, manifest, query, settings, partition_filter=None, partition_fie partitions, = struct.unpack_from('>I', data, offset) offset += 4 _require(partitions <= (limit - offset) // 16) - matches = [] + matches = None if partition_filter is None else [] unique = set() for _ in range(partitions): _require(offset + 4 <= limit) @@ -338,9 +338,7 @@ def select(data, manifest, query, settings, partition_filter=None, partition_fie _require(partition_fields is None or arity == len(partition_fields)) _require(partition not in unique) unique.add(partition) - if partition_filter is None: - matches.append(True) - else: + if partition_filter is not None: _require(partition_fields is not None) matches.append(partition_filter.test(GenericRowDeserializer.from_bytes(partition, partition_fields))) offset += length @@ -373,7 +371,8 @@ def select(data, manifest, query, settings, partition_filter=None, partition_fie id_, = struct.unpack_from('>i', data, offset + 4 + 4 * j) _require(previous < id_ < partitions) previous = id_ - partition_hit |= matches[id_] + if not partition_hit: + partition_hit = matches[id_] offset += payload_length _require(offset + 5 <= limit) row_encoding, payload_length = struct.unpack_from('>BI', data, offset) @@ -387,10 +386,13 @@ def select(data, manifest, query, settings, partition_filter=None, partition_fie ranges, = struct.unpack_from('>I', data, offset) _require(ranges > 0 and 4 + 16 * ranges == payload_length) min_row_id, first_end = PAIR.unpack_from(data, offset + 4) - max_row_id = first_end if ranges == 1 else LONG.unpack_from(data, offset + payload_length - 8)[0] - _require(min_row_id >= 0 and first_end >= min_row_id and max_row_id >= first_end) - candidate = query is None or query.intersects(min_row_id, max_row_id) - row_hit = query is None or (candidate and (ranges == 1 or query.intersects(min_row_id, first_end))) + _require(min_row_id >= 0 and first_end >= min_row_id) + candidate = False + if partition_hit and query is not None: + max_row_id = first_end if ranges == 1 else LONG.unpack_from(data, offset + payload_length - 8)[0] + _require(max_row_id >= first_end) + candidate = query.intersects(min_row_id, max_row_id) + row_hit = candidate and (ranges == 1 or query.intersects(min_row_id, first_end)) previous = first_end for range_position in range(1, ranges): start, end = PAIR.unpack_from(data, offset + 4 + 16 * range_position) @@ -410,7 +412,7 @@ def select(data, manifest, query, settings, partition_filter=None, partition_fie _require(payload_length >= 4) pairs, = struct.unpack_from('>I', data, offset) _require(pairs > 0 and payload_length == 4 + 8 * pairs) - bucket_hit = bucket_filter is None + bucket_hit = not partition_hit or not row_hit or bucket_filter is None previous = (-1, -1) for j in range(pairs): bucket, total_buckets = struct.unpack_from('>ii', data, offset + 4 + 8 * j) diff --git a/paimon-python/pypaimon/tests/manifest/manifest_block_index_test.py b/paimon-python/pypaimon/tests/manifest/manifest_block_index_test.py index c2bccc2c058a..688e2351b30e 100644 --- a/paimon-python/pypaimon/tests/manifest/manifest_block_index_test.py +++ b/paimon-python/pypaimon/tests/manifest/manifest_block_index_test.py @@ -21,11 +21,12 @@ import random from pathlib import Path from types import SimpleNamespace -from unittest.mock import patch +from unittest.mock import Mock, patch import pytest -from pypaimon.manifest.manifest_sidecar import Builder, Settings, select +from pypaimon.manifest import manifest_sidecar +from pypaimon.manifest.manifest_sidecar import Builder, Query, Settings, select from pypaimon.schema.data_types import AtomicType, DataField from pypaimon.table.row.generic_row import GenericRow, GenericRowSerializer from pypaimon.tests.manifest.manifest_sidecar_test import avro_header, golden, golden_meta @@ -140,6 +141,51 @@ def checksum(data): return data +def test_partition_miss_skips_row_and_bucket_matching(): + rows = Query([Range(0, 0)]) + buckets = Mock(return_value=True) + with patch.object(rows, 'intersects', wraps=rows.intersects) as intersects, \ + patch.object(manifest_sidecar, 'LONG', wraps=manifest_sidecar.LONG) as bounds: + selected = select(fixture('indexWithBuckets'), golden_meta(), rows, Settings(), part(99), FIELDS, buckets) + assert not selected.blocks + intersects.assert_not_called() + bounds.unpack_from.assert_not_called() + buckets.assert_not_called() + + +@pytest.mark.parametrize('partition_filter', [None, part(7)]) +def test_row_miss_skips_bucket_matching(partition_filter): + buckets = Mock(return_value=True) + selected = select(fixture('indexWithBuckets'), golden_meta(), [Range(15, 15)], + Settings(), partition_filter, FIELDS, buckets) + assert not selected.blocks + buckets.assert_not_called() + + +def test_absent_partition_filter_keeps_row_and_bucket_matching(): + buckets = Mock(side_effect=lambda bucket, total: bucket == 1) + with patch('pypaimon.manifest.manifest_sidecar.GenericRowDeserializer.from_bytes') as decode_partition: + selected = select(fixture('indexWithBuckets'), golden_meta(), [Range(20, 20)], + Settings(), None, FIELDS, buckets) + assert [b.first_record for b in selected.blocks] == [0] + decode_partition.assert_not_called() + assert [call.args for call in buckets.call_args_list] == [(1, 4), (0, 1), (3, 4)] + + +def test_absent_row_or_bucket_filters_keep_remaining_dimensions(): + data = fixture('indexWithBuckets') + buckets = Mock(side_effect=lambda bucket, total: bucket == 1) + with patch.object(manifest_sidecar, 'LONG', wraps=manifest_sidecar.LONG) as bounds: + selected = select(data, golden_meta(), None, Settings(), part(7), FIELDS, buckets) + assert [b.first_record for b in selected.blocks] == [0] + bounds.unpack_from.assert_not_called() + assert [call.args for call in buckets.call_args_list] == [(1, 4), (2, 4), (2, 8), (0, 1), (3, 4)] + selected = select(data, golden_meta(), [Range(20, 20)], Settings(), part(7), FIELDS) + assert [b.first_record for b in selected.blocks] == [0, 5] + selected = select(data, golden_meta(), None, Settings()) + assert [b.first_record for b in selected.blocks] == [0, 3, 5] + + def test_unsigned_unknown_encodings_skip_only_one_payload_and_validate_lengths(): good = fixture('indexWithBuckets') block, p, r, b = positions(good)[0] From 853c45b2548d7d0c113f913283448d58828ef0c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Mon, 14 Sep 2026 15:29:59 +0800 Subject: [PATCH 16/23] [core] Omit length fields for unavailable sidecar payloads --- docs/docs/concepts/spec/manifest.md | 22 ++- .../paimon/manifest/ManifestSidecar.java | 166 +++++++++-------- .../manifest/ManifestBlockIndexTest.java | 80 ++++++-- .../paimon/manifest/ManifestSidecarTest.java | 2 +- .../src/test/resources/manifest-sidecar.txt | 4 +- .../pypaimon/manifest/manifest_sidecar.py | 174 ++++++++++-------- .../manifest/manifest_block_index_test.py | 56 +++++- .../tests/manifest/manifest_sidecar_test.py | 2 +- 8 files changed, 308 insertions(+), 198 deletions(-) diff --git a/docs/docs/concepts/spec/manifest.md b/docs/docs/concepts/spec/manifest.md index 4f192192d646..5a8e2007a478 100644 --- a/docs/docs/concepts/spec/manifest.md +++ b/docs/docs/concepts/spec/manifest.md @@ -100,14 +100,17 @@ blocks[] // original physical order length : long // complete encoded block, including sync marker recordCount : long partitionEncoding : byte - partitionPayloadLength : int - partitionPayload : bytes + if partitionEncoding != 0: + partitionPayloadLength : int + partitionPayload : bytes rowIdEncoding : byte - rowIdPayloadLength : int - rowIdPayload : bytes + if rowIdEncoding != 0: + rowIdPayloadLength : int + rowIdPayload : bytes bucketEncoding : byte - bucketPayloadLength : int - bucketPayload : bytes + if bucketEncoding != 0: + bucketPayloadLength : int + bucketPayload : bytes checksum : 32 bytes // SHA-256 of all preceding bytes ``` @@ -118,13 +121,14 @@ serialized tuple. Partition predicates are evaluated once per dictionary entry. | Dimension | Encoding | Payload | | --- | --- | --- | -| Any | `0` | Unavailable; payload length must be zero. | +| Any | `0` | Unavailable; only the encoding byte is present. | | Partition | `1` | Positive `partitionIdCount: int` followed by sorted unique dictionary IDs (`int`). | | Row ID | `1` | Positive `rangeCount: int` followed by sorted disjoint inclusive `(start: long, end: long)` pairs. Coverage may conservatively include gaps. | | Bucket | `1` | Positive `pairCount: int` followed by sorted unique `(bucket: int, totalBuckets: int)` pairs. | | Any | Other nonzero ID | Skip exactly the bounded payload length; treat only this dimension as unavailable. | -Payload lengths exclude the encoding and length fields. Invalid lengths, known-payload +Only nonzero encodings are followed by a length and payload. Payload lengths exclude +the encoding and length fields. Invalid lengths, known-payload framing, dictionary references, interval order, checksums or physical coverage invalidate the container. Byte spans must cover the entire original manifest after its header; record counts must sum to the manifest entry count. Readers continue validating blocks @@ -135,7 +139,7 @@ Bucket encoding 1 contains a positive `pairCount: int` followed by that many and deduplicated. They preserve bucket-count changes between writes; the bucket number alone is not sufficient for point lookup after rescaling. A valid pair satisfies `0 <= bucket < totalBuckets`. Missing, invalid, negative/synthetic or over-budget bucket -metadata makes that block's bucket coverage unavailable (encoding 0, length 0). Partition +metadata makes that block's bucket coverage unavailable (encoding 0, no length or payload). Partition and row-ID coverage remain independently usable; no mutual-exclusion restriction is imposed. Readers test bucket-only queries using the existing bucket-selection logic, including diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java index 0fe56837d350..d9a12a34e19c 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java @@ -68,7 +68,7 @@ public final class ManifestSidecar { private static final long MAGIC = 0x5041494d53434152L; private static final int FORMAT_VERSION = 1; private static final int HEADER_BYTES = 60; - private static final int BLOCK_BYTES = 39; + private static final int BLOCK_BYTES = 27; private static final int MAX_BLOCKS = 131072; private static final byte[] EMPTY = new byte[0]; private static final int DIGEST_BYTES = 32; @@ -269,7 +269,7 @@ public void add( end = Math.max(end, next.getValue()); ranges.remove(next.getKey()); } - if (4L + 16L * (ranges.size() + 1L) > settings.maxBytes - optionalBytes) { + if (8L + 16L * (ranges.size() + 1L) > settings.maxBytes - optionalBytes) { coarse = true; ranges.clear(); } else { @@ -292,7 +292,7 @@ private void addBucket(@Nullable Integer bucket, @Nullable Integer totalBuckets) } long pair = ((long) bucket << 32) | totalBuckets; if (!bucketPairs.contains(pair) - && 4L + 8L * (bucketPairs.size() + 1L) > settings.maxBytes - optionalBytes) { + && 8L + 8L * (bucketPairs.size() + 1L) > settings.maxBytes - optionalBytes) { bucketAvailable = false; bucketPairs.clear(); } else { @@ -331,40 +331,40 @@ public void endBlock() throws IOException { byte[] rowPayload = EMPTY; byte[] partitionPayload = EMPTY; if (rowAvailable) { - if (coarse || 4L + 16L * ranges.size() > settings.maxBytes - optionalBytes) { + if (coarse || 8L + 16L * ranges.size() > settings.maxBytes - optionalBytes) { ranges.clear(); ranges.put(min, max); } - if (4L + 16L * ranges.size() <= settings.maxBytes - optionalBytes) { + if (8L + 16L * ranges.size() <= settings.maxBytes - optionalBytes) { ByteBuffer out = ByteBuffer.allocate(4 + 16 * ranges.size()); out.putInt(ranges.size()); for (Map.Entry range : ranges.entrySet()) { out.putLong(range.getKey()).putLong(range.getValue()); } rowPayload = out.array(); - optionalBytes += rowPayload.length; + optionalBytes += payloadSize(rowPayload); } } if (partitionAvailable - && 4L + 4L * partitionIds.size() <= settings.maxBytes - optionalBytes) { + && 8L + 4L * partitionIds.size() <= settings.maxBytes - optionalBytes) { ByteBuffer out = ByteBuffer.allocate(4 + 4 * partitionIds.size()); out.putInt(partitionIds.size()); for (int id : partitionIds) { out.putInt(id); } partitionPayload = out.array(); - optionalBytes += partitionPayload.length; + optionalBytes += payloadSize(partitionPayload); } byte[] bucketPayload = EMPTY; if (bucketAvailable - && 4L + 8L * bucketPairs.size() <= settings.maxBytes - optionalBytes) { + && 8L + 8L * bucketPairs.size() <= settings.maxBytes - optionalBytes) { ByteBuffer out = ByteBuffer.allocate(4 + 8 * bucketPairs.size()); out.putInt(bucketPairs.size()); for (long pair : bucketPairs) { out.putInt((int) (pair >>> 32)).putInt((int) pair); } bucketPayload = out.array(); - optionalBytes += bucketPayload.length; + optionalBytes += payloadSize(bucketPayload); } blocks.add(new IndexedBlock(current, partitionPayload, rowPayload, bucketPayload)); nextOffset = Math.addExact(current.offset, current.length); @@ -394,16 +394,16 @@ public byte[] serialize(String name, long fileSize, long entryCount) throws IOEx if (size <= settings.maxBytes) { break; } - size -= block.rowIds.length; - optionalBytes -= block.rowIds.length; + size -= payloadSize(block.rowIds); + optionalBytes -= payloadSize(block.rowIds); block.rowIds = EMPTY; } for (IndexedBlock block : blocks) { if (size <= settings.maxBytes) { break; } - size -= block.buckets.length; - optionalBytes -= block.buckets.length; + size -= payloadSize(block.buckets); + optionalBytes -= payloadSize(block.buckets); block.buckets = EMPTY; } if (size > settings.maxBytes) { @@ -411,8 +411,8 @@ public byte[] serialize(String name, long fileSize, long entryCount) throws IOEx dictionaryBytes = 0; dictionary.clear(); for (IndexedBlock block : blocks) { - size -= block.partitions.length; - optionalBytes -= block.partitions.length; + size -= payloadSize(block.partitions); + optionalBytes -= payloadSize(block.partitions); block.partitions = EMPTY; } } @@ -444,10 +444,16 @@ public byte[] serialize(String name, long fileSize, long entryCount) throws IOEx return buffer.toByteArray(); } + private static int payloadSize(byte[] payload) { + return payload.length == 0 ? 0 : Integer.BYTES + payload.length; + } + private static void writePayload(DataOutputStream out, byte[] payload) throws IOException { out.writeByte(payload.length == 0 ? 0 : 1); - out.writeInt(payload.length); - out.write(payload); + if (payload.length > 0) { + out.writeInt(payload.length); + out.write(payload); + } } } @@ -584,77 +590,81 @@ public static Selection select( require(offset == nextOffset && length > 0 && length <= manifest.fileSize() - offset); require(records > 0 && records <= entries - firstRecord); int partitionEncoding = Byte.toUnsignedInt(in.get()); - ByteBuffer partitionPayload = payload(in); boolean partitionHit = true; - if (partitionEncoding == 0) { - require(!partitionPayload.hasRemaining()); - } else if (partitionEncoding == 1) { - require(partitionPayload.remaining() >= 4); - int ids = partitionPayload.getInt(); - require(ids > 0 && ids <= partitions && partitionPayload.remaining() == 4L * ids); - partitionHit = partitionFilter == null; - int previous = -1; - for (int j = 0; j < ids; j++) { - int id = partitionPayload.getInt(); - require(id > previous && id < partitions); - previous = id; - if (!partitionHit) { - partitionHit = matches[id]; + if (partitionEncoding != 0) { + ByteBuffer partitionPayload = payload(in); + if (partitionEncoding == 1) { + require(partitionPayload.remaining() >= 4); + int ids = partitionPayload.getInt(); + require( + ids > 0 + && ids <= partitions + && partitionPayload.remaining() == 4L * ids); + partitionHit = partitionFilter == null; + int previous = -1; + for (int j = 0; j < ids; j++) { + int id = partitionPayload.getInt(); + require(id > previous && id < partitions); + previous = id; + if (!partitionHit) { + partitionHit = matches[id]; + } } } } - require(in.remaining() >= 5); + require(in.remaining() >= 1); int rowEncoding = Byte.toUnsignedInt(in.get()); - ByteBuffer rowPayload = payload(in); boolean rowHit = true; - if (rowEncoding == 0) { - require(!rowPayload.hasRemaining()); - } else if (rowEncoding == 1) { - require(rowPayload.remaining() >= 4); - int ranges = rowPayload.getInt(); - require(ranges > 0 && rowPayload.remaining() == 16L * ranges); - long min = rowPayload.getLong(); - long firstEnd = rowPayload.getLong(); - require(min >= 0 && firstEnd >= min); - boolean candidate = false; - if (partitionHit && query != null) { - long max = ranges == 1 ? firstEnd : rowPayload.getLong(rowPayload.limit() - 8); - require(max >= firstEnd); - candidate = query.intersects(min, max); - rowHit = candidate && (ranges == 1 || query.intersects(min, firstEnd)); - } - long previous = firstEnd; - for (int rangeIndex = 1; rangeIndex < ranges; rangeIndex++) { - long start = rowPayload.getLong(); - long end = rowPayload.getLong(); - require(start >= 0 && end >= start && start > previous); - previous = end; - if (candidate && !rowHit) { - rowHit = query.intersects(start, end); + if (rowEncoding != 0) { + ByteBuffer rowPayload = payload(in); + if (rowEncoding == 1) { + require(rowPayload.remaining() >= 4); + int ranges = rowPayload.getInt(); + require(ranges > 0 && rowPayload.remaining() == 16L * ranges); + long min = rowPayload.getLong(); + long firstEnd = rowPayload.getLong(); + require(min >= 0 && firstEnd >= min); + boolean candidate = false; + if (partitionHit && query != null) { + long max = + ranges == 1 ? firstEnd : rowPayload.getLong(rowPayload.limit() - 8); + require(max >= firstEnd); + candidate = query.intersects(min, max); + rowHit = candidate && (ranges == 1 || query.intersects(min, firstEnd)); + } + long previous = firstEnd; + for (int rangeIndex = 1; rangeIndex < ranges; rangeIndex++) { + long start = rowPayload.getLong(); + long end = rowPayload.getLong(); + require(start >= 0 && end >= start && start > previous); + previous = end; + if (candidate && !rowHit) { + rowHit = query.intersects(start, end); + } } } } - require(in.remaining() >= 5); + require(in.remaining() >= 1); int bucketEncoding = Byte.toUnsignedInt(in.get()); - ByteBuffer bucketPayload = payload(in); boolean bucketHit = true; - if (bucketEncoding == 0) { - require(!bucketPayload.hasRemaining()); - } else if (bucketEncoding == 1) { - require(bucketPayload.remaining() >= 4); - int pairs = bucketPayload.getInt(); - require(pairs > 0 && bucketPayload.remaining() == 8L * pairs); - bucketHit = !partitionHit || !rowHit || bucketFilter == null; - long previous = -1; - for (int j = 0; j < pairs; j++) { - int bucket = bucketPayload.getInt(); - int totalBuckets = bucketPayload.getInt(); - require(bucket >= 0 && totalBuckets > bucket); - long pair = ((long) bucket << 32) | totalBuckets; - require(pair > previous); - previous = pair; - if (!bucketHit) { - bucketHit = bucketFilter.mayContain(bucket, totalBuckets); + if (bucketEncoding != 0) { + ByteBuffer bucketPayload = payload(in); + if (bucketEncoding == 1) { + require(bucketPayload.remaining() >= 4); + int pairs = bucketPayload.getInt(); + require(pairs > 0 && bucketPayload.remaining() == 8L * pairs); + bucketHit = !partitionHit || !rowHit || bucketFilter == null; + long previous = -1; + for (int j = 0; j < pairs; j++) { + int bucket = bucketPayload.getInt(); + int totalBuckets = bucketPayload.getInt(); + require(bucket >= 0 && totalBuckets > bucket); + long pair = ((long) bucket << 32) | totalBuckets; + require(pair > previous); + previous = pair; + if (!bucketHit) { + bucketHit = bucketFilter.mayContain(bucket, totalBuckets); + } } } } diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java index 846c73d50db3..e9f2dc9cc592 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java @@ -204,23 +204,23 @@ private List positions(byte[] data) { for (int i = 0; i < blocks; i++) { int block = in.position(); in.position(block + 24); - int partition = in.position(); - in.get(); - int length = in.getInt(); - in.position(in.position() + length); - int row = in.position(); - in.get(); - length = in.getInt(); - in.position(in.position() + length); - int bucket = in.position(); - in.get(); - length = in.getInt(); - in.position(in.position() + length); + int partition = skipPayload(in); + int row = skipPayload(in); + int bucket = skipPayload(in); result.add(new int[] {block, partition, row, bucket}); } return result; } + private int skipPayload(ByteBuffer in) { + int position = in.position(); + if (in.get() != 0) { + int length = in.getInt(); + in.position(in.position() + length); + } + return position; + } + private byte[] checksum(byte[] data) throws Exception { byte[] hash = MessageDigest.getInstance("SHA-256").digest(Arrays.copyOf(data, data.length - 32)); @@ -344,7 +344,7 @@ data, meta, query(20), part(99), type, noBucket, defaults) .isEmpty(); for (int position : new int[] {first[1], first[2], first[3]}) { byte[] bad = good.clone(); - bad[position] = 0; // encoding 0 cannot have payload bytes + bad[position] = 0; // encoding 0 cannot have a length or payload bytes checksum(bad); assertThatThrownBy(() -> ManifestSidecar.select(bad, meta, query(0), defaults)) .isInstanceOf(IOException.class); @@ -488,7 +488,7 @@ void unknownInvalidOrOverBudgetBucketPayloadIsUnavailable() throws Exception { byte[] data = builder.serialize("m", header.length + 200, 3 + extraPairs); int bucket = positions(data).get(0)[3]; assertThat(data[bucket]).isZero(); - assertThat(ByteBuffer.wrap(data).getInt(bucket + 1)).isZero(); + assertThat(positions(data).get(1)[0]).isEqualTo(bucket + 1); ManifestFileMeta meta = meta("m", header.length + 200, 3 + extraPairs); assertThat( ManifestSidecar.select( @@ -584,10 +584,58 @@ void payloadsCanExceedFormerLimitsWithinByteBudget() throws Exception { .isEmpty(); } + @Test + void absentPayloadsOmitLengthFieldsForEveryDimensionCombination() throws Exception { + byte[] header = fixture("avroHeader"); + ManifestSidecar.Builder builder = new ManifestSidecar.Builder(defaults, header); + for (int mask = 0; mask < 8; mask++) { + builder.beginBlock(header.length + mask * 100L, 100, 1); + builder.add( + (mask & 2) == 0 ? null : 100L + mask, + 1, + (mask & 1) == 0 ? null : partition(7, "left"), + (mask & 4) == 0 ? null : 0, + (mask & 4) == 0 ? null : 1); + builder.endBlock(); + } + byte[] data = builder.serialize("m", header.length + 800, 8); + List positions = positions(data); + int[] presentSizes = {13, 25, 17}; + for (int mask = 0; mask < 8; mask++) { + for (int dimension = 0; dimension < 3; dimension++) { + int start = positions.get(mask)[dimension + 1]; + int end = + dimension < 2 + ? positions.get(mask)[dimension + 2] + : mask < 7 ? positions.get(mask + 1)[0] : data.length - 32; + boolean present = (mask & (1 << dimension)) != 0; + assertThat(data[start]).isEqualTo((byte) (present ? 1 : 0)); + assertThat(end - start).isEqualTo(present ? presentSizes[dimension] : 1); + } + } + ManifestFileMeta meta = meta("m", header.length + 800, 8); + assertThat(ManifestSidecar.select(data, meta, null, part(99), type, defaults).blocks()) + .extracting(block -> block.firstRecord) + .containsExactly(0L, 2L, 4L, 6L); + assertThat(ManifestSidecar.select(data, meta, query(999), defaults).blocks()) + .extracting(block -> block.firstRecord) + .containsExactly(0L, 1L, 4L, 5L); + BucketFilter buckets = BucketFilter.create(false, 99, null, null); + assertThat(ManifestSidecar.select(data, meta, null, null, type, buckets, defaults).blocks()) + .extracting(block -> block.firstRecord) + .containsExactly(0L, 1L, 2L, 3L); + assertThat( + ManifestSidecar.select( + data, meta, query(999), part(99), type, buckets, defaults) + .blocks()) + .extracting(block -> block.firstRecord) + .containsExactly(0L); + } + @Test void tightByteBudgetKeepsAllDescriptorsOrOmitsTheWholeFile() throws Exception { Options options = new Options(); - options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES, new MemorySize(280)); + options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES, new MemorySize(250)); ManifestSidecar.Settings settings = new ManifestSidecar.Settings(options); byte[] header = fixture("avroHeader"); ManifestSidecar.Builder builder = new ManifestSidecar.Builder(settings, header); @@ -597,7 +645,7 @@ void tightByteBudgetKeepsAllDescriptorsOrOmitsTheWholeFile() throws Exception { builder.endBlock(); } byte[] data = builder.serialize("m", header.length + 300, 3); - assertThat(data.length).isLessThanOrEqualTo(280); + assertThat(data.length).isLessThanOrEqualTo(250); assertThat( ManifestSidecar.select( data, diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java index 067a59ce4c8a..c5283ee21c3e 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java @@ -248,7 +248,7 @@ void singleIntervalHandlesBoundariesAndAbsentQueries() throws Exception { @Test void malformedIntervalsStillFallbackAfterMinMaxRejectionOrAnEarlyHit() throws Exception { - int firstBlockIntervals = 60 + 4 + header().length + 4 + 4 + 24 + 5 + 5 + 4; + int firstBlockIntervals = 60 + 4 + header().length + 4 + 4 + 24 + 1 + 5 + 4; ManifestFileMeta meta = goldenMeta(); for (long[] mutation : new long[][] {{0, -1}, {8, -1}, {8, 30}, {16, 9}, {24, 19}}) { byte[] data = golden(); diff --git a/paimon-core/src/test/resources/manifest-sidecar.txt b/paimon-core/src/test/resources/manifest-sidecar.txt index f8a914f451d8..f2199ce2e4f0 100644 --- a/paimon-core/src/test/resources/manifest-sidecar.txt +++ b/paimon-core/src/test/resources/manifest-sidecar.txt @@ -18,6 +18,6 @@ avroHeader=T2JqAQQUYXZyby5jb2RlYwhudWxsFmF2cm8uc2NoZW1hDCJsb25nIgAAAAAAAAAAAAAAAAAAAAAA partitionA=AAAAAgAAAAAAAAAABwAAAAAAAABsZWZ0AAAAhA== partitionB=AAAAAgACAAAAAAAACQAAAAAAAAAAAAAAAAAAAA== -index=UEFJTVNDQVIAAAABL0eWubkHdL6ioNdIPj3jGtp9TAk83wbtAsmc+xJVwM8AAAAAAAAByQAAAAAAAAAHAAAAOU9iagEEFGF2cm8uY29kZWMIbnVsbBZhdnJvLnNjaGVtYQwibG9uZyIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAADkAAAAAAAAAZAAAAAAAAAADAAAAAAABAAAAJAAAAAIAAAAAAAAAAAAAAAAAAAAJAAAAAAAAABQAAAAAAAAAGAAAAAAAAAAAAAAAAJ0AAAAAAAAAyAAAAAAAAAACAAAAAAABAAAAJAAAAAIAAAAA/////gAAAAEAAAACAAAHgcw4bGUAAAeBzDhsZQAAAAAAAAAAAAAAAWUAAAAAAAAAZAAAAAAAAAACAAAAAAABAAAAJAAAAAIAAAAAAAAAFAAAAAAAAAAYf/////////9//////////wAAAAAAvl+JhwByNVfzaEZmgXTZEMYB85Mh4bR1hf61USimKec= -indexWithPartitions=UEFJTVNDQVIAAAABL0eWubkHdL6ioNdIPj3jGtp9TAk83wbtAsmc+xJVwM8AAAAAAAAByQAAAAAAAAAHAAAAOU9iagEEFGF2cm8uY29kZWMIbnVsbBZhdnJvLnNjaGVtYQwibG9uZyIAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAcAAAAAgAAAAAAAAAABwAAAAAAAABsZWZ0AAAAhAAAABwAAAACAAIAAAAAAAAJAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAA5AAAAAAAAAGQAAAAAAAAAAwEAAAAMAAAAAgAAAAAAAAABAQAAACQAAAACAAAAAAAAAAAAAAAAAAAACQAAAAAAAAAUAAAAAAAAABgAAAAAAAAAAAAAAACdAAAAAAAAAMgAAAAAAAAAAgEAAAAMAAAAAgAAAAAAAAABAQAAACQAAAACAAAAAP////4AAAABAAAAAgAAB4HMOGxlAAAHgcw4bGUAAAAAAAAAAAAAAAFlAAAAAAAAAGQAAAAAAAAAAgEAAAAMAAAAAgAAAAAAAAABAQAAACQAAAACAAAAAAAAABQAAAAAAAAAGH//////////f/////////8AAAAAACeaHd6e41hdWskZ6DqWibQKE2TYT04EVjoN1YBmh9VF +index=UEFJTVNDQVIAAAABL0eWubkHdL6ioNdIPj3jGtp9TAk83wbtAsmc+xJVwM8AAAAAAAAByQAAAAAAAAAHAAAAOU9iagEEFGF2cm8uY29kZWMIbnVsbBZhdnJvLnNjaGVtYQwibG9uZyIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAADkAAAAAAAAAZAAAAAAAAAADAAEAAAAkAAAAAgAAAAAAAAAAAAAAAAAAAAkAAAAAAAAAFAAAAAAAAAAYAAAAAAAAAACdAAAAAAAAAMgAAAAAAAAAAgABAAAAJAAAAAIAAAAA/////gAAAAEAAAACAAAHgcw4bGUAAAeBzDhsZQAAAAAAAAABZQAAAAAAAABkAAAAAAAAAAIAAQAAACQAAAACAAAAAAAAABQAAAAAAAAAGH//////////f/////////8AlOHbHSWYVK7dgM+l6yo0+LjOLGYNf4lnkt2OtValVYA= +indexWithPartitions=UEFJTVNDQVIAAAABL0eWubkHdL6ioNdIPj3jGtp9TAk83wbtAsmc+xJVwM8AAAAAAAAByQAAAAAAAAAHAAAAOU9iagEEFGF2cm8uY29kZWMIbnVsbBZhdnJvLnNjaGVtYQwibG9uZyIAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAcAAAAAgAAAAAAAAAABwAAAAAAAABsZWZ0AAAAhAAAABwAAAACAAIAAAAAAAAJAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAA5AAAAAAAAAGQAAAAAAAAAAwEAAAAMAAAAAgAAAAAAAAABAQAAACQAAAACAAAAAAAAAAAAAAAAAAAACQAAAAAAAAAUAAAAAAAAABgAAAAAAAAAAJ0AAAAAAAAAyAAAAAAAAAACAQAAAAwAAAACAAAAAAAAAAEBAAAAJAAAAAIAAAAA/////gAAAAEAAAACAAAHgcw4bGUAAAeBzDhsZQAAAAAAAAABZQAAAAAAAABkAAAAAAAAAAIBAAAADAAAAAIAAAAAAAAAAQEAAAAkAAAAAgAAAAAAAAAUAAAAAAAAABh//////////3//////////AInpuqbYtRIEgHOW0YNU5V+bvWbXg6ARpSP/hXdwewB8 indexWithBuckets=UEFJTVNDQVIAAAABL0eWubkHdL6ioNdIPj3jGtp9TAk83wbtAsmc+xJVwM8AAAAAAAAByQAAAAAAAAAHAAAAOU9iagEEFGF2cm8uY29kZWMIbnVsbBZhdnJvLnNjaGVtYQwibG9uZyIAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAcAAAAAgAAAAAAAAAABwAAAAAAAABsZWZ0AAAAhAAAABwAAAACAAIAAAAAAAAJAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAA5AAAAAAAAAGQAAAAAAAAAAwEAAAAMAAAAAgAAAAAAAAABAQAAACQAAAACAAAAAAAAAAAAAAAAAAAACQAAAAAAAAAUAAAAAAAAABgBAAAAFAAAAAIAAAABAAAABAAAAAEAAAAIAAAAAAAAAJ0AAAAAAAAAyAAAAAAAAAACAQAAAAwAAAACAAAAAAAAAAEBAAAAJAAAAAIAAAAA/////gAAAAEAAAACAAAHgcw4bGUAAAeBzDhsZQEAAAAUAAAAAgAAAAIAAAAEAAAAAgAAAAgAAAAAAAABZQAAAAAAAABkAAAAAAAAAAIBAAAADAAAAAIAAAAAAAAAAQEAAAAkAAAAAgAAAAAAAAAUAAAAAAAAABh//////////3//////////AQAAABQAAAACAAAAAAAAAAEAAAADAAAABGCAy0ixcqKywlCau1+E/XLEorySmCmL1zl5q9/dwdBu diff --git a/paimon-python/pypaimon/manifest/manifest_sidecar.py b/paimon-python/pypaimon/manifest/manifest_sidecar.py index 66541d4270ea..8ec2c2010463 100644 --- a/paimon-python/pypaimon/manifest/manifest_sidecar.py +++ b/paimon-python/pypaimon/manifest/manifest_sidecar.py @@ -42,7 +42,7 @@ HEADER = struct.Struct('>8sI32sqq') BLOCK = struct.Struct('>qqq') MAX_BLOCKS = 131072 -BLOCK_BYTES = 39 +BLOCK_BYTES = 27 PAIR = struct.Struct('>qq') LONG = struct.Struct('>q') _PROPAGATED_ERRORS = (InterruptedError, CancelledError, ArrowCancelled, MemoryError, RecursionError) @@ -94,6 +94,10 @@ def intersects(self, first, last): return candidate < len(self.starts) and self.starts[candidate] <= last +def _payload_size(payload): + return 4 + len(payload) if payload else 0 + + class Builder: def __init__(self, settings, header): self.settings = settings @@ -159,7 +163,7 @@ def add(self, first, count, partition=None, bucket=None, total_buckets=None): first = min(first, self.ranges[right][0]) end = max(end, self.ranges[right][1]) right += 1 - if (4 + 16 * (len(self.ranges) - (right - left) + 1) + if (8 + 16 * (len(self.ranges) - (right - left) + 1) > self.settings.max_bytes - self.optional_bytes): self.coarse = True self.ranges.clear() @@ -176,7 +180,7 @@ def _add_bucket(self, bucket, total_buckets): return pair = (bucket, total_buckets) if (pair not in self.bucket_pairs - and 4 + 8 * (len(self.bucket_pairs) + 1) > self.settings.max_bytes - self.optional_bytes): + and 8 + 8 * (len(self.bucket_pairs) + 1) > self.settings.max_bytes - self.optional_bytes): self.bucket_available = False self.bucket_pairs.clear() else: @@ -209,22 +213,22 @@ def end_block(self): row_payload = partition_payload = b'' if self.row_available: if (self.coarse - or 4 + 16 * len(self.ranges) > self.settings.max_bytes - self.optional_bytes): + or 8 + 16 * len(self.ranges) > self.settings.max_bytes - self.optional_bytes): self.ranges = [(self.min, self.max)] - if 4 + 16 * len(self.ranges) <= self.settings.max_bytes - self.optional_bytes: + if 8 + 16 * len(self.ranges) <= self.settings.max_bytes - self.optional_bytes: row_payload = struct.pack('>I', len(self.ranges)) + b''.join(PAIR.pack(*r) for r in self.ranges) - self.optional_bytes += len(row_payload) + self.optional_bytes += _payload_size(row_payload) if (self.partition_available - and 4 + 4 * len(self.partition_ids) <= self.settings.max_bytes - self.optional_bytes): + and 8 + 4 * len(self.partition_ids) <= self.settings.max_bytes - self.optional_bytes): partition_payload = struct.pack('>I', len(self.partition_ids)) partition_payload += b''.join(struct.pack('>I', id_) for id_ in sorted(self.partition_ids)) - self.optional_bytes += len(partition_payload) + self.optional_bytes += _payload_size(partition_payload) bucket_payload = b'' if (self.bucket_available - and 4 + 8 * len(self.bucket_pairs) <= self.settings.max_bytes - self.optional_bytes): + and 8 + 8 * len(self.bucket_pairs) <= self.settings.max_bytes - self.optional_bytes): bucket_payload = struct.pack('>I', len(self.bucket_pairs)) bucket_payload += b''.join(struct.pack('>ii', *pair) for pair in sorted(self.bucket_pairs)) - self.optional_bytes += len(bucket_payload) + self.optional_bytes += _payload_size(bucket_payload) self.blocks.append([block, partition_payload, row_payload, bucket_payload]) self.next_offset = block.offset + block.length self.next_record = block.first_record + block.record_count @@ -242,22 +246,22 @@ def serialize(self, name, file_size, entry_count): for item in self.blocks: if size <= self.settings.max_bytes: break - size -= len(item[2]) - self.optional_bytes -= len(item[2]) + size -= _payload_size(item[2]) + self.optional_bytes -= _payload_size(item[2]) item[2] = b'' for item in self.blocks: if size <= self.settings.max_bytes: break - size -= len(item[3]) - self.optional_bytes -= len(item[3]) + size -= _payload_size(item[3]) + self.optional_bytes -= _payload_size(item[3]) item[3] = b'' if size > self.settings.max_bytes: size -= self.dictionary_bytes self.dictionary_bytes = 0 self.dictionary.clear() for item in self.blocks: - size -= len(item[1]) - self.optional_bytes -= len(item[1]) + size -= _payload_size(item[1]) + self.optional_bytes -= _payload_size(item[1]) item[1] = b'' _require(size <= self.settings.max_bytes) data = bytearray(HEADER.pack( @@ -272,8 +276,10 @@ def serialize(self, name, file_size, entry_count): for block, partitions, row_ids, buckets in self.blocks: data.extend(BLOCK.pack(block.offset, block.length, block.record_count)) for payload in (partitions, row_ids, buckets): - data.extend(struct.pack('>BI', 1 if payload else 0, len(payload))) - data.extend(payload) + data.append(1 if payload else 0) + if payload: + data.extend(struct.pack('>I', len(payload))) + data.extend(payload) return bytes(data) + hashlib.sha256(data).digest() @@ -355,72 +361,78 @@ def select(data, manifest, query, settings, partition_filter=None, partition_fie offset += BLOCK.size _require(file_offset == next_offset and 0 < length <= size - file_offset) _require(0 < count <= entries - first_record) - partition_encoding, payload_length = struct.unpack_from('>BI', data, offset) - offset += 5 - _require(payload_length <= limit - offset) + partition_encoding = data[offset] + offset += 1 partition_hit = True - if partition_encoding == 0: - _require(payload_length == 0) - elif partition_encoding == 1: - _require(payload_length >= 4) - ids, = struct.unpack_from('>I', data, offset) - _require(0 < ids <= partitions and 4 + 4 * ids == payload_length) - partition_hit = partition_filter is None - previous = -1 - for j in range(ids): - id_, = struct.unpack_from('>i', data, offset + 4 + 4 * j) - _require(previous < id_ < partitions) - previous = id_ - if not partition_hit: - partition_hit = matches[id_] - offset += payload_length - _require(offset + 5 <= limit) - row_encoding, payload_length = struct.unpack_from('>BI', data, offset) - offset += 5 - _require(payload_length <= limit - offset) + if partition_encoding != 0: + _require(offset + 4 <= limit) + payload_length, = struct.unpack_from('>I', data, offset) + offset += 4 + _require(payload_length <= limit - offset) + if partition_encoding == 1: + _require(payload_length >= 4) + ids, = struct.unpack_from('>I', data, offset) + _require(0 < ids <= partitions and 4 + 4 * ids == payload_length) + partition_hit = partition_filter is None + previous = -1 + for j in range(ids): + id_, = struct.unpack_from('>i', data, offset + 4 + 4 * j) + _require(previous < id_ < partitions) + previous = id_ + if not partition_hit: + partition_hit = matches[id_] + offset += payload_length + _require(offset + 1 <= limit) + row_encoding = data[offset] + offset += 1 row_hit = True - if row_encoding == 0: - _require(payload_length == 0) - elif row_encoding == 1: - _require(payload_length >= 4) - ranges, = struct.unpack_from('>I', data, offset) - _require(ranges > 0 and 4 + 16 * ranges == payload_length) - min_row_id, first_end = PAIR.unpack_from(data, offset + 4) - _require(min_row_id >= 0 and first_end >= min_row_id) - candidate = False - if partition_hit and query is not None: - max_row_id = first_end if ranges == 1 else LONG.unpack_from(data, offset + payload_length - 8)[0] - _require(max_row_id >= first_end) - candidate = query.intersects(min_row_id, max_row_id) - row_hit = candidate and (ranges == 1 or query.intersects(min_row_id, first_end)) - previous = first_end - for range_position in range(1, ranges): - start, end = PAIR.unpack_from(data, offset + 4 + 16 * range_position) - _require(start >= 0 and end >= start and start > previous) - previous = end - if candidate and not row_hit: - row_hit = query.intersects(start, end) - offset += payload_length - _require(offset + 5 <= limit) - bucket_encoding, payload_length = struct.unpack_from('>BI', data, offset) - offset += 5 - _require(payload_length <= limit - offset) + if row_encoding != 0: + _require(offset + 4 <= limit) + payload_length, = struct.unpack_from('>I', data, offset) + offset += 4 + _require(payload_length <= limit - offset) + if row_encoding == 1: + _require(payload_length >= 4) + ranges, = struct.unpack_from('>I', data, offset) + _require(ranges > 0 and 4 + 16 * ranges == payload_length) + min_row_id, first_end = PAIR.unpack_from(data, offset + 4) + _require(min_row_id >= 0 and first_end >= min_row_id) + candidate = False + if partition_hit and query is not None: + max_row_id = first_end if ranges == 1 else LONG.unpack_from(data, offset + payload_length - 8)[0] + _require(max_row_id >= first_end) + candidate = query.intersects(min_row_id, max_row_id) + row_hit = candidate and (ranges == 1 or query.intersects(min_row_id, first_end)) + previous = first_end + for range_position in range(1, ranges): + start, end = PAIR.unpack_from(data, offset + 4 + 16 * range_position) + _require(start >= 0 and end >= start and start > previous) + previous = end + if candidate and not row_hit: + row_hit = query.intersects(start, end) + offset += payload_length + _require(offset + 1 <= limit) + bucket_encoding = data[offset] + offset += 1 bucket_hit = True - if bucket_encoding == 0: - _require(payload_length == 0) - elif bucket_encoding == 1: - _require(payload_length >= 4) - pairs, = struct.unpack_from('>I', data, offset) - _require(pairs > 0 and payload_length == 4 + 8 * pairs) - bucket_hit = not partition_hit or not row_hit or bucket_filter is None - previous = (-1, -1) - for j in range(pairs): - bucket, total_buckets = struct.unpack_from('>ii', data, offset + 4 + 8 * j) - _require(0 <= bucket < total_buckets and (bucket, total_buckets) > previous) - previous = (bucket, total_buckets) - if not bucket_hit: - bucket_hit = bucket_filter(bucket, total_buckets) - offset += payload_length + if bucket_encoding != 0: + _require(offset + 4 <= limit) + payload_length, = struct.unpack_from('>I', data, offset) + offset += 4 + _require(payload_length <= limit - offset) + if bucket_encoding == 1: + _require(payload_length >= 4) + pairs, = struct.unpack_from('>I', data, offset) + _require(pairs > 0 and payload_length == 4 + 8 * pairs) + bucket_hit = not partition_hit or not row_hit or bucket_filter is None + previous = (-1, -1) + for j in range(pairs): + bucket, total_buckets = struct.unpack_from('>ii', data, offset + 4 + 8 * j) + _require(0 <= bucket < total_buckets and (bucket, total_buckets) > previous) + previous = (bucket, total_buckets) + if not bucket_hit: + bucket_hit = bucket_filter(bucket, total_buckets) + offset += payload_length if partition_hit and row_hit and bucket_hit: selected.append(Block(file_offset, length, first_record, count)) next_offset = file_offset + length diff --git a/paimon-python/pypaimon/tests/manifest/manifest_block_index_test.py b/paimon-python/pypaimon/tests/manifest/manifest_block_index_test.py index 688e2351b30e..b298e3515b5b 100644 --- a/paimon-python/pypaimon/tests/manifest/manifest_block_index_test.py +++ b/paimon-python/pypaimon/tests/manifest/manifest_block_index_test.py @@ -126,13 +126,16 @@ def positions(data): result = [] for _ in range(count): block = offset - p = offset + 24 - r = p + 5 + struct.unpack_from('>I', data, p + 1)[0] - offset = r + 5 + struct.unpack_from('>I', data, r + 1)[0] - bucket = offset - length, = struct.unpack_from('>I', data, offset + 1) - offset += 5 + length - result.append((block, p, r, bucket)) + offset += 24 + payloads = [] + for _ in range(3): + payloads.append(offset) + encoding = data[offset] + offset += 1 + if encoding != 0: + length, = struct.unpack_from('>I', data, offset) + offset += 4 + length + result.append((block, *payloads)) return result @@ -232,14 +235,14 @@ def test_unsigned_unknown_encodings_skip_only_one_payload_and_validate_lengths() def test_optional_payload_exhaustion_never_truncates_the_block_directory(): - settings, header = Settings(max_bytes=280), avro_header() + settings, header = Settings(max_bytes=250), avro_header() builder = Builder(settings, header) for i in range(3): builder.begin_block(len(header) + 100 * i, 100, 1) builder.add(100 * i, 10, partition(7, 'left')) builder.end_block() data = builder.serialize('m', len(header) + 300, 3) - assert len(data) <= 280 + assert len(data) <= 250 selected = select(data, meta('m', len(header) + 300, 3), [Range(999, 999)], settings, part(99), FIELDS) assert [b.first_record for b in selected.blocks] == [0, 1, 2] assert builder.serialize('m', len(header) + 300, 3) == data @@ -316,7 +319,9 @@ def test_bucket_budget_and_unknown_pairs_degrade_only_bucket_payload(pair): builder.add(300, 10, partition(7, 'left'), 1, 4) builder.end_block() data = builder.serialize('m', len(header) + 200, 3 + extra_pairs) - assert struct.unpack_from('>BI', data, positions(data)[0][3]) == (0, 0) + bucket = positions(data)[0][3] + assert data[bucket] == 0 + assert positions(data)[1][0] == bucket + 1 metadata = meta('m', len(header) + 200, 3 + extra_pairs) selected = select(data, metadata, None, settings, bucket_filter=lambda bucket, total: False) assert [b.first_record for b in selected.blocks] == [0] @@ -347,6 +352,37 @@ def test_payloads_can_exceed_former_limits_within_byte_budget(): bucket_filter=lambda bucket, total: bucket == entries_per_block).blocks +def test_absent_payloads_omit_length_fields_for_every_dimension_combination(): + header, settings = avro_header(), Settings() + builder = Builder(settings, header) + for mask in range(8): + builder.begin_block(len(header) + mask * 100, 100, 1) + builder.add(100 + mask if mask & 2 else None, 1, + partition(7, 'left') if mask & 1 else None, + 0 if mask & 4 else None, 1 if mask & 4 else None) + builder.end_block() + data = builder.serialize('m', len(header) + 800, 8) + locations = positions(data) + for mask in range(8): + for dimension, present_size in enumerate((13, 25, 17)): + start = locations[mask][dimension + 1] + end = (locations[mask][dimension + 2] if dimension < 2 + else locations[mask + 1][0] if mask < 7 else len(data) - 32) + present = bool(mask & (1 << dimension)) + assert data[start] == int(present) + assert end - start == (present_size if present else 1) + metadata = meta('m', len(header) + 800, 8) + selected = select(data, metadata, None, settings, part(99), FIELDS) + assert [b.first_record for b in selected.blocks] == [0, 2, 4, 6] + selected = select(data, metadata, [Range(999, 999)], settings) + assert [b.first_record for b in selected.blocks] == [0, 1, 4, 5] + buckets = lambda bucket, total: False + selected = select(data, metadata, None, settings, None, FIELDS, buckets) + assert [b.first_record for b in selected.blocks] == [0, 1, 2, 3] + selected = select(data, metadata, [Range(999, 999)], settings, part(99), FIELDS, buckets) + assert [b.first_record for b in selected.blocks] == [0] + + def test_malformed_bucket_payload_invalidates_the_container(): good = fixture('indexWithBuckets') payload = positions(good)[0][3] + 1 diff --git a/paimon-python/pypaimon/tests/manifest/manifest_sidecar_test.py b/paimon-python/pypaimon/tests/manifest/manifest_sidecar_test.py index 5e2177c842b7..43ccf2b8b269 100644 --- a/paimon-python/pypaimon/tests/manifest/manifest_sidecar_test.py +++ b/paimon-python/pypaimon/tests/manifest/manifest_sidecar_test.py @@ -315,7 +315,7 @@ def test_single_interval_decodes_pair_once(self): bounds.unpack_from.assert_not_called() def test_rejected_and_early_hit_blocks_still_validate_every_interval(self): - first_block_intervals = 60 + 4 + len(avro_header()) + 4 + 4 + 24 + 5 + 5 + 4 + first_block_intervals = 60 + 4 + len(avro_header()) + 4 + 4 + 24 + 1 + 5 + 4 for relative_offset, value in [(0, -1), (8, -1), (8, 30), (16, 9), (24, 19)]: data = bytearray(golden()) struct.pack_into('>q', data, first_block_intervals + relative_offset, value) From 4e8776a2362b2e48ec3dfdec71521af101b011b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Mon, 14 Sep 2026 15:42:56 +0800 Subject: [PATCH 17/23] [core] Skip unused manifest sidecar payload decoding --- docs/docs/concepts/spec/manifest.md | 8 +- .../paimon/manifest/ManifestSidecar.java | 153 +++++++++--------- .../manifest/ManifestBlockIndexTest.java | 51 +++++- .../paimon/manifest/ManifestSidecarTest.java | 3 +- .../pypaimon/manifest/manifest_sidecar.py | 141 ++++++++-------- .../manifest/manifest_block_index_test.py | 39 ++++- .../tests/manifest/manifest_sidecar_test.py | 7 +- 7 files changed, 233 insertions(+), 169 deletions(-) diff --git a/docs/docs/concepts/spec/manifest.md b/docs/docs/concepts/spec/manifest.md index 5a8e2007a478..78043c67fe95 100644 --- a/docs/docs/concepts/spec/manifest.md +++ b/docs/docs/concepts/spec/manifest.md @@ -131,8 +131,10 @@ Only nonzero encodings are followed by a length and payload. Payload lengths exc the encoding and length fields. Invalid lengths, known-payload framing, dictionary references, interval order, checksums or physical coverage invalidate the container. Byte spans must cover the entire original manifest after its header; -record counts must sum to the manifest entry count. Readers continue validating blocks -and known payloads even when a predicate has already rejected a block. +record counts must sum to the manifest entry count. Readers validate the checksum, +payload framing (including known count/length consistency), and the complete block directory +even when a block is rejected. Block payload contents are decoded and validated only for +dimensions still needed by the filters. Bucket encoding 1 contains a positive `pairCount: int` followed by that many `(bucket: int, totalBuckets: int)` pairs. Pairs are sorted by bucket, then totalBuckets, @@ -169,7 +171,7 @@ also incurs bounded object/buffer overhead. Query concurrency multiplies per-rea For conjunctive filters a block is retained only if each dimension is either unavailable or matches. Matching skips absent filters and short-circuits after a dimension rejects a -block, while known payloads remain validated. Matches in different dimensions can come +block, skipping the contents of later payloads. Matches in different dimensions can come from different entries in the block, so entry filtering and deletion merging remain necessary. Block min/max is derived from the first/last interval before testing the individual intervals. diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java index d9a12a34e19c..c21041cc6b53 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java @@ -508,7 +508,7 @@ public static Selection select( return select(data, manifest, query, null, null, settings); } - /** Validates framing and known payloads before applying independently available dimensions. */ + /** Validates framing and applies independently available dimensions in order. */ public static Selection select( byte[] data, ManifestFileMeta manifest, @@ -589,102 +589,99 @@ public static Selection select( long records = in.getLong(); require(offset == nextOffset && length > 0 && length <= manifest.fileSize() - offset); require(records > 0 && records <= entries - firstRecord); - int partitionEncoding = Byte.toUnsignedInt(in.get()); - boolean partitionHit = true; - if (partitionEncoding != 0) { - ByteBuffer partitionPayload = payload(in); - if (partitionEncoding == 1) { - require(partitionPayload.remaining() >= 4); - int ids = partitionPayload.getInt(); - require( - ids > 0 - && ids <= partitions - && partitionPayload.remaining() == 4L * ids); - partitionHit = partitionFilter == null; - int previous = -1; - for (int j = 0; j < ids; j++) { - int id = partitionPayload.getInt(); - require(id > previous && id < partitions); - previous = id; - if (!partitionHit) { - partitionHit = matches[id]; - } + ByteBuffer partitionPayload = payload(in, Integer.BYTES); + ByteBuffer rowPayload = payload(in, 2 * Long.BYTES); + ByteBuffer bucketPayload = payload(in, 2 * Integer.BYTES); + long blockFirstRecord = firstRecord; + nextOffset = offset + length; + firstRecord += records; + + if (partitionFilter != null && partitionPayload != null) { + boolean partitionHit = false; + int previous = -1; + while (partitionPayload.hasRemaining()) { + int id = partitionPayload.getInt(); + require(id > previous && id < partitions); + previous = id; + if (!partitionHit) { + partitionHit = matches[id]; } } + if (!partitionHit) { + continue; + } } - require(in.remaining() >= 1); - int rowEncoding = Byte.toUnsignedInt(in.get()); - boolean rowHit = true; - if (rowEncoding != 0) { - ByteBuffer rowPayload = payload(in); - if (rowEncoding == 1) { - require(rowPayload.remaining() >= 4); - int ranges = rowPayload.getInt(); - require(ranges > 0 && rowPayload.remaining() == 16L * ranges); - long min = rowPayload.getLong(); - long firstEnd = rowPayload.getLong(); - require(min >= 0 && firstEnd >= min); - boolean candidate = false; - if (partitionHit && query != null) { - long max = - ranges == 1 ? firstEnd : rowPayload.getLong(rowPayload.limit() - 8); - require(max >= firstEnd); - candidate = query.intersects(min, max); - rowHit = candidate && (ranges == 1 || query.intersects(min, firstEnd)); - } - long previous = firstEnd; - for (int rangeIndex = 1; rangeIndex < ranges; rangeIndex++) { - long start = rowPayload.getLong(); - long end = rowPayload.getLong(); - require(start >= 0 && end >= start && start > previous); - previous = end; - if (candidate && !rowHit) { - rowHit = query.intersects(start, end); - } + + if (query != null && rowPayload != null) { + boolean singleRange = rowPayload.remaining() == 2 * Long.BYTES; + long min = rowPayload.getLong(); + long firstEnd = rowPayload.getLong(); + long max = + singleRange + ? firstEnd + : rowPayload.getLong(rowPayload.limit() - Long.BYTES); + require(min >= 0 && firstEnd >= min && max >= firstEnd); + boolean candidate = query.intersects(min, max); + boolean rowHit = candidate && (singleRange || query.intersects(min, firstEnd)); + long previous = firstEnd; + while (rowPayload.hasRemaining()) { + long rangeStart = rowPayload.getLong(); + long rangeEnd = rowPayload.getLong(); + require(rangeStart >= 0 && rangeEnd >= rangeStart && rangeStart > previous); + previous = rangeEnd; + if (candidate && !rowHit) { + rowHit = query.intersects(rangeStart, rangeEnd); } } + if (!rowHit) { + continue; + } } - require(in.remaining() >= 1); - int bucketEncoding = Byte.toUnsignedInt(in.get()); - boolean bucketHit = true; - if (bucketEncoding != 0) { - ByteBuffer bucketPayload = payload(in); - if (bucketEncoding == 1) { - require(bucketPayload.remaining() >= 4); - int pairs = bucketPayload.getInt(); - require(pairs > 0 && bucketPayload.remaining() == 8L * pairs); - bucketHit = !partitionHit || !rowHit || bucketFilter == null; - long previous = -1; - for (int j = 0; j < pairs; j++) { - int bucket = bucketPayload.getInt(); - int totalBuckets = bucketPayload.getInt(); - require(bucket >= 0 && totalBuckets > bucket); - long pair = ((long) bucket << 32) | totalBuckets; - require(pair > previous); - previous = pair; - if (!bucketHit) { - bucketHit = bucketFilter.mayContain(bucket, totalBuckets); - } + + if (bucketFilter != null && bucketPayload != null) { + boolean bucketHit = false; + long previous = -1; + while (bucketPayload.hasRemaining()) { + int bucket = bucketPayload.getInt(); + int totalBuckets = bucketPayload.getInt(); + require(bucket >= 0 && totalBuckets > bucket); + long pair = ((long) bucket << 32) | totalBuckets; + require(pair > previous); + previous = pair; + if (!bucketHit) { + bucketHit = bucketFilter.mayContain(bucket, totalBuckets); } } + if (!bucketHit) { + continue; + } } - if (partitionHit && rowHit && bucketHit) { - selected.add(new Block(offset, length, firstRecord, records)); - } - nextOffset = offset + length; - firstRecord += records; + selected.add(new Block(offset, length, blockFirstRecord, records)); } require(!in.hasRemaining() && nextOffset == manifest.fileSize() && firstRecord == entries); return new Selection(header, selected, selected.size() == count); } - private static ByteBuffer payload(ByteBuffer in) throws IOException { - require(in.remaining() >= 4); + /** Reads framing and exposes known payload elements without decoding their contents. */ + @Nullable + private static ByteBuffer payload(ByteBuffer in, int elementBytes) throws IOException { + require(in.hasRemaining()); + int encoding = Byte.toUnsignedInt(in.get()); + if (encoding == 0) { + return null; + } + require(in.remaining() >= Integer.BYTES); int length = in.getInt(); require(length >= 0 && length <= in.remaining()); ByteBuffer result = in.slice(); result.limit(length); in.position(in.position() + length); + if (encoding != 1) { + return null; + } + require(result.remaining() >= Integer.BYTES); + int count = result.getInt(); + require(count > 0 && result.remaining() == (long) elementBytes * count); return result; } diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java index e9f2dc9cc592..c3d4241f860c 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java @@ -231,6 +231,10 @@ private byte[] checksum(byte[] data) throws Exception { @Test void partitionMissSkipsRowAndBucketMatching() throws Exception { byte[] data = fixture("indexWithBuckets"); + int[] first = positions(data).get(0); + ByteBuffer.wrap(data).putLong(first[2] + 9, -1); + ByteBuffer.wrap(data).putInt(first[3] + 9, -1); + checksum(data); ManifestFileMeta meta = meta("manifest-golden", fixture("avroHeader").length + 400, 7); RowRangeIndex rows = spy(query(0)); BucketFilter buckets = mock(BucketFilter.class); @@ -244,6 +248,8 @@ void partitionMissSkipsRowAndBucketMatching() throws Exception { @Test void rowMissSkipsBucketMatchingWithOrWithoutPartitionFilter() throws Exception { byte[] data = fixture("indexWithBuckets"); + ByteBuffer.wrap(data).putInt(positions(data).get(0)[3] + 9, -1); + checksum(data); ManifestFileMeta meta = meta("manifest-golden", fixture("avroHeader").length + 400, 7); for (PartitionPredicate partition : Arrays.asList(null, part(7))) { BucketFilter buckets = mock(BucketFilter.class); @@ -260,6 +266,8 @@ data, meta, query(15), partition, type, buckets, @Test void absentPartitionFilterKeepsRowAndBucketMatching() throws Exception { byte[] data = fixture("indexWithBuckets"); + ByteBuffer.wrap(data).putInt(positions(data).get(0)[1] + 9, 999); + checksum(data); ManifestFileMeta meta = meta("manifest-golden", fixture("avroHeader").length + 400, 7); BucketFilter buckets = spy(BucketFilter.create(false, 1, null, null)); assertThat( @@ -299,6 +307,34 @@ void absentRowOrBucketFiltersKeepRemainingDimensions() throws Exception { .containsExactly(0L, 3L, 5L); } + @Test + void skippedPayloadsStillRequireValidFramingAndDirectory() throws Exception { + byte[] good = fixture("indexWithBuckets"); + int[] first = positions(good).get(0); + ManifestFileMeta meta = meta("manifest-golden", fixture("avroHeader").length + 400, 7); + List invalid = new ArrayList<>(); + byte[] bad = good.clone(); + ByteBuffer.wrap(bad).putInt(first[3] + 1, -1); + invalid.add(bad); + bad = good.clone(); + ByteBuffer.wrap(bad).putInt(first[2] + 5, 0); + invalid.add(bad); + bad = good.clone(); + ByteBuffer.wrap(bad).putLong(positions(good).get(1)[0], 0); + invalid.add(bad); + bad = good.clone(); + ByteBuffer.wrap(bad).putLong(first[0] + 16, 2); + invalid.add(bad); + for (byte[] data : invalid) { + checksum(data); + assertThatThrownBy( + () -> + ManifestSidecar.select( + data, meta, query(0), part(99), type, defaults)) + .isInstanceOf(IOException.class); + } + } + @Test void unknownUnsignedEncodingsSkipOnlyTheirDimensionAndMalformedPayloadsFail() throws Exception { byte[] good = fixture("indexWithBuckets"); @@ -369,12 +405,15 @@ data, meta, query(20), part(99), type, noBucket, defaults) assertThatThrownBy( () -> ManifestSidecar.select( - badRange, meta, query(999), part(99), type, defaults)) + badRange, meta, query(999), part(7), type, defaults)) .isInstanceOf(IOException.class); byte[] badId = good.clone(); ByteBuffer.wrap(badId).putInt(first[1] + 9, 999); checksum(badId); - assertThatThrownBy(() -> ManifestSidecar.select(badId, meta, query(999), defaults)) + assertThatThrownBy( + () -> + ManifestSidecar.select( + badId, meta, query(999), part(7), type, defaults)) .isInstanceOf(IOException.class); } @@ -537,7 +576,13 @@ void malformedBucketPayloadInvalidatesTheContainer() throws Exception { assertThatThrownBy( () -> ManifestSidecar.select( - bad, meta, query(999), part(99), type, defaults)) + bad, + meta, + null, + null, + type, + BucketFilter.create(false, 99, null, null), + defaults)) .isInstanceOf(IOException.class); } } diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java index c5283ee21c3e..40211d39d30a 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java @@ -262,8 +262,7 @@ void malformedIntervalsStillFallbackAfterMinMaxRejectionOrAnEarlyHit() throws Ex Arrays.asList( RowRangeIndex.create(Collections.singletonList(new Range(30, 30))), RowRangeIndex.create(Collections.singletonList(new Range(0, 0))), - RowRangeIndex.create(Collections.emptyList()), - null)) { + RowRangeIndex.create(Collections.emptyList()))) { assertThat( ManifestSidecar.read( LocalFileIO.create(), diff --git a/paimon-python/pypaimon/manifest/manifest_sidecar.py b/paimon-python/pypaimon/manifest/manifest_sidecar.py index 8ec2c2010463..8bf16ad619e5 100644 --- a/paimon-python/pypaimon/manifest/manifest_sidecar.py +++ b/paimon-python/pypaimon/manifest/manifest_sidecar.py @@ -355,92 +355,85 @@ def select(data, manifest, query, settings, partition_filter=None, partition_fie next_offset = header_length first_record = 0 selected = [] + view = memoryview(data) for _ in range(blocks): _require(offset + BLOCK_BYTES <= limit) file_offset, length, count = BLOCK.unpack_from(data, offset) offset += BLOCK.size _require(file_offset == next_offset and 0 < length <= size - file_offset) _require(0 < count <= entries - first_record) - partition_encoding = data[offset] - offset += 1 - partition_hit = True - if partition_encoding != 0: - _require(offset + 4 <= limit) - payload_length, = struct.unpack_from('>I', data, offset) - offset += 4 - _require(payload_length <= limit - offset) - if partition_encoding == 1: - _require(payload_length >= 4) - ids, = struct.unpack_from('>I', data, offset) - _require(0 < ids <= partitions and 4 + 4 * ids == payload_length) - partition_hit = partition_filter is None - previous = -1 - for j in range(ids): - id_, = struct.unpack_from('>i', data, offset + 4 + 4 * j) - _require(previous < id_ < partitions) - previous = id_ - if not partition_hit: - partition_hit = matches[id_] - offset += payload_length - _require(offset + 1 <= limit) - row_encoding = data[offset] - offset += 1 - row_hit = True - if row_encoding != 0: - _require(offset + 4 <= limit) - payload_length, = struct.unpack_from('>I', data, offset) - offset += 4 - _require(payload_length <= limit - offset) - if row_encoding == 1: - _require(payload_length >= 4) - ranges, = struct.unpack_from('>I', data, offset) - _require(ranges > 0 and 4 + 16 * ranges == payload_length) - min_row_id, first_end = PAIR.unpack_from(data, offset + 4) - _require(min_row_id >= 0 and first_end >= min_row_id) - candidate = False - if partition_hit and query is not None: - max_row_id = first_end if ranges == 1 else LONG.unpack_from(data, offset + payload_length - 8)[0] - _require(max_row_id >= first_end) - candidate = query.intersects(min_row_id, max_row_id) - row_hit = candidate and (ranges == 1 or query.intersects(min_row_id, first_end)) - previous = first_end - for range_position in range(1, ranges): - start, end = PAIR.unpack_from(data, offset + 4 + 16 * range_position) - _require(start >= 0 and end >= start and start > previous) - previous = end - if candidate and not row_hit: - row_hit = query.intersects(start, end) - offset += payload_length - _require(offset + 1 <= limit) - bucket_encoding = data[offset] - offset += 1 - bucket_hit = True - if bucket_encoding != 0: - _require(offset + 4 <= limit) - payload_length, = struct.unpack_from('>I', data, offset) - offset += 4 - _require(payload_length <= limit - offset) - if bucket_encoding == 1: - _require(payload_length >= 4) - pairs, = struct.unpack_from('>I', data, offset) - _require(pairs > 0 and payload_length == 4 + 8 * pairs) - bucket_hit = not partition_hit or not row_hit or bucket_filter is None - previous = (-1, -1) - for j in range(pairs): - bucket, total_buckets = struct.unpack_from('>ii', data, offset + 4 + 8 * j) - _require(0 <= bucket < total_buckets and (bucket, total_buckets) > previous) - previous = (bucket, total_buckets) - if not bucket_hit: - bucket_hit = bucket_filter(bucket, total_buckets) - offset += payload_length - if partition_hit and row_hit and bucket_hit: - selected.append(Block(file_offset, length, first_record, count)) + partition_payload, offset = _payload(view, offset, limit, 4) + row_payload, offset = _payload(view, offset, limit, 16) + bucket_payload, offset = _payload(view, offset, limit, 8) + block_first_record = first_record next_offset = file_offset + length first_record += count + + if partition_filter is not None and partition_payload is not None: + partition_hit = False + previous = -1 + for position in range(0, len(partition_payload), 4): + id_, = struct.unpack_from('>i', partition_payload, position) + _require(previous < id_ < partitions) + previous = id_ + if not partition_hit: + partition_hit = matches[id_] + if not partition_hit: + continue + + if query is not None and row_payload is not None: + single_range = len(row_payload) == 16 + min_row_id, first_end = PAIR.unpack_from(row_payload) + max_row_id = first_end if single_range else LONG.unpack_from(row_payload, len(row_payload) - 8)[0] + _require(min_row_id >= 0 and first_end >= min_row_id and max_row_id >= first_end) + candidate = query.intersects(min_row_id, max_row_id) + row_hit = candidate and (single_range or query.intersects(min_row_id, first_end)) + previous = first_end + for position in range(16, len(row_payload), 16): + start, end = PAIR.unpack_from(row_payload, position) + _require(start >= 0 and end >= start and start > previous) + previous = end + if candidate and not row_hit: + row_hit = query.intersects(start, end) + if not row_hit: + continue + + if bucket_filter is not None and bucket_payload is not None: + bucket_hit = False + previous = (-1, -1) + for position in range(0, len(bucket_payload), 8): + bucket, total_buckets = struct.unpack_from('>ii', bucket_payload, position) + _require(0 <= bucket < total_buckets and (bucket, total_buckets) > previous) + previous = (bucket, total_buckets) + if not bucket_hit: + bucket_hit = bucket_filter(bucket, total_buckets) + if not bucket_hit: + continue + selected.append(Block(file_offset, length, block_first_record, count)) _require(offset == limit and next_offset == size and first_record == entries) return Selection(header, tuple(selected)) +def _payload(data, offset, limit, element_bytes): + """Read framing and expose known payload elements without decoding their contents.""" + _require(offset < limit) + encoding = data[offset] + offset += 1 + if encoding == 0: + return None, offset + _require(offset + 4 <= limit) + length, = struct.unpack_from('>I', data, offset) + offset += 4 + _require(length <= limit - offset) + end = offset + length + if encoding != 1: + return None, end + _require(length >= 4) + count, = struct.unpack_from('>I', data, offset) + _require(count > 0 and length == 4 + element_bytes * count) + return data[offset + 4:end], end + + def sidecar_file_name(manifest): return next((name for name in manifest.extra_files or [] if name.endswith(SUFFIX)), None) diff --git a/paimon-python/pypaimon/tests/manifest/manifest_block_index_test.py b/paimon-python/pypaimon/tests/manifest/manifest_block_index_test.py index b298e3515b5b..f0a761f7813c 100644 --- a/paimon-python/pypaimon/tests/manifest/manifest_block_index_test.py +++ b/paimon-python/pypaimon/tests/manifest/manifest_block_index_test.py @@ -145,30 +145,43 @@ def checksum(data): def test_partition_miss_skips_row_and_bucket_matching(): + data = bytearray(fixture('indexWithBuckets')) + _, _, row, bucket = positions(data)[0] + struct.pack_into('>q', data, row + 9, -1) + struct.pack_into('>i', data, bucket + 9, -1) + checksum(data) rows = Query([Range(0, 0)]) buckets = Mock(return_value=True) with patch.object(rows, 'intersects', wraps=rows.intersects) as intersects, \ - patch.object(manifest_sidecar, 'LONG', wraps=manifest_sidecar.LONG) as bounds: - selected = select(fixture('indexWithBuckets'), golden_meta(), rows, Settings(), part(99), FIELDS, buckets) + patch.object(manifest_sidecar, 'LONG', wraps=manifest_sidecar.LONG) as bounds, \ + patch.object(manifest_sidecar, 'PAIR', wraps=manifest_sidecar.PAIR) as pairs: + selected = select(data, golden_meta(), rows, Settings(), part(99), FIELDS, buckets) assert not selected.blocks intersects.assert_not_called() bounds.unpack_from.assert_not_called() + pairs.unpack_from.assert_not_called() buckets.assert_not_called() @pytest.mark.parametrize('partition_filter', [None, part(7)]) def test_row_miss_skips_bucket_matching(partition_filter): + data = bytearray(fixture('indexWithBuckets')) + struct.pack_into('>i', data, positions(data)[0][3] + 9, -1) + checksum(data) buckets = Mock(return_value=True) - selected = select(fixture('indexWithBuckets'), golden_meta(), [Range(15, 15)], + selected = select(data, golden_meta(), [Range(15, 15)], Settings(), partition_filter, FIELDS, buckets) assert not selected.blocks buckets.assert_not_called() def test_absent_partition_filter_keeps_row_and_bucket_matching(): + data = bytearray(fixture('indexWithBuckets')) + struct.pack_into('>i', data, positions(data)[0][1] + 9, 999) + checksum(data) buckets = Mock(side_effect=lambda bucket, total: bucket == 1) with patch('pypaimon.manifest.manifest_sidecar.GenericRowDeserializer.from_bytes') as decode_partition: - selected = select(fixture('indexWithBuckets'), golden_meta(), [Range(20, 20)], + selected = select(data, golden_meta(), [Range(20, 20)], Settings(), None, FIELDS, buckets) assert [b.first_record for b in selected.blocks] == [0] decode_partition.assert_not_called() @@ -189,6 +202,18 @@ def test_absent_row_or_bucket_filters_keep_remaining_dimensions(): assert [b.first_record for b in selected.blocks] == [0, 3, 5] +def test_skipped_payloads_still_require_valid_framing_and_directory(): + good = fixture('indexWithBuckets') + block, _, row, bucket = positions(good)[0] + mutations = [('>i', bucket + 1, -1), ('>i', row + 5, 0), + ('>q', positions(good)[1][0], 0), ('>q', block + 16, 2)] + for fmt, position, value in mutations: + data = bytearray(good) + struct.pack_into(fmt, data, position, value) + with pytest.raises(ValueError): + select(checksum(data), golden_meta(), [Range(0, 0)], Settings(), part(99), FIELDS) + + def test_unsigned_unknown_encodings_skip_only_one_payload_and_validate_lengths(): good = fixture('indexWithBuckets') block, p, r, b = positions(good)[0] @@ -227,11 +252,11 @@ def test_unsigned_unknown_encodings_skip_only_one_payload_and_validate_lengths() data = bytearray(good) struct.pack_into('>i', data, p + 9, 999) # out-of-dictionary ID with pytest.raises(ValueError): - select(checksum(data), golden_meta(), None, Settings()) + select(checksum(data), golden_meta(), None, Settings(), part(7), FIELDS) data = bytearray(good) struct.pack_into('>q', data, r + 9 + 16, 9) # overlap first interval, even in a rejected block with pytest.raises(ValueError): - select(checksum(data), golden_meta(), [Range(999, 999)], Settings(), part(99), FIELDS) + select(checksum(data), golden_meta(), [Range(999, 999)], Settings(), part(7), FIELDS) def test_optional_payload_exhaustion_never_truncates_the_block_directory(): @@ -391,4 +416,4 @@ def test_malformed_bucket_payload_invalidates_the_container(): bad = bytearray(good) struct.pack_into('>i', bad, offset, value) with pytest.raises(ValueError): - select(checksum(bad), golden_meta(), [Range(999, 999)], Settings(), part(99), FIELDS) + select(checksum(bad), golden_meta(), None, Settings(), bucket_filter=lambda bucket, total: False) diff --git a/paimon-python/pypaimon/tests/manifest/manifest_sidecar_test.py b/paimon-python/pypaimon/tests/manifest/manifest_sidecar_test.py index 43ccf2b8b269..2df90ab4d17c 100644 --- a/paimon-python/pypaimon/tests/manifest/manifest_sidecar_test.py +++ b/paimon-python/pypaimon/tests/manifest/manifest_sidecar_test.py @@ -311,7 +311,10 @@ def test_single_interval_decodes_pair_once(self): patch.object(manifest_sidecar, 'LONG', wraps=manifest_sidecar.LONG) as bounds: selected = select(data, meta, ranges, Settings()) self.assertEqual(len(selected.blocks), expected) - pairs.unpack_from.assert_called_once() + if ranges is None: + pairs.unpack_from.assert_not_called() + else: + pairs.unpack_from.assert_called_once() bounds.unpack_from.assert_not_called() def test_rejected_and_early_hit_blocks_still_validate_every_interval(self): @@ -320,7 +323,7 @@ def test_rejected_and_early_hit_blocks_still_validate_every_interval(self): data = bytearray(golden()) struct.pack_into('>q', data, first_block_intervals + relative_offset, value) data[-32:] = hashlib.sha256(data[:-32]).digest() - for ranges in ([Range(30, 30)], [Range(0, 0)], None, []): + for ranges in ([Range(30, 30)], [Range(0, 0)], []): with self.subTest(offset=relative_offset, value=value, ranges=ranges), \ self.assertRaises(ValueError): select(data, golden_meta(), ranges, Settings()) From d99bb142305f8eb1464891b34a06dce7850690d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Mon, 14 Sep 2026 15:48:45 +0800 Subject: [PATCH 18/23] Fix minus --- .../apache/paimon/manifest/ManifestFile.java | 20 +------------------ .../LegacyDataEvolutionRowIdReassigner.java | 3 ++- 2 files changed, 3 insertions(+), 20 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java index cd96bd611ab1..9dcdaaa9e379 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java @@ -131,25 +131,7 @@ public List read( bucketFilter, readFilter, readTFilter, - Function.identity()); - } - - public List read( - String fileName, - @Nullable Long fileSize, - @Nullable PartitionPredicate partitionFilter, - @Nullable BucketFilter bucketFilter, - Filter readFilter, - Filter readTFilter, - Function convertor) { - return read( - fileName, - fileSize, - partitionFilter, - bucketFilter, - readFilter, - readTFilter, - convertor, + Function.identity(), null); } diff --git a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/LegacyDataEvolutionRowIdReassigner.java b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/LegacyDataEvolutionRowIdReassigner.java index 641541499028..e113923f31cd 100644 --- a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/LegacyDataEvolutionRowIdReassigner.java +++ b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/LegacyDataEvolutionRowIdReassigner.java @@ -886,7 +886,8 @@ private List readPlanningManifestEntries( null, Filter.alwaysTrue(), entry -> partitionPredicate == null || partitionPredicate.test(entry.partition()), - ManifestEntry::copyWithoutStats); + ManifestEntry::copyWithoutStats, + null); } private Comparator entryComparator() { From a0e153b8c4033f66937339102daeb2077d13d3d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Mon, 14 Sep 2026 16:13:06 +0800 Subject: [PATCH 19/23] [core] Share the manifest cache with sidecar reads --- docs/docs/concepts/spec/manifest.md | 5 + .../apache/paimon/manifest/ManifestFile.java | 3 +- .../paimon/manifest/ManifestSidecar.java | 75 ++++++++++---- .../org/apache/paimon/utils/ObjectsCache.java | 4 + .../paimon/manifest/ManifestFileTest.java | 58 +++++++++++ .../paimon/manifest/ManifestSidecarTest.java | 98 +++++++++++++++++++ 6 files changed, 221 insertions(+), 22 deletions(-) diff --git a/docs/docs/concepts/spec/manifest.md b/docs/docs/concepts/spec/manifest.md index 78043c67fe95..bffe6bff639a 100644 --- a/docs/docs/concepts/spec/manifest.md +++ b/docs/docs/concepts/spec/manifest.md @@ -181,6 +181,11 @@ therefore reads row-ID payload bytes too; payload lengths save decoding work for encodings, not storage I/O. Selected compressed blocks are read by byte range with adjacent spans coalesced. Existing immutable manifests are not backfilled by enabling the write option. +Java readers share the existing manifest cache for complete sidecar bytes, keyed by the +explicit sidecar path and subject to the same memory budget and single-file threshold. +Only successful reads and selections populate the cache. Each query creates independent +views and reapplies its filters and byte budget; query-specific selections are not cached. + Java selections covering every block can reuse the full-manifest cache; partial selections bypass it. PyPaimon explain scans disable sidecar pruning to preserve complete entry counters. diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java index 9dcdaaa9e379..301b05de8e35 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java @@ -390,7 +390,8 @@ public ManifestSidecar.Selection selectBlocks( partitionFilter, partitionType, bucketFilter, - sidecarSettings); + sidecarSettings, + cache == null ? null : cache.segmentsCache()); } public boolean mayContainRowIds(ManifestFileMeta manifest, @Nullable RowRangeIndex query) { diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java index c21041cc6b53..df497970a163 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java @@ -20,6 +20,7 @@ import org.apache.paimon.CoreOptions; import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.data.Segments; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.SeekableInputStream; @@ -27,6 +28,7 @@ import org.apache.paimon.partition.PartitionPredicate; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.RowRangeIndex; +import org.apache.paimon.utils.SegmentsCache; import org.apache.paimon.utils.SerializationUtils; import org.slf4j.Logger; @@ -705,7 +707,8 @@ public static Selection read( @Nullable PartitionPredicate partitionFilter, @Nullable RowType partitionType, Settings settings) { - return read(io, path, manifest, query, partitionFilter, partitionType, null, settings); + return read( + io, path, manifest, query, partitionFilter, partitionType, null, settings, null); } @Nullable @@ -717,31 +720,33 @@ public static Selection read( @Nullable PartitionPredicate partitionFilter, @Nullable RowType partitionType, @Nullable BucketFilter bucketFilter, - Settings settings) { + Settings settings, + @Nullable SegmentsCache cache) { String sidecarFileName = fileName(manifest); if (sidecarFileName == null) { return null; } try { - byte[] data; - try (InputStream in = io.newInputStream(new Path(path.getParent(), sidecarFileName))) { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - byte[] buffer = new byte[Math.min(READ_BUFFER_BYTES, settings.maxBytes + 1)]; - int n; - while ((n = - in.read( - buffer, - 0, - Math.min( - buffer.length, settings.maxBytes + 1 - out.size()))) - != -1) { - out.write(buffer, 0, n); - require(out.size() <= settings.maxBytes); - } - data = out.toByteArray(); - } - return select( - data, manifest, query, partitionFilter, partitionType, bucketFilter, settings); + Path sidecarPath = new Path(path.getParent(), sidecarFileName); + Segments cached = cache == null ? null : cache.getIfPresents(sidecarPath); + boolean cacheHit = cached instanceof CachedBytes; + byte[] data = + cacheHit + ? ((CachedBytes) cached).data + : readBytes(io, sidecarPath, settings.maxBytes); + Selection selection = + select( + data, + manifest, + query, + partitionFilter, + partitionType, + bucketFilter, + settings); + if (cache != null && !cacheHit && data.length <= cache.maxElementSize()) { + cache.put(sidecarPath, new CachedBytes(data)); + } + return selection; } catch (CancellationException failure) { throw failure; } catch (IOException | RuntimeException failure) { @@ -779,6 +784,34 @@ public static Selection read( } } + private static byte[] readBytes(FileIO io, Path path, int maxBytes) throws IOException { + try (InputStream in = io.newInputStream(path)) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buffer = new byte[Math.min(READ_BUFFER_BYTES, maxBytes + 1)]; + int n; + while ((n = in.read(buffer, 0, Math.min(buffer.length, maxBytes + 1 - out.size()))) + != -1) { + out.write(buffer, 0, n); + require(out.size() <= maxBytes); + } + return out.toByteArray(); + } + } + + /** Immutable bytes shared by queries, weighted within the existing manifest cache. */ + private static final class CachedBytes implements Segments { + private final byte[] data; + + private CachedBytes(byte[] data) { + this.data = data; + } + + @Override + public long totalMemorySize() { + return data.length; + } + } + private static UncheckedIOException interrupted(Throwable failure) { InterruptedIOException interrupted = new InterruptedIOException("Interrupted reading manifest sidecar"); diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/ObjectsCache.java b/paimon-core/src/main/java/org/apache/paimon/utils/ObjectsCache.java index 4530f698e291..a27d0d791b88 100644 --- a/paimon-core/src/main/java/org/apache/paimon/utils/ObjectsCache.java +++ b/paimon-core/src/main/java/org/apache/paimon/utils/ObjectsCache.java @@ -63,6 +63,10 @@ public void withCacheMetrics(@Nullable CacheMetrics cacheMetrics) { this.cacheMetrics = cacheMetrics; } + public SegmentsCache segmentsCache() { + return cache; + } + public List read(K key, @Nullable Long fileSize, Filters filters) throws IOException { return read(key, fileSize, filters, Function.identity()); } diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java index a1f4b4778ef5..9c1f86eb609a 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java @@ -1378,6 +1378,64 @@ void testDisabledSidecarsDoNotConstrainManifestTargetSize() { } } + @Test + void testSidecarCacheUsesExplicitPathsAndSharesTheManifestCache() throws Exception { + Options options = new Options(); + options.set(CoreOptions.MANIFEST_SIDECAR_WRITE, true); + options.set(CoreOptions.MANIFEST_SIDECAR_READ, true); + RecordingFileIO io = new RecordingFileIO(); + SegmentsCache cache = + new SegmentsCache<>(1024, MemorySize.ofMebiBytes(1), Long.MAX_VALUE, null, false); + ManifestFile.Factory factory = + createManifestFileFactory(tempDir.toString(), Long.MAX_VALUE, options, io, cache); + List entries = new ArrayList<>(); + for (int i = 0; i < 2; i++) { + ManifestEntry entry = gen.next(); + entries.add( + ManifestEntry.create( + FileKind.ADD, + entry.partition(), + entry.bucket(), + entry.totalBuckets(), + entry.file().newFirstRowId(i * 1000000L))); + } + ManifestFileMeta written = factory.create().write(entries).get(0); + String sidecarName = "cached-explicit" + ManifestSidecar.SUFFIX; + java.nio.file.Path sidecar = tempDir.resolve("manifest").resolve(sidecarName); + java.nio.file.Files.move( + tempDir.resolve("manifest").resolve(ManifestSidecar.fileName(written)), sidecar); + ManifestFileMeta meta = withExtraFiles(written, Collections.singletonList(sidecarName)); + Path sidecarPath = new Path(tempDir.toString(), "manifest/" + sidecarName); + + io.reset(); + assertThat( + factory.create() + .selectBlocks( + meta, + RowRangeIndex.create( + Collections.singletonList( + new Range(Long.MAX_VALUE, Long.MAX_VALUE)))) + .blocks()) + .isEmpty(); + assertThat(io.opened).containsExactly(sidecarPath); + assertThat(cache.getIfPresents(sidecarPath).totalMemorySize()) + .isEqualTo(java.nio.file.Files.size(sidecar)); + + io.reset(); + RowRangeIndex hit = RowRangeIndex.create(Collections.singletonList(new Range(0, 0))); + assertThat(factory.create().selectBlocks(meta, hit).blocks()).isNotEmpty(); + assertThat(io.opened).isEmpty(); + assertThat(factory.create().read(meta.fileName())) + .containsExactlyInAnyOrderElementsOf(entries); + assertThat(cache.estimatedSize()).isEqualTo(2); + + io.reset(); + assertThat(factory.create().selectBlocks(meta, hit).blocks()).isNotEmpty(); + assertThat(factory.create().read(meta.fileName())) + .containsExactlyInAnyOrderElementsOf(entries); + assertThat(io.opened).isEmpty(); + } + @Test void testReadsOnlySelectedBlocksAndPreservesPhysicalOrdinals() throws Exception { Options options = new Options(); diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java index 40211d39d30a..8999b130e49a 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java @@ -28,6 +28,7 @@ import org.apache.paimon.utils.IOUtils; import org.apache.paimon.utils.Range; import org.apache.paimon.utils.RowRangeIndex; +import org.apache.paimon.utils.SegmentsCache; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -333,6 +334,103 @@ void hugeRangesAreNotExpandedAndInvalidCoverageRetainsBlocks() throws Exception assertThat(builder.serialize("m", 1, 2)).isNull(); } + @Test + void cacheRespectsElementThresholdAndPerReadByteBudget() throws Exception { + byte[] data = golden(); + Path path = new Path(temp.toString(), "manifest-golden"); + Path sidecar = ManifestSidecar.path(path); + Files.write(temp.resolve(sidecar.getName()), data); + ManifestFileMeta meta = goldenMeta(); + FileIO io = spy(LocalFileIO.create()); + SegmentsCache tooSmall = + new SegmentsCache<>(1024, MemorySize.ofMebiBytes(1), data.length - 1L, null, false); + assertThat(readCached(io, path, meta, settings, tooSmall).blocks()).hasSize(2); + assertThat(readCached(io, path, meta, settings, tooSmall).blocks()).hasSize(2); + assertThat(tooSmall.getIfPresents(sidecar)).isNull(); + verify(io, times(2)).newInputStream(sidecar); + + SegmentsCache cache = + new SegmentsCache<>(1024, MemorySize.ofMebiBytes(1), data.length, null, false); + assertThat(readCached(io, path, meta, settings, cache).blocks()).hasSize(2); + Options options = new Options(); + options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES, new MemorySize(data.length - 1)); + assertThat(readCached(io, path, meta, new ManifestSidecar.Settings(options), cache)) + .isNull(); + assertThat(readCached(io, path, meta, settings, cache).blocks()).hasSize(2); + verify(io, times(3)).newInputStream(sidecar); + } + + @Test + void missingAndInvalidSidecarsAreNotCached() throws Exception { + byte[] data = golden(); + Path path = new Path(temp.toString(), "manifest-golden"); + Path sidecar = ManifestSidecar.path(path); + ManifestFileMeta meta = goldenMeta(); + FileIO io = spy(LocalFileIO.create()); + SegmentsCache cache = + new SegmentsCache<>(1024, MemorySize.ofMebiBytes(1), Long.MAX_VALUE, null, false); + assertThat(readCached(io, path, meta, settings, cache)).isNull(); + assertThat(cache.getIfPresents(sidecar)).isNull(); + byte[] corrupt = data.clone(); + corrupt[0] ^= 1; + Files.write(temp.resolve(sidecar.getName()), corrupt); + assertThat(readCached(io, path, meta, settings, cache)).isNull(); + assertThat(cache.getIfPresents(sidecar)).isNull(); + Files.write(temp.resolve(sidecar.getName()), data); + assertThat(readCached(io, path, meta, settings, cache).blocks()).hasSize(2); + assertThat(readCached(io, path, meta, settings, cache).blocks()).hasSize(2); + verify(io, times(3)).newInputStream(sidecar); + } + + @Test + void cachedBytesPreservePerQueryCancellation() throws Exception { + Path path = new Path(temp.toString(), "manifest-golden"); + Path sidecar = ManifestSidecar.path(path); + Files.write(temp.resolve(sidecar.getName()), golden()); + ManifestFileMeta meta = goldenMeta(); + FileIO io = spy(LocalFileIO.create()); + SegmentsCache cache = + new SegmentsCache<>(1024, MemorySize.ofMebiBytes(1), Long.MAX_VALUE, null, false); + RowRangeIndex cancelled = mock(RowRangeIndex.class); + when(cancelled.intersects(anyLong(), anyLong())) + .thenThrow(new CancellationException("cancelled")); + assertThatThrownBy( + () -> + ManifestSidecar.read( + io, path, meta, cancelled, null, null, null, settings, + cache)) + .isInstanceOf(CancellationException.class); + assertThat(cache.getIfPresents(sidecar)).isNull(); + + assertThat(readCached(io, path, meta, settings, cache).blocks()).hasSize(2); + assertThatThrownBy( + () -> + ManifestSidecar.read( + io, path, meta, cancelled, null, null, null, settings, + cache)) + .isInstanceOf(CancellationException.class); + assertThat(readCached(io, path, meta, settings, cache).blocks()).hasSize(2); + verify(io, times(2)).newInputStream(sidecar); + } + + private ManifestSidecar.Selection readCached( + FileIO io, + Path path, + ManifestFileMeta meta, + ManifestSidecar.Settings settings, + SegmentsCache cache) { + return ManifestSidecar.read( + io, + path, + meta, + RowRangeIndex.create(Collections.singletonList(new Range(20, 20))), + null, + null, + null, + settings, + cache); + } + @Test void corruptMissingIncompleteAndMismatchedIndexesFallback() throws Exception { Path manifest = new Path(temp.toString(), "manifest-golden"); From 1306d7c99b9a68192ea80d8d7a5eb1521d14e644 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Mon, 14 Sep 2026 16:25:36 +0800 Subject: [PATCH 20/23] [core] Reuse SingleSegments for cached sidecar bytes --- .../paimon/manifest/ManifestSidecar.java | 34 ++++++++----------- .../paimon/manifest/ManifestSidecarTest.java | 19 +++++++++++ 2 files changed, 33 insertions(+), 20 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java index df497970a163..b941fe565234 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java @@ -21,9 +21,11 @@ import org.apache.paimon.CoreOptions; import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.Segments; +import org.apache.paimon.data.SingleSegments; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.SeekableInputStream; +import org.apache.paimon.memory.MemorySegment; import org.apache.paimon.options.Options; import org.apache.paimon.partition.PartitionPredicate; import org.apache.paimon.types.RowType; @@ -729,11 +731,17 @@ public static Selection read( try { Path sidecarPath = new Path(path.getParent(), sidecarFileName); Segments cached = cache == null ? null : cache.getIfPresents(sidecarPath); - boolean cacheHit = cached instanceof CachedBytes; - byte[] data = - cacheHit - ? ((CachedBytes) cached).data - : readBytes(io, sidecarPath, settings.maxBytes); + boolean cacheHit = cached instanceof SingleSegments; + byte[] data; + if (cacheHit) { + SingleSegments segments = (SingleSegments) cached; + byte[] bytes = segments.segment().getHeapMemory(); + int length = segments.limit(); + require(length >= 0 && length <= bytes.length && length <= settings.maxBytes); + data = length == bytes.length ? bytes : Arrays.copyOf(bytes, length); + } else { + data = readBytes(io, sidecarPath, settings.maxBytes); + } Selection selection = select( data, @@ -744,7 +752,7 @@ public static Selection read( bucketFilter, settings); if (cache != null && !cacheHit && data.length <= cache.maxElementSize()) { - cache.put(sidecarPath, new CachedBytes(data)); + cache.put(sidecarPath, new SingleSegments(MemorySegment.wrap(data), data.length)); } return selection; } catch (CancellationException failure) { @@ -798,20 +806,6 @@ private static byte[] readBytes(FileIO io, Path path, int maxBytes) throws IOExc } } - /** Immutable bytes shared by queries, weighted within the existing manifest cache. */ - private static final class CachedBytes implements Segments { - private final byte[] data; - - private CachedBytes(byte[] data) { - this.data = data; - } - - @Override - public long totalMemorySize() { - return data.length; - } - } - private static UncheckedIOException interrupted(Throwable failure) { InterruptedIOException interrupted = new InterruptedIOException("Interrupted reading manifest sidecar"); diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java index 8999b130e49a..196d1da6a794 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java @@ -19,10 +19,12 @@ package org.apache.paimon.manifest; import org.apache.paimon.CoreOptions; +import org.apache.paimon.data.SingleSegments; import org.apache.paimon.fs.ByteArraySeekableStream; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.memory.MemorySegment; import org.apache.paimon.options.MemorySize; import org.apache.paimon.options.Options; import org.apache.paimon.utils.IOUtils; @@ -56,6 +58,7 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; /** Cross-language format, physical block positions, completeness and allocation bounds. */ @@ -360,6 +363,22 @@ void cacheRespectsElementThresholdAndPerReadByteBudget() throws Exception { verify(io, times(3)).newInputStream(sidecar); } + @Test + void cachedSegmentsRespectTheirLogicalLimit() throws Exception { + byte[] data = golden(); + byte[] padded = Arrays.copyOf(data, data.length + 32); + Arrays.fill(padded, data.length, padded.length, (byte) 127); + Path path = new Path(temp.toString(), "manifest-golden"); + SegmentsCache cache = + new SegmentsCache<>(1024, MemorySize.ofMebiBytes(1), Long.MAX_VALUE, null, false); + cache.put( + ManifestSidecar.path(path), + new SingleSegments(MemorySegment.wrap(padded), data.length)); + FileIO io = mock(FileIO.class); + assertThat(readCached(io, path, goldenMeta(), settings, cache).blocks()).hasSize(2); + verifyNoInteractions(io); + } + @Test void missingAndInvalidSidecarsAreNotCached() throws Exception { byte[] data = golden(); From cd6ce6211fb95c21d4db70533b945197f1620a93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Mon, 14 Sep 2026 16:33:28 +0800 Subject: [PATCH 21/23] [core] Remove fixed manifest sidecar header and block limits --- docs/docs/concepts/spec/manifest.md | 6 +-- .../paimon/manifest/ManifestAvroReader.java | 1 - .../paimon/manifest/ManifestSidecar.java | 23 +++----- .../paimon/manifest/ManifestFileTest.java | 53 +++++++++++++++++++ .../paimon/manifest/ManifestSidecarTest.java | 2 +- .../org/apache/avro/file/RawBlockReader.java | 14 ++--- .../paimon/format/avro/AvroBlockReader.java | 1 - .../pypaimon/manifest/manifest_sidecar.py | 10 ++-- .../tests/manifest/manifest_sidecar_test.py | 28 +++++++++- 9 files changed, 101 insertions(+), 37 deletions(-) diff --git a/docs/docs/concepts/spec/manifest.md b/docs/docs/concepts/spec/manifest.md index bffe6bff639a..e225e4ea4610 100644 --- a/docs/docs/concepts/spec/manifest.md +++ b/docs/docs/concepts/spec/manifest.md @@ -163,11 +163,11 @@ excludes a block with unavailable partition coverage. Later blocks can still use partition dictionary and all three payload types. It accepts memory sizes such as `16 mb` and, when unset, defaults to twice the configured `manifest.target-file-size` (16 MiB with the default 8 MiB manifest target). An explicit sidecar size overrides this default. -The Avro header is also capped at 1 MiB and the directory at 131072 blocks. +The Avro header and block directory share this byte budget without separate size or count limits. Writers discard optional row-ID payloads, bucket payloads, then partition payloads/dictionary if necessary, to fit the complete directory. If the directory itself cannot fit, no sidecar is published. -No emitted sidecar omits block descriptors. These are encoded-size bounds; construction -also incurs bounded object/buffer overhead. Query concurrency multiplies per-reader costs. +No emitted sidecar omits block descriptors. These are encoded-size bounds; Avro header parsing +and sidecar construction also incur object/buffer overhead. Query concurrency multiplies per-reader costs. For conjunctive filters a block is retained only if each dimension is either unavailable or matches. Matching skips absent filters and short-circuits after a dimension rejects a diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestAvroReader.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestAvroReader.java index 79500c97a60a..26e02f4ec418 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestAvroReader.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestAvroReader.java @@ -70,7 +70,6 @@ public final class ManifestAvroReader implements AutoCloseable { } } - @Nullable public byte[] headerBytes() { return blockReader.headerBytes(); } diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java index b941fe565234..077f30e8cc17 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java @@ -73,10 +73,8 @@ public final class ManifestSidecar { private static final int FORMAT_VERSION = 1; private static final int HEADER_BYTES = 60; private static final int BLOCK_BYTES = 27; - private static final int MAX_BLOCKS = 131072; private static final byte[] EMPTY = new byte[0]; private static final int DIGEST_BYTES = 32; - private static final int MAX_AVRO_HEADER = 1024 * 1024; private static final int READ_BUFFER_BYTES = 1024 * 1024; private ManifestSidecar() {} @@ -181,7 +179,6 @@ public Builder(Settings settings, @Nullable byte[] header) { this.header = header; complete = header != null - && header.length <= MAX_AVRO_HEADER && HEADER_BYTES + DIGEST_BYTES + 12L + header.length <= settings.maxBytes; nextOffset = header == null ? 0 : header.length; @@ -197,13 +194,12 @@ public void beginBlock(long offset, long length, long records) throws IOExceptio } require(current == null && offset == nextOffset && length > 0 && records > 0); // Optional payloads can be discarded later, but descriptors must never be truncated. - if (blocks.size() == MAX_BLOCKS - || HEADER_BYTES - + DIGEST_BYTES - + 12L - + header.length - + (blocks.size() + 1L) * BLOCK_BYTES - > settings.maxBytes) { + if (HEADER_BYTES + + DIGEST_BYTES + + 12L + + header.length + + (blocks.size() + 1L) * BLOCK_BYTES + > settings.maxBytes) { complete = false; blocks.clear(); dictionary.clear(); @@ -551,10 +547,7 @@ public static Selection select( long entries = Math.addExact(manifest.numAddedFiles(), manifest.numDeletedFiles()); require(in.getLong() == entries); int headerLength = in.getInt(); - require( - headerLength >= 21 - && headerLength <= MAX_AVRO_HEADER - && headerLength <= in.remaining() - 8); + require(headerLength >= 21 && headerLength <= in.remaining() - 8); byte[] header = new byte[headerLength]; in.get(header); require(header[0] == 'O' && header[1] == 'b' && header[2] == 'j' && header[3] == 1); @@ -582,7 +575,7 @@ public static Selection select( } require(in.remaining() >= 4); int count = in.getInt(); - require(count >= 0 && count <= MAX_BLOCKS && count <= in.remaining() / BLOCK_BYTES); + require(count >= 0 && count <= in.remaining() / BLOCK_BYTES); long nextOffset = headerLength; long firstRecord = 0; List selected = new ArrayList<>(); diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java index 9c1f86eb609a..df07e4d58c1b 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java @@ -25,6 +25,8 @@ import org.apache.paimon.format.FileFormat; import org.apache.paimon.format.FormatWriter; import org.apache.paimon.format.SimpleColStats; +import org.apache.paimon.format.avro.AvroRowDatumWriter; +import org.apache.paimon.format.avro.AvroSchemaConverter; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.FileIOFinder; import org.apache.paimon.fs.Path; @@ -54,6 +56,7 @@ import org.apache.paimon.utils.RowRangeIndex; import org.apache.paimon.utils.SegmentsCache; +import org.apache.avro.file.DataFileWriter; import org.junit.jupiter.api.RepeatedTest; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -1378,6 +1381,56 @@ void testDisabledSidecarsDoNotConstrainManifestTargetSize() { } } + @Test + void testSidecarSupportsLargeAvroHeadersWithinByteBudget() throws Exception { + ManifestEntry source = gen.next(); + ManifestEntry entry = + ManifestEntry.create( + FileKind.ADD, + source.partition(), + source.bucket(), + source.totalBuckets(), + source.file().newFirstRowId(42L)); + ManifestEntrySerializer serializer = new ManifestEntrySerializer(); + java.nio.file.Path file = tempDir.resolve("large-header.avro"); + try (DataFileWriter writer = + new DataFileWriter<>(new AvroRowDatumWriter(ManifestEntry.MANIFEST_ROW_TYPE))) { + writer.setMeta("large-metadata", new byte[1 << 20]); + writer.create( + AvroSchemaConverter.convertToSchema( + ManifestEntry.MANIFEST_ROW_TYPE, Collections.emptyMap()), + file.toFile()); + writer.append(serializer.toRow(entry)); + } + + LocalFileIO io = LocalFileIO.create(); + Path path = new Path(file.toUri()); + long size = io.getFileSize(path); + ManifestSidecar.Settings settings = new ManifestSidecar.Settings(new Options()); + byte[] data = ManifestSidecar.build(io, path, size, 1, settings); + assertThat(data).isNotNull(); + ManifestFileMeta meta = ManifestSidecarTest.meta(path.getName(), size, 1); + RowRangeIndex query = RowRangeIndex.create(Collections.singletonList(new Range(42, 42))); + ManifestSidecar.Selection selected = ManifestSidecar.select(data, meta, query, settings); + assertThat(selected.blocks()).hasSize(1); + assertThat(selected.blocks().get(0).offset).isGreaterThan(1 << 20); + try (ManifestAvroReader reader = + new ManifestAvroReader(ManifestSidecar.openManifest(io, path, selected)); + CloseableIterator rows = + reader.read(ManifestEntry.MANIFEST_ROW_TYPE, null, null)) { + assertThat(rows.hasNext()).isTrue(); + assertThat(serializer.fromRow(rows.next())).isEqualTo(entry); + assertThat(rows.hasNext()).isFalse(); + } + + Options options = new Options(); + options.set(CoreOptions.MANIFEST_SIDECAR_MAX_BYTES, MemorySize.ofMebiBytes(1)); + ManifestSidecar.Settings smallBudget = new ManifestSidecar.Settings(options); + assertThat(ManifestSidecar.build(io, path, size, 1, smallBudget)).isNull(); + assertThatThrownBy(() -> ManifestSidecar.select(data, meta, query, smallBudget)) + .isInstanceOf(IOException.class); + } + @Test void testSidecarCacheUsesExplicitPathsAndSharesTheManifestCache() throws Exception { Options options = new Options(); diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java index 196d1da6a794..a799eb730854 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java @@ -624,7 +624,7 @@ public org.apache.paimon.fs.SeekableInputStream newInputStream(Path path) @Test void indexReadsUseBoundedBulkRequests() throws Exception { byte[] header = header(); - for (int blockCount : new int[] {5000, 25000}) { + for (int blockCount : new int[] {5000, 25000, 131073}) { ManifestSidecar.Builder builder = new ManifestSidecar.Builder(settings, header); for (int blockNumber = 0; blockNumber < blockCount; blockNumber++) { builder.beginBlock(header.length + blockNumber * 100L, 100, 1); diff --git a/paimon-format/src/main/java/org/apache/avro/file/RawBlockReader.java b/paimon-format/src/main/java/org/apache/avro/file/RawBlockReader.java index 382f55f93d60..39d453472aa3 100644 --- a/paimon-format/src/main/java/org/apache/avro/file/RawBlockReader.java +++ b/paimon-format/src/main/java/org/apache/avro/file/RawBlockReader.java @@ -45,15 +45,12 @@ private RawBlockReader(CountingInput input) throws IOException { super(input, new NoOpDatumReader()); this.input = input; long length = position(); - this.headerBytes = - length <= CountingInput.MAX_HEADER - ? Arrays.copyOf(input.prefix.toByteArray(), (int) length) - : null; + this.headerBytes = Arrays.copyOf(input.prefix.toByteArray(), (int) length); input.prefix = null; } public byte[] headerBytes() { - return headerBytes == null ? null : headerBytes.clone(); + return headerBytes.clone(); } public long blockOffset() { @@ -90,7 +87,6 @@ public RawBlock nextRawBlock(RawBlock reuse) throws IOException { } private static final class CountingInput extends FilterInputStream { - private static final int MAX_HEADER = 1024 * 1024; private long position; private ByteArrayOutputStream prefix = new ByteArrayOutputStream(); @@ -103,7 +99,7 @@ public int read() throws IOException { int value = in.read(); if (value >= 0) { position++; - if (prefix != null && prefix.size() < MAX_HEADER) { + if (prefix != null) { prefix.write(value); } } @@ -115,8 +111,8 @@ public int read(byte[] bytes, int offset, int length) throws IOException { int n = in.read(bytes, offset, length); if (n > 0) { position += n; - if (prefix != null && prefix.size() < MAX_HEADER) { - prefix.write(bytes, offset, Math.min(n, MAX_HEADER - prefix.size())); + if (prefix != null) { + prefix.write(bytes, offset, n); } } return n; diff --git a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroBlockReader.java b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroBlockReader.java index eeca62bb18c8..8f0e0a96c158 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroBlockReader.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroBlockReader.java @@ -54,7 +54,6 @@ public AvroBlockReader(InputStream input) throws IOException { } } - @Nullable public byte[] headerBytes() { return reader.headerBytes(); } diff --git a/paimon-python/pypaimon/manifest/manifest_sidecar.py b/paimon-python/pypaimon/manifest/manifest_sidecar.py index 8bf16ad619e5..fd7b986f804f 100644 --- a/paimon-python/pypaimon/manifest/manifest_sidecar.py +++ b/paimon-python/pypaimon/manifest/manifest_sidecar.py @@ -37,11 +37,9 @@ MAGIC = b'PAIMSCAR' FORMAT_VERSION = 1 MAX_ROW_ID = (1 << 63) - 1 -MAX_AVRO_HEADER = 1024 * 1024 READ_BUFFER_BYTES = 1024 * 1024 HEADER = struct.Struct('>8sI32sqq') BLOCK = struct.Struct('>qqq') -MAX_BLOCKS = 131072 BLOCK_BYTES = 27 PAIR = struct.Struct('>qq') LONG = struct.Struct('>q') @@ -102,7 +100,7 @@ class Builder: def __init__(self, settings, header): self.settings = settings self.header = header - self.complete = (header is not None and len(header) <= MAX_AVRO_HEADER + self.complete = (header is not None and HEADER.size + 44 + len(header) <= settings.max_bytes) self.dictionary = {} self.dictionary_bytes = 0 @@ -119,7 +117,7 @@ def begin_block(self, offset, length, records): if not self.complete: return _require(self.current is None and offset == self.next_offset and length > 0 and records > 0) - if (len(self.blocks) == MAX_BLOCKS or HEADER.size + 44 + len(self.header) + if (HEADER.size + 44 + len(self.header) + (len(self.blocks) + 1) * BLOCK_BYTES > self.settings.max_bytes): self.complete = False self.blocks.clear() @@ -324,7 +322,7 @@ def select(data, manifest, query, settings, partition_filter=None, partition_fie _require(size == manifest.file_size and entries == manifest.num_added_files + manifest.num_deleted_files) header_length, = struct.unpack_from('>I', data, HEADER.size) offset = HEADER.size + 4 - _require(21 <= header_length <= MAX_AVRO_HEADER and header_length <= limit - offset - 8) + _require(21 <= header_length <= limit - offset - 8) header = bytes(data[offset:offset + header_length]) _require(header[:4] == b'Obj\x01') offset += header_length @@ -351,7 +349,7 @@ def select(data, manifest, query, settings, partition_filter=None, partition_fie _require(offset + 4 <= limit) blocks, = struct.unpack_from('>I', data, offset) offset += 4 - _require(blocks <= MAX_BLOCKS and blocks <= (limit - offset) // BLOCK_BYTES) + _require(blocks <= (limit - offset) // BLOCK_BYTES) next_offset = header_length first_record = 0 selected = [] diff --git a/paimon-python/pypaimon/tests/manifest/manifest_sidecar_test.py b/paimon-python/pypaimon/tests/manifest/manifest_sidecar_test.py index 2df90ab4d17c..983e246d4469 100644 --- a/paimon-python/pypaimon/tests/manifest/manifest_sidecar_test.py +++ b/paimon-python/pypaimon/tests/manifest/manifest_sidecar_test.py @@ -122,7 +122,7 @@ def close(self): class ManifestSidecarReadTest(unittest.TestCase): def test_index_reads_use_bounded_bulk_requests(self): header = avro_header() - for block_count in (5000, 25000): + for block_count in (5000, 25000, 131073): with self.subTest(block_count=block_count): builder = Builder(Settings(), header) for block_number in range(block_count): @@ -212,6 +212,32 @@ def test_block_short_reads_and_truncation(self): class ManifestSidecarFormatTest(unittest.TestCase): + def test_large_avro_headers_within_byte_budget(self): + stream = BytesIO() + fastavro.writer(stream, 'long', [42], metadata={'large-metadata': 'x' * (1 << 20)}) + avro_bytes = stream.getvalue() + block = next(fastavro.block_reader(BytesIO(avro_bytes))) + header = avro_bytes[:block.offset] + self.assertGreater(len(header), 1 << 20) + builder = Builder(Settings(), header) + builder.begin_block(block.offset, block.size, block.num_records) + builder.add(42, 1) + builder.end_block() + meta = SimpleNamespace(file_name='large-header.avro', file_size=len(avro_bytes), + num_added_files=1, num_deleted_files=0) + data = builder.serialize(meta.file_name, meta.file_size, 1) + self.assertIsNotNone(data) + selected = select(data, meta, [Range(42, 42)], Settings()) + self.assertEqual(selected.header, header) + file_io = SimpleNamespace(new_input_stream=lambda path: BytesIO(avro_bytes)) + restored = read_selected_bytes(file_io, meta.file_name, selected) + self.assertEqual(restored, avro_bytes) + self.assertEqual(list(fastavro.reader(BytesIO(restored))), [42]) + small_budget = Settings(max_bytes=1 << 20) + self.assertIsNone(Builder(small_budget, header).serialize(meta.file_name, meta.file_size, 1)) + with self.assertRaises(ValueError): + select(data, meta, [Range(42, 42)], small_budget) + def test_settings_use_memory_sizes_and_follow_the_manifest_target(self): options = CoreOptions(Options({})) self.assertEqual(Settings().max_bytes, 16 * 1024 * 1024) From 6383bc6b1580a7d65228747eaa222f03d911ac67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Mon, 14 Sep 2026 16:38:36 +0800 Subject: [PATCH 22/23] [core] Use 4 MiB manifest sidecar read buffers --- .../java/org/apache/paimon/manifest/ManifestSidecar.java | 2 +- .../org/apache/paimon/manifest/ManifestSidecarTest.java | 8 ++++---- paimon-python/pypaimon/manifest/manifest_sidecar.py | 2 +- .../pypaimon/tests/manifest/manifest_sidecar_test.py | 8 ++++---- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java index 077f30e8cc17..e4f26fa1c1c9 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java @@ -75,7 +75,7 @@ public final class ManifestSidecar { private static final int BLOCK_BYTES = 27; private static final byte[] EMPTY = new byte[0]; private static final int DIGEST_BYTES = 32; - private static final int READ_BUFFER_BYTES = 1024 * 1024; + private static final int READ_BUFFER_BYTES = 4 * 1024 * 1024; private ManifestSidecar() {} diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java index a799eb730854..94b6a6070cb4 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java @@ -647,8 +647,8 @@ void indexReadsUseBoundedBulkRequests() throws Exception { settings); assertThat(actual.blocks()).hasSize(1); assertThat(actual.blocks().get(0).offset).isEqualTo(header.length); - assertThat(stream.readLengths).hasSize((data.length + (1 << 20) - 1) / (1 << 20)); - assertThat(stream.requests).allMatch(request -> request <= 1 << 20); + assertThat(stream.readLengths).hasSize((data.length + (4 << 20) - 1) / (4 << 20)); + assertThat(stream.requests).allMatch(request -> request <= 4 << 20); assertThat(stream.closed).isTrue(); } } @@ -767,7 +767,7 @@ void largeBlockSpansUseBoundedReads() throws Exception { byte[] header = header(); ManifestSidecar.Builder builder = new ManifestSidecar.Builder(settings, header); long offset = header.length; - for (int length : new int[] {512 * 1024, 512 * 1024, 257}) { + for (int length : new int[] {2 * 1024 * 1024, 2 * 1024 * 1024, 257}) { builder.beginBlock(offset, length, 1); builder.add(20L, 1); builder.endBlock(); @@ -784,7 +784,7 @@ void largeBlockSpansUseBoundedReads() throws Exception { io, path, select(data, meta("manifest-large", offset, 3), 20))) { assertThat(IOUtils.readFully(input, false)).isEqualTo(manifest); } - assertThat(stream.readLengths).containsExactly(1 << 20, 257); + assertThat(stream.readLengths).containsExactly(4 << 20, 257); assertThat(stream.seeks).containsExactly((long) header.length); assertThat(stream.closed).isTrue(); } diff --git a/paimon-python/pypaimon/manifest/manifest_sidecar.py b/paimon-python/pypaimon/manifest/manifest_sidecar.py index fd7b986f804f..f2400a367819 100644 --- a/paimon-python/pypaimon/manifest/manifest_sidecar.py +++ b/paimon-python/pypaimon/manifest/manifest_sidecar.py @@ -37,7 +37,7 @@ MAGIC = b'PAIMSCAR' FORMAT_VERSION = 1 MAX_ROW_ID = (1 << 63) - 1 -READ_BUFFER_BYTES = 1024 * 1024 +READ_BUFFER_BYTES = 4 * 1024 * 1024 HEADER = struct.Struct('>8sI32sqq') BLOCK = struct.Struct('>qqq') BLOCK_BYTES = 27 diff --git a/paimon-python/pypaimon/tests/manifest/manifest_sidecar_test.py b/paimon-python/pypaimon/tests/manifest/manifest_sidecar_test.py index 983e246d4469..0987fccd9a31 100644 --- a/paimon-python/pypaimon/tests/manifest/manifest_sidecar_test.py +++ b/paimon-python/pypaimon/tests/manifest/manifest_sidecar_test.py @@ -139,8 +139,8 @@ def test_index_reads_use_bounded_bulk_requests(self): actual = read_sidecar(file_io, '/manifest/manifest-large', meta, [Range(0, 0)], Settings()) self.assertEqual(actual, select(data, meta, [Range(0, 0)], Settings())) - self.assertEqual(len(stream.reads), (len(data) + (1 << 20) - 1) // (1 << 20)) - self.assertLessEqual(max(stream.requests), 1 << 20) + self.assertEqual(len(stream.reads), (len(data) + (4 << 20) - 1) // (4 << 20)) + self.assertLessEqual(max(stream.requests), 4 << 20) self.assertTrue(stream.closed) def test_index_short_reads_and_exact_budget(self): @@ -185,7 +185,7 @@ def test_adjacent_blocks_share_reads_without_reading_gaps(self): def test_large_block_spans_use_bounded_reads(self): header = avro_header() - block_size = 512 * 1024 + block_size = 2 * 1024 * 1024 body = bytes(2 * block_size + 257) selected = Selection(header, (Block(len(header), block_size, 0, 1), Block(len(header) + block_size, block_size, 1, 1), @@ -193,7 +193,7 @@ def test_large_block_spans_use_bounded_reads(self): stream = CountingInput(header + body) file_io = SimpleNamespace(new_input_stream=lambda path: stream) self.assertEqual(read_selected_bytes(file_io, '/manifest/manifest-large', selected), header + body) - self.assertEqual(stream.reads, [(len(header), 1 << 20), (len(header) + (1 << 20), 257)]) + self.assertEqual(stream.reads, [(len(header), 4 << 20), (len(header) + (4 << 20), 257)]) self.assertEqual(stream.seeks, [len(header)]) self.assertTrue(stream.closed) From c300ff5eef745acff8a78e51a6d82f6281ea762b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Mon, 14 Sep 2026 16:47:35 +0800 Subject: [PATCH 23/23] [python] Reuse local file cache for manifest sidecars --- docs/docs/concepts/spec/manifest.md | 8 ++++ .../pypaimon/tests/file_type_test.py | 3 ++ .../tests/manifest/manifest_sidecar_test.py | 45 ++++++++++++++++++- paimon-python/pypaimon/utils/file_type.py | 4 +- 4 files changed, 57 insertions(+), 3 deletions(-) diff --git a/docs/docs/concepts/spec/manifest.md b/docs/docs/concepts/spec/manifest.md index e225e4ea4610..754867f1f7de 100644 --- a/docs/docs/concepts/spec/manifest.md +++ b/docs/docs/concepts/spec/manifest.md @@ -186,6 +186,14 @@ explicit sidecar path and subject to the same memory budget and single-file thre Only successful reads and selections populate the cache. Each query creates independent views and reapplies its filters and byte budget; query-specific selections are not cached. +PyPaimon reuses `CachingFileIO` for sidecar bytes. Enable `local-cache.enabled` on the catalog +and include `meta` in `local-cache.whitelist` (included by default). Files ending in +`.avro.sidecar`, including custom names, use the same cache as other metadata. The cache +stores raw byte blocks by full path and block index, sharing `local-cache.max-size` and +`local-cache.block-size`. Without `local-cache.dir` it uses memory; setting that option +enables disk caching. Sidecar validation, filters and the read byte budget are reapplied +on every query. Local caching is disabled by default. + Java selections covering every block can reuse the full-manifest cache; partial selections bypass it. PyPaimon explain scans disable sidecar pruning to preserve complete entry counters. diff --git a/paimon-python/pypaimon/tests/file_type_test.py b/paimon-python/pypaimon/tests/file_type_test.py index b82604eaede1..0efabd1833ee 100644 --- a/paimon-python/pypaimon/tests/file_type_test.py +++ b/paimon-python/pypaimon/tests/file_type_test.py @@ -48,6 +48,9 @@ def test_manifest(self): self.assertEqual(FileType.META, FileType.classify("manifest-abc123")) self.assertEqual(FileType.META, FileType.classify("manifest-list-abc")) self.assertEqual(FileType.META, FileType.classify("index-manifest-abc")) + self.assertEqual(FileType.META, FileType.classify("manifest-abc123.avro.sidecar")) + self.assertEqual(FileType.META, FileType.classify("custom.avro.sidecar")) + self.assertEqual(FileType.META, FileType.classify("index-abc123.avro.sidecar")) def test_hint_files(self): self.assertEqual(FileType.META, FileType.classify("EARLIEST")) diff --git a/paimon-python/pypaimon/tests/manifest/manifest_sidecar_test.py b/paimon-python/pypaimon/tests/manifest/manifest_sidecar_test.py index 0987fccd9a31..0989535c31dc 100644 --- a/paimon-python/pypaimon/tests/manifest/manifest_sidecar_test.py +++ b/paimon-python/pypaimon/tests/manifest/manifest_sidecar_test.py @@ -28,11 +28,13 @@ from pyarrow import ArrowCancelled from dataclasses import replace from pathlib import Path +from tempfile import TemporaryDirectory from types import SimpleNamespace -from unittest.mock import patch +from unittest.mock import Mock, call, patch from pypaimon.common.options.core_options import CoreOptions from pypaimon.common.options.options import Options +from pypaimon.filesystem.caching_file_io import CachingFileIO from pypaimon.globalindex.global_index_result import GlobalIndexResult from pypaimon.manifest import manifest_sidecar from pypaimon.manifest.manifest_sidecar import ( @@ -120,6 +122,47 @@ def close(self): class ManifestSidecarReadTest(unittest.TestCase): + def test_local_cache_shares_sidecar_bytes_across_queries_and_readers(self): + data, meta = golden(), golden_meta() + meta.extra_files = ['custom' + SUFFIX] + for disk in (False, True): + with self.subTest(disk=disk), TemporaryDirectory() as directory: + options = Options({'local-cache.enabled': True, 'local-cache.max-size': '1 mb', + 'local-cache.block-size': '128 bytes'}) + if disk: + options.set(CoreOptions.LOCAL_CACHE_DIR, directory) + cache = CachingFileIO.create_cache_manager(options) + delegate = SimpleNamespace(new_input_stream=Mock(side_effect=lambda path: BytesIO(data)), + get_file_size=Mock(return_value=len(data))) + for parent in ('/table-a', '/table-b'): + path = parent + '/' + meta.file_name + for point, expected in ((20, [0, 5]), (0, [0]), (16, [])): + file_io = CachingFileIO.wrap_with_caching_if_needed(delegate, options, cache) + selected = read_sidecar(file_io, path, meta, [Range(point, point)], Settings()) + self.assertEqual([b.first_record for b in selected.blocks], expected) + self.assertIsNone(read_sidecar(file_io, path, meta, [Range(20, 20)], Settings(max_bytes=128))) + expected_calls = [call('/table-a/custom' + SUFFIX), call('/table-b/custom' + SUFFIX)] + self.assertEqual(delegate.new_input_stream.call_args_list, expected_calls) + self.assertEqual(delegate.get_file_size.call_args_list, expected_calls) + + def test_local_cache_respects_disable_whitelist_and_byte_budget(self): + data, meta = golden(), golden_meta() + meta.extra_files = ['custom' + SUFFIX] + for overrides in ({'local-cache.enabled': False}, {'local-cache.whitelist': 'global-index'}, + {'local-cache.max-size': '128 bytes'}): + with self.subTest(overrides=overrides): + options = Options({'local-cache.enabled': True, 'local-cache.max-size': '1 mb', + 'local-cache.block-size': '128 bytes', **overrides}) + cache = CachingFileIO.create_cache_manager(options) + delegate = SimpleNamespace(new_input_stream=Mock(side_effect=lambda path: BytesIO(data)), + get_file_size=Mock(return_value=len(data))) + file_io = CachingFileIO.wrap_with_caching_if_needed(delegate, options, cache) + for _ in range(2): + selected = read_sidecar(file_io, '/table/' + meta.file_name, meta, + [Range(20, 20)], Settings()) + self.assertEqual([b.first_record for b in selected.blocks], [0, 5]) + self.assertEqual(delegate.new_input_stream.call_count, 2) + def test_index_reads_use_bounded_bulk_requests(self): header = avro_header() for block_count in (5000, 25000, 131073): diff --git a/paimon-python/pypaimon/utils/file_type.py b/paimon-python/pypaimon/utils/file_type.py index 92819f3618bc..7dadc93af631 100644 --- a/paimon-python/pypaimon/utils/file_type.py +++ b/paimon-python/pypaimon/utils/file_type.py @@ -28,7 +28,7 @@ class FileType(Enum): """Classification of Paimon files. - - META: snapshot, schema, manifest, statistics, tag, changelog metadata, + - META: snapshot, schema, manifest, manifest sidecar, statistics, tag, changelog metadata, hint files, _SUCCESS, consumer, service files - DATA: data files and any unrecognized files (default) - BUCKET_INDEX: bucket level index files (Hash, DV) @@ -67,7 +67,7 @@ def classify(file_path: str) -> 'FileType': return FileType.GLOBAL_INDEX return FileType.FILE_INDEX - if "manifest" in name: + if "manifest" in name or name.endswith(".avro.sidecar"): return FileType.META if name.startswith("index-"):